Several fixes and improvements

build_scripts/build_runner.py

@@ -43,7 +43,8 @@
 import logging
 from collections import OrderedDict
 from copy import deepcopy
-from datetime import datetime
+
+import dateutil.parser
 from tenacity import retry, stop_after_attempt, wait_exponential
 
 sys.path.append(str(pathlib.Path(__file__).resolve().parents[1]))
@@ -243,7 +244,7 @@ def __init__(self, build_config_path, root_dir, build_type, product_type, build_
         :type stage: String
 
         :param commit_time: Time for getting slice of commits of repositories
-        :type commit_time: datetime
+        :type commit_time: String
 
         :param changed_repo: Information about changed source repository
         :type changed_repo: String
@@ -260,13 +261,13 @@ def __init__(self, build_config_path, root_dir, build_type, product_type, build_
         """
 
         self._default_stage = Stage.BUILD.value
-        super().__init__(root_dir, build_config_path, stage)
+        super().__init__(root_dir, build_config_path, stage, custom_cli_args)
 
         self._product_repos = {}
         self._product = None
         self._product_type = product_type
         self._build_event = build_event
-        self._commit_time = commit_time
+        self._commit_time = dateutil.parser.parse(commit_time) if commit_time else None
         self._changed_repo = changed_repo
         self._repo_states = None
         self._build_state_file = root_dir / "build_state"
@@ -281,7 +282,6 @@ def __init__(self, build_config_path, root_dir, build_type, product_type, build_
         })
         self._dev_pkg_data_to_archive = []
         self._install_pkg_data_to_archive = []
-        self._custom_cli_args = custom_cli_args
         self._target_arch = target_arch
         self._target_branch = target_branch
         self._manifest_file = manifest_file
@@ -326,7 +326,6 @@ def _update_global_vars(self):
             'vs_component': self._vs_component,
             'stage': Stage,
             'copy_win_files': copy_win_files,
-            'args': self._custom_cli_args,
             'product_type': self._product_type,
             'build_event': self._build_event,
             # TODO should be in lower case
@@ -1013,7 +1012,7 @@ def main():
                         choices=[stage.value for stage in Stage],
                         help="Current executable stage")
     parser.add_argument('-t', "--commit-time", metavar='datetime',
-                        help="Time of commits (ex. 2017-11-02 07:36:40)")
+                        help="Time of commits (e.g. 2017-11-02 07:36:40)")
     parser.add_argument('-ta', "--target-arch",
                         nargs='*',
                         default=[target_arch.value for target_arch in TargetArch],
@@ -1022,15 +1021,15 @@ def main():
     parser.add_argument('-tb', "--target-branch",
                         help=f'All not triggered repos will be checkout to this branch.')
 
-    parsed_args, unknown_args = parser.parse_known_args()
+    args, unknown_args = parser.parse_known_args()
 
     configure_logger()
-    if parsed_args.stage != Stage.CLEAN.value:
-        configure_logger(logs_path=pathlib.Path(parsed_args.root_dir) / 'logs' / f'{parsed_args.stage}.log')
+    if args.stage != Stage.CLEAN.value:
+        configure_logger(logs_path=pathlib.Path(args.root_dir) / 'logs' / f'{args.stage}.log')
     log = logging.getLogger('build_runner.main')
 
     # remove duplicated values
-    target_arch = list(set(parsed_args.target_arch))
+    target_arch = list(set(args.target_arch))
 
     custom_cli_args = {}
     if unknown_args:
@@ -1042,57 +1041,52 @@ def main():
                 log.exception(f'Wrong argument layout: {arg}')
                 exit(ErrorCode.CRITICAL)
 
-    if parsed_args.commit_time:
-        commit_time = datetime.strptime(parsed_args.commit_time, '%Y-%m-%d %H:%M:%S')
-    else:
-        commit_time = None
-
     try:
         build_config = BuildGenerator(
-            build_config_path=pathlib.Path(parsed_args.build_config).absolute(),
-            root_dir=pathlib.Path(parsed_args.root_dir).absolute(),
-            build_type=parsed_args.build_type,
-            product_type=parsed_args.product_type,
-            build_event=parsed_args.build_event,
-            commit_time=commit_time,
-            changed_repo=parsed_args.changed_repo,
-            repo_states_file_path=parsed_args.repo_states,
+            build_config_path=pathlib.Path(args.build_config).absolute(),
+            root_dir=pathlib.Path(args.root_dir).absolute(),
+            build_type=args.build_type,
+            product_type=args.product_type,
+            build_event=args.build_event,
+            commit_time=args.commit_time,
+            changed_repo=args.changed_repo,
+            repo_states_file_path=args.repo_states,
             custom_cli_args=custom_cli_args,
-            stage=parsed_args.stage,
+            stage=args.stage,
             target_arch=target_arch,
-            target_branch=parsed_args.target_branch,
-            manifest_file=parsed_args.manifest,
-            component_name=parsed_args.component
+            target_branch=args.target_branch,
+            manifest_file=args.manifest,
+            component_name=args.component
         )
 
-        if not parsed_args.changed_repo \
-                and not parsed_args.repo_states \
-                and (not parsed_args.manifest or not parsed_args.component):
+        if not args.changed_repo \
+                and not args.repo_states \
+                and (not args.manifest or not args.component):
             log.warning('"--changed-repo" or "--repo-states" or "--manifest" and "--component" '
                         'arguments are not set, "HEAD" revision and "master" branch will be used')
-        elif parsed_args.changed_repo and parsed_args.repo_states:
+        elif args.changed_repo and args.repo_states:
             log.warning('The --repo-states argument is ignored because the --changed-repo is set')
 
         # prepare build configuration
         if build_config.generate_config():
-            no_errors = build_config.run_stage(parsed_args.stage)
+            no_errors = build_config.run_stage(args.stage)
         else:
             no_errors = False
 
     except Exception:
         no_errors = False
         log.exception('Exception occurred')
 
-    build_state_file = pathlib.Path(parsed_args.root_dir) / 'build_state'
+    build_state_file = pathlib.Path(args.root_dir) / 'build_state'
     if no_errors:
         if not build_state_file.exists():
             build_state_file.write_text(json.dumps({'status': "PASS"}))
         log.info('-' * 50)
-        log.info("%s STAGE COMPLETED", parsed_args.stage.upper())
+        log.info("%s STAGE COMPLETED", args.stage.upper())
     else:
         build_state_file.write_text(json.dumps({'status': "FAIL"}))
         log.error('-' * 50)
-        log.error("%s STAGE FAILED", parsed_args.stage.upper())
+        log.error("%s STAGE FAILED", args.stage.upper())
         exit(ErrorCode.CRITICAL.value)
 
 

build_scripts/common_runner.py

@@ -177,9 +177,11 @@ def _parse_logs(self, stdout):
 class ConfigGenerator:
     _default_stage = None
 
-    def __init__(self, root_dir, config_path, current_stage):
+   def __init__(self, root_dir, config_path, current_stage, custom_cli_args=None):
         self._config_path = config_path
         self._current_stage = current_stage
+        self._custom_cli_args = custom_cli_args or {}
+
         self._actions = defaultdict(list)
         self._options = {
             "ROOT_DIR": pathlib.Path(root_dir).absolute(),
@@ -193,9 +195,12 @@ def __init__(self, root_dir, config_path, current_stage):
 
         self._config_variables = {}
         self._global_vars = {
-            'action': self._action,
             'options': self._options,
-            'log': self._log
+            'args': self._custom_cli_args,
+            'log': self._log,
+            'configs_dir': self._config_path.parent,
+            'infra_dir': pathlib.Path(__file__).resolve().parents[1],
+            'action': self._action,
         }
 
         if not self._default_stage:

build_scripts/tests_runner.py

@@ -30,6 +30,8 @@
 import pathlib
 import argparse
 
+import dateutil.parser
+
 sys.path.append(str(pathlib.Path(__file__).resolve().parents[1]))
 from build_scripts.common_runner import ConfigGenerator, RunnerException
 from test_scripts.components_installer import install_components
@@ -60,10 +62,11 @@ class TestRunner(ConfigGenerator):
     Contains commands for testing product.
     """
 
-    def __init__(self, artifacts, root_dir, current_stage):
+    def __init__(self, artifacts, root_dir, current_stage, commit_time=None, custom_cli_args=None):
         self._artifacts_dir = None
         self._manifest = None
         self._infrastructure_path = pathlib.Path(__file__).resolve().parents[1]
+        self._commit_time = dateutil.parser.parse(commit_time) if commit_time else None
 
         if artifacts.exists():
             if artifacts.is_file():
@@ -85,13 +88,15 @@ def __init__(self, artifacts, root_dir, current_stage):
             raise ArtifactsNotFoundException(f'"{artifacts}" does not exist')
 
         self._default_stage = TestStage.TEST.value
-        super().__init__(root_dir, config_path, current_stage)
+        super().__init__(root_dir, config_path, current_stage, custom_cli_args)
 
     def _update_global_vars(self):
         self._global_vars.update({
             'stage': TestStage,
+            # TODO: remove infra_path
             'infra_path': self._infrastructure_path,
             'artifacts_dir': self._artifacts_dir,
+            'commit_time': self._commit_time,
         })
 
     def _clean(self):
@@ -166,17 +171,31 @@ def main():
     parser.add_argument("--stage", default=TestStage.TEST.value,
                         choices=[stage.value for stage in TestStage],
                         help="Current executable stage")
-    args = parser.parse_args()
+    parser.add_argument('-t', "--commit-time", metavar='datetime',
+                        help="Time of commits (e.g. 2017-11-02 07:36:40)")
+    args, unknown_args = parser.parse_known_args()
 
     configure_logger()
     if args.stage != TestStage.CLEAN.value:
         configure_logger(logs_path=pathlib.Path(args.root_dir) / 'logs' / f'{args.stage}.log')
     log = logging.getLogger('tests_runner.main')
 
+    custom_cli_args = {}
+    if unknown_args:
+        for arg in unknown_args:
+            try:
+                arg = arg.split('=')
+                custom_cli_args[arg[0]] = arg[1]
+            except Exception:
+                log.exception(f'Wrong argument layout: {arg}')
+                exit(ErrorCode.CRITICAL)
+
     try:
         tests_runner = TestRunner(artifacts=pathlib.Path(args.artifacts).absolute(),
                                   root_dir=pathlib.Path(args.root_dir).absolute(),
-                                  current_stage=args.stage)
+                                  current_stage=args.stage,
+                                  commit_time=args.commit_time,
+                                  custom_cli_args=custom_cli_args)
 
         if tests_runner.generate_config():
             no_errors = tests_runner.run_stage(args.stage)

common/extract_repo.py

@@ -29,7 +29,6 @@
 import shutil
 from distutils.dir_util import copy_tree
 from importlib import reload
-from datetime import datetime
 
 sys.path.append(str(pathlib.Path(__file__).resolve().parents[1]))
 from common import git_worker
@@ -74,12 +73,9 @@ def extract_repo(root_repo_dir, repo_name, branch, commit_id=None, commit_time=N
                         f'Release branch {branch} does not exist in the repo {repo.repo_name}')
 
                 # repo.branch = branch
-                repo.change_repo_state(branch_name=branch,
-                                       commit_time=datetime.strptime(commit_time,
-                                                                     '%Y-%m-%d %H:%M:%S').timestamp())
+                repo.change_repo_state(branch_name=branch, commit_time=commit_time)
             else:
-                repo.change_repo_state(
-                    commit_time=datetime.strptime(commit_time, '%Y-%m-%d %H:%M:%S').timestamp())
+                repo.change_repo_state(commit_time=commit_time)
         else:
             log.info('Commit id and timestamp not specified, clone HEAD of repository')
             repo = git_worker.GitRepo(root_repo_dir=root_repo_dir, repo_name=repo_name,
@@ -278,8 +274,8 @@ def main():
                         help='Branch name, by default "master" branch')
     parser.add_argument("--commit-id", metavar="String",
                         help='SHA of commit')
-    parser.add_argument("--commit-time", metavar="String",
-                        help='Will switch to the commit before specified time')
+    parser.add_argument("--commit-time", metavar="datetime",
+                        help='Switch to a commit before the time (e.g. 2017-11-02 07:36:40)')
     args = parser.parse_args()
 
     configure_logger()