Merge branch 'master' into snyk-fix-4cc2868dfbe153c384dc3e7fd0c06488

Author: simbuerg

benchbuild/environments/adapters/podman.py

@@ -157,6 +157,12 @@ def _create(
         interactive = bool(CFG['container']['interactive'])
 
         create_cmd = bb_podman('create', '--replace')
+
+        if seccomp_config := CFG['container']['seccomp_config']:
+            create_cmd = create_cmd['--security-opt',
+                                    f'seccomp={seccomp_config}', '--cap-add',
+                                    'PERFMON']
+
         if interactive:
             create_cmd = create_cmd['-it', '--entrypoint', '/bin/sh']
 

benchbuild/environments/domain/commands.py

@@ -75,6 +75,9 @@ class RunProjectContainer(model.Command):
     name: str = attr.ib(converter=oci_compliant_name)
 
     build_dir: str = attr.ib()
+    tmp_dir: str = attr.ib()
+    mount_build_dir: bool = attr.ib()
+    mount_tmp_dir: bool = attr.ib()
     args: tp.Sequence[str] = attr.ib(default=attr.Factory(list))
 
 

benchbuild/environments/entrypoints/cli.py

@@ -72,6 +72,16 @@ def set_group(self, groups: tp.List[str]) -> None:
                            requires=['experiment', 'debug'],
                            help='Run a container interactively.')
 
+    no_mount_build_dir = cli.Flag(
+        ['no-mount-build-dir'],
+        default=True,
+        help="Do not mount benchbuild's build directory."
+    )
+
+    mount_tmp_dir = cli.Flag(['mount-tmp-dir'],
+                             default=False,
+                             help="Mount benchbuild's tmp directory.")
+
     def main(self, *projects: str) -> int:
         plugins.discover()
 
@@ -116,7 +126,8 @@ def main(self, *projects: str) -> int:
                 ),
             "Run":
                 partial(
-                    run_experiment_images, wanted_experiments, wanted_projects
+                    run_experiment_images, wanted_experiments, wanted_projects,
+                    not self.no_mount_build_dir, self.mount_tmp_dir
                 )
         }
 
@@ -422,6 +433,7 @@ def create_project_images(
                   Tuples.
     """
     build_dir = local.path(BB_APP_ROOT) / 'results'
+    tmp_dir = local.path(BB_APP_ROOT) / 'tmp'
     publish = bootstrap.bus()
 
     for prj in enumerate_projects(experiments, projects):
@@ -432,9 +444,10 @@ def create_project_images(
         layers.context(partial(__pull_sources_in_context, prj))
         layers.add('.', BB_APP_ROOT)
         layers.run('mkdir', str(build_dir))
+        layers.run('mkdir', str(tmp_dir))
         layers.env(
             BB_BUILD_DIR=str(build_dir),
-            BB_TMP_DIR=BB_APP_ROOT,
+            BB_TMP_DIR=str(tmp_dir),
             BB_PLUGINS_PROJECTS=f'["{prj.__module__}"]'
         )
         layers.workingdir(BB_APP_ROOT)
@@ -491,7 +504,8 @@ def create_experiment_images(
 
 
 def run_experiment_images(
-    experiments: ExperimentIndex, projects: ProjectIndex
+    experiments: ExperimentIndex, projects: ProjectIndex, mount_build_dir: bool,
+    mount_tmp_dir: bool
 ) -> None:
     """
     Run experiments on given projects.
@@ -503,6 +517,7 @@ def run_experiment_images(
         projects: Index of projects to run.
     """
     build_dir = str(CFG['build_dir'])
+    tmp_dir = str(CFG['tmp_dir'])
     publish = bootstrap.bus()
 
     for exp in enumerate_experiments(experiments, projects):
@@ -516,7 +531,8 @@ def run_experiment_images(
 
             publish(
                 commands.RunProjectContainer(
-                    image_tag, container_name, build_dir
+                    image_tag, container_name, build_dir, tmp_dir,
+                    mount_build_dir, mount_tmp_dir
                 )
             )
 

benchbuild/environments/service_layer/handlers.py

@@ -70,14 +70,25 @@ def run_project_container(
     with uow:
         ensure.container_image_exists(cmd.image, uow)
 
-        build_dir = uow.registry.env(cmd.image, 'BB_BUILD_DIR')
-        if build_dir:
-            uow.registry.mount(cmd.image, cmd.build_dir, build_dir)
-        else:
-            LOG.warning(
-                'The image misses a configured "BB_BUILD_DIR" variable.'
-            )
-            LOG.warning('No result artifacts will be copied out.')
+        if cmd.mount_build_dir:
+            build_dir = uow.registry.env(cmd.image, 'BB_BUILD_DIR')
+            if build_dir:
+                uow.registry.mount(cmd.image, cmd.build_dir, build_dir)
+            else:
+                LOG.warning(
+                    'The image misses a configured "BB_BUILD_DIR" variable.'
+                )
+                LOG.warning('No result artifacts will be copied out.')
+
+        if cmd.mount_tmp_dir:
+            tmp_dir = uow.registry.env(cmd.image, 'BB_TMP_DIR')
+            if tmp_dir:
+                uow.registry.mount(cmd.image, cmd.tmp_dir, tmp_dir)
+            else:
+                LOG.warning(
+                    'The image misses a configured "BB_TMP_DIR" variable.'
+                )
+                LOG.warning('Temporary files will not be shared.')
 
         container = uow.create(cmd.image, cmd.name, cmd.args)
         uow.start(container)

benchbuild/project.py

@@ -476,6 +476,9 @@ def discovered() -> tp.Dict[str, ProjectT]:
 
 
 def __add_single_filter__(project: ProjectT, version: str) -> ProjectT:
+    if version == "*":
+        return project
+
     sources = project.SOURCE
     sources = [src for src in project.SOURCE if src.is_expandable]
 
@@ -493,6 +496,8 @@ def __add_indexed_filters__(
     sources = [src for src in project.SOURCE if src.is_expandable]
 
     for i in range(min(len(sources), len(versions))):
+        if versions[i] == "*":
+            continue
         sources[i] = source.SingleVersionFilter(sources[i], versions[i])
 
     project.SOURCE = sources
@@ -508,6 +513,9 @@ def __add_named_filters__(
         s.key: s for s in sources
     }
     for k, v in versions.items():
+        if v == "*":
+            continue
+
         if k in named_sources:
             victim: source.base.FetchableSource = named_sources[k]
             victim = source.SingleVersionFilter(victim, v)

benchbuild/settings.py

@@ -392,6 +392,10 @@
         "desc": "Storage options for containers."
                 "If 'null', ignore 'storage.conf'."
     },
+    "seccomp_config": {
+        "default": None,
+        "desc": "Path to a custom seccomp.json to use."
+    },
     "input": {
         "default": "container.tar.bz2",
         "desc": "Input container file/folder."

benchbuild/source/git.py

@@ -1,9 +1,9 @@
 """
 Declare a git source.
 """
+import logging
 import os
 import typing as tp
-import logging
 from pathlib import Path
 
 import plumbum as pb
@@ -18,6 +18,8 @@
 VarRemotes = tp.Union[str, tp.Dict[str, str]]
 Remotes = tp.Dict[str, str]
 
+_fetched_cache: tp.Set['Git'] = set()
+
 
 class Git(base.FetchableSource):
     """
@@ -32,6 +34,7 @@ def __init__(
         limit: tp.Optional[int] = 10,
         refspec: str = 'HEAD',
         shallow: bool = True,
+        submodule_set_urls: tp.Optional[tp.Dict[str, str]] = None,
         version_filter: tp.Callable[[str], bool] = lambda version: True
     ):
         super().__init__(local, remote)
@@ -40,6 +43,7 @@ def __init__(
         self.limit = limit
         self.refspec = refspec
         self.shallow = shallow
+        self.submodule_set_urls = submodule_set_urls
         self.version_filter = version_filter
 
     @property
@@ -62,18 +66,35 @@ def fetch(self) -> pb.LocalPath:
             str: [description]
         """
         prefix = base.target_prefix()
-        clone = maybe_shallow(
-            git['clone', '--recurse-submodules'], self.shallow
-        )
+        clone = maybe_shallow(git['clone'], self.shallow)
         fetch = git['fetch', '--update-shallow', '--all']
+        checkout = git['checkout', '-f', '--recurse-submodules']
+        set_url = git['submodule', 'set-url']
+        submodule_update = git['submodule', 'update', '--init', '--recursive']
+
         flat_local = self.local.replace(os.sep, '-')
         cache_path = pb.local.path(prefix) / flat_local
 
         if clone_needed(self.remote, cache_path):
             clone(self.remote, cache_path)
-        else:
+
             with pb.local.cwd(cache_path):
-                fetch()
+                if "HEAD" not in self.refspec:
+                    checkout(self.refspec.split('/')[-1])
+
+                if self.submodule_set_urls:
+                    for submodule, url in self.submodule_set_urls.items():
+                        LOG.debug('Setting url for submodule %s to %s.', submodule, url)
+                        set_url(submodule, url)
+                submodule_update()
+        else:
+            if self in _fetched_cache:
+                LOG.debug('Already fetched %s, skipping.', self.local)
+            else:
+                LOG.debug('Fetching %s.', self.local)
+                _fetched_cache.add(self)
+                with pb.local.cwd(cache_path):
+                    fetch()
 
         return cache_path
 
@@ -99,7 +120,9 @@ def version(self, target_dir: str, version: str = 'HEAD') -> pb.LocalPath:
         clone = git['clone']
         pull = git['pull']
         rev_parse = git['rev-parse']
-        checkout = git['checkout']
+        set_url = git['submodule', 'set-url']
+        submodule_update = git['submodule', 'update', '--init', '--recursive']
+        checkout = git['checkout', '-f']
 
         with pb.local.cwd(src_loc):
             is_shallow = rev_parse('--is-shallow-repository').strip()
@@ -114,12 +137,15 @@ def version(self, target_dir: str, version: str = 'HEAD') -> pb.LocalPath:
         else:
             mkdir('-p', tgt_loc)
             with pb.local.cwd(tgt_loc):
-                clone(
-                    '--dissociate', '--recurse-submodules', '--reference',
-                    src_loc, self.remote, '.'
-                )
+                clone('--dissociate', '--reference', src_loc, self.remote, '.')
                 checkout('--detach', version)
 
+                if self.submodule_set_urls:
+                    for submodule, url in self.submodule_set_urls.items():
+                        LOG.debug('Setting url for submodule %s to %s.', submodule, url)
+                        set_url(submodule, url)
+                submodule_update()
+
         ln('-nsf', tgt_subdir, active_loc)
         return tgt_loc
 

benchbuild/source/http.py

@@ -17,6 +17,15 @@ class HTTP(base.FetchableSource):
     Fetch the downloadable source via http.
     """
 
+    def __init__(
+        self,
+        local: str,
+        remote: tp.Union[str, tp.Dict[str, str]],
+        check_certificate: bool = True
+    ):
+        super().__init__(local, remote)
+        self._check_certificate = check_certificate
+
     @property
     def default(self) -> base.Variant:
         return self.versions()[0]
@@ -43,7 +52,7 @@ def fetch_version(self, version: str) -> pb.LocalPath:
         url = remotes[version]
         target_name = versioned_target_name(self.local, version)
         cache_path = pb.local.path(prefix) / target_name
-        download_single_version(url, cache_path)
+        download_single_version(url, cache_path, self._check_certificate)
 
         return cache_path
 
@@ -102,7 +111,7 @@ def version(self, target_dir: str, version: str) -> pb.LocalPath:
         active_loc = pb.local.path(target_dir) / self.local
 
         mkdir(target_path)
-        tar("-x", "-C", target_path, "-f", archive_path)
+        tar("-x", "--no-same-owner", "-C", target_path, "-f", archive_path)
 
         ln('-sf', target_path, active_loc)
 
@@ -118,9 +127,10 @@ def __init__(
         self,
         local: str,
         remote: tp.Union[str, tp.Dict[str, str]],
-        files: tp.List[str]
+        files: tp.List[str],
+        check_certificate: bool = True
     ):
-        super().__init__(local, remote)
+        super().__init__(local, remote, check_certificate)
         self._files = files
 
     def fetch_version(self, version: str) -> pb.LocalPath:
@@ -134,7 +144,9 @@ def fetch_version(self, version: str) -> pb.LocalPath:
         mkdir('-p', cache_path)
 
         for file in self._files:
-            download_single_version(f'{url}/{file}', cache_path / file)
+            download_single_version(
+                f'{url}/{file}', cache_path / file, self._check_certificate
+            )
 
         return cache_path
 
@@ -153,11 +165,17 @@ def versioned_target_name(target_name: str, version: str) -> str:
     return "{}-{}".format(version, target_name)
 
 
-def download_single_version(url: str, target_path: str) -> str:
+def download_single_version(
+    url: str, target_path: str, check_certificate: bool
+) -> str:
     if not download_required(target_path):
         return target_path
 
-    wget(url, '-O', target_path)
+    if check_certificate:
+        wget(url, '-O', target_path)
+    else:
+        wget(url, '--no-check-certificate', '-O', target_path)
+
     from benchbuild.utils.download import update_hash
     update_hash(target_path)
     return target_path