From bcc5685206aba0dd6073d38c4847f3878cc856ac Mon Sep 17 00:00:00 2001 From: Phillip Henderson Date: Wed, 7 Jul 2021 14:09:38 -0400 Subject: [PATCH 1/8] DAOS-623 build: Test build with new daos-devel RPM. Signed-off-by: Phillip Henderson --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index a3410ea..cece2f7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -39,5 +39,6 @@ // To use a test branch (i.e. PR) until it lands to master // I.e. for testing library changes //@Library(value="pipeline-lib@your_branch") _ +@Library(value="daos@6127") _ packageBuildingPipeline(['distros' : ['centos7', 'centos8', 'leap15']]) \ No newline at end of file From e3965a9eca5d83ebae1a14fee2c063cb2cfbbbe7 Mon Sep 17 00:00:00 2001 From: Phillip Henderson Date: Wed, 7 Jul 2021 14:18:28 -0400 Subject: [PATCH 2/8] Fixed it. Signed-off-by: Phillip Henderson --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index cece2f7..953f1da 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -39,6 +39,6 @@ // To use a test branch (i.e. PR) until it lands to master // I.e. for testing library changes //@Library(value="pipeline-lib@your_branch") _ -@Library(value="daos@6127") _ +@Library(value="daos@PR-6127") _ packageBuildingPipeline(['distros' : ['centos7', 'centos8', 'leap15']]) \ No newline at end of file From d3c707298d079a4c968d86164db842a4c477c2cc Mon Sep 17 00:00:00 2001 From: Phillip Henderson Date: Wed, 7 Jul 2021 14:26:58 -0400 Subject: [PATCH 3/8] Updated packaging. Signed-off-by: Phillip Henderson --- packaging/Dockerfile.mockbuild | 1 - packaging/Dockerfile.ubuntu.20.04 | 12 ++--- packaging/Makefile_distro_vars.mk | 16 ++----- packaging/Makefile_packaging.mk | 29 +++++++---- packaging/rpm_chrootbuild | 80 +++++++++++++++++-------------- 5 files changed, 74 insertions(+), 64 deletions(-) diff --git a/packaging/Dockerfile.mockbuild b/packaging/Dockerfile.mockbuild index a55b2be..d062286 100644 --- a/packaging/Dockerfile.mockbuild +++ b/packaging/Dockerfile.mockbuild @@ -28,4 +28,3 @@ RUN usermod -a -G mock $USER RUN grep use_nspawn /etc/mock/site-defaults.cfg || \ echo "config_opts['use_nspawn'] = False" >> /etc/mock/site-defaults.cfg -RUN chmod g+w /etc/mock/* diff --git a/packaging/Dockerfile.ubuntu.20.04 b/packaging/Dockerfile.ubuntu.20.04 index cf16bce..f956145 100644 --- a/packaging/Dockerfile.ubuntu.20.04 +++ b/packaging/Dockerfile.ubuntu.20.04 @@ -1,11 +1,11 @@ # -# Copyright 2019-2020, Intel Corporation +# Copyright 2019-2021, Intel Corporation # # 'recipe' for Docker to build an Debian package # # Pull base image FROM ubuntu:20.04 -MAINTAINER daos-stack +LABEL org.opencontainers.image.authors="daos@daos.groups.io" # use same UID as host and default value of 1000 if not specified ARG UID=1000 @@ -14,10 +14,10 @@ ARG REPO_UBUNTU_20_04="" # Install basic tools RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - autoconf bash curl debhelper dh-make dpkg-dev doxygen gcc \ - git git-buildpackage locales make patch pbuilder rpm wget \ - ca-certificates scons python3-distutils pkg-config \ - python3-dev python3-distro + autoconf bash ca-certificates curl debhelper dh-make \ + dpkg-dev dh-python doxygen gcc git git-buildpackage locales \ + make patch pbuilder pkg-config python3-dev python3-distro \ + python3-distutils rpm scons wget # rpmdevtools RUN echo "deb [trusted=yes] ${REPO_URL}${REPO_UBUNTU_20_04} focal main" > /etc/apt/sources.list.d/daos-stack-ubuntu-stable-local.list diff --git a/packaging/Makefile_distro_vars.mk b/packaging/Makefile_distro_vars.mk index c126c81..392a5d6 100644 --- a/packaging/Makefile_distro_vars.mk +++ b/packaging/Makefile_distro_vars.mk @@ -40,23 +40,17 @@ DISTRO_ID := el8 DISTRO_BASE := EL_8 SED_EXPR := 1s/$(DIST)//p endif -ifeq ($(CHROOT_NAME),opensuse-leap-15.1-x86_64) -VERSION_ID := 15.1 -DISTRO_ID := sl15.1 -DISTRO_BASE := LEAP_15 -SED_EXPR := 1p -endif ifeq ($(CHROOT_NAME),opensuse-leap-15.2-x86_64) VERSION_ID := 15.2 DISTRO_ID := sl15.2 DISTRO_BASE := LEAP_15 SED_EXPR := 1p endif -ifeq ($(CHROOT_NAME),leap-42.3-x86_64) -# TBD if support is ever resurrected -endif -ifeq ($(CHROOT_NAME),sles-12.3-x86_64) -# TBD if support is ever resurrected +ifeq ($(CHROOT_NAME),opensuse-leap-15.3-x86_64) +VERSION_ID := 15.3 +DISTRO_ID := sl15.3 +DISTRO_BASE := LEAP_15 +SED_EXPR := 1p endif endif ifeq ($(ID),centos) diff --git a/packaging/Makefile_packaging.mk b/packaging/Makefile_packaging.mk index f395d77..955a0d7 100644 --- a/packaging/Makefile_packaging.mk +++ b/packaging/Makefile_packaging.mk @@ -9,11 +9,8 @@ SHELL=/bin/bash # Put site overrides (i.e. REPOSITORY_URL, DAOS_STACK_*_LOCAL_REPO) in here -include Makefile.local -ifeq ($(MAKECMDGOALS),chrootbuild) # default to Leap 15 distro for chrootbuild CHROOT_NAME ?= opensuse-leap-15.2-x86_64 -endif - include packaging/Makefile_distro_vars.mk ifeq ($(DEB_NAME),) @@ -31,6 +28,11 @@ RPM_BUILD_OPTIONS += $(EXTERNAL_RPM_BUILD_OPTIONS) # some defaults the caller can override PACKAGING_CHECK_DIR ?= ../packaging LOCAL_REPOS ?= true +ifeq ($(ID_LIKE),debian) +DAOS_REPO_TYPE ?= LOCAL +else +DAOS_REPO_TYPE ?= STABLE +endif TEST_PACKAGES ?= ${NAME} # unfortunately we cannot always name the repo the same as the project @@ -294,17 +296,26 @@ ls: $(TARGETS) # *_GROUP_* repos may not supply a repomd.xml.key. ifeq ($(LOCAL_REPOS),true) ifneq ($(REPOSITORY_URL),) -ifneq ($(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO),) +ifneq ($(DAOS_STACK_$(DISTRO_BASE)_$(DAOS_REPO_TYPE)_REPO),) ifeq ($(ID_LIKE),debian) # $(DISTRO_BASE)_LOCAL_REPOS is a list separated by | because you cannot pass lists # of values with spaces as environment variables $(DISTRO_BASE)_LOCAL_REPOS := [trusted=yes] endif -$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS) $(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)/ +$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS) $(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_$(DAOS_REPO_TYPE)_REPO)/ endif $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)| -ifneq ($(DAOS_STACK_$(DISTRO_BASE)_GROUP_REPO),) -$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)$(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_GROUP_REPO)/| +ifneq ($(DAOS_STACK_$(DISTRO_BASE)_DOCKER_$(DAOS_REPO_TYPE)_REPO),) +DISTRO_REPOS = $(DAOS_STACK_$(DISTRO_BASE)_DOCKER_$(DAOS_REPO_TYPE)_REPO) +$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)$(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_DOCKER_$(DAOS_REPO_TYPE)_REPO)/| +endif +ifneq ($(DAOS_STACK_$(DISTRO_BASE)_APPSTREAM_REPO),) +$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)$(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_APPSTREAM_REPO)| +endif +ifneq ($(ID_LIKE),debian) +ifneq ($(DAOS_STACK_INTEL_ONEAPI_REPO),) +$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)$(REPOSITORY_URL)$(DAOS_STACK_INTEL_ONEAPI_REPO)| +endif endif endif endif @@ -334,6 +345,7 @@ chrootbuild: $(SRPM) $(CALLING_MAKEFILE) DISTRO_BASE_LOCAL_REPOS="$($(DISTRO_BASE)_LOCAL_REPOS)" \ MOCK_OPTIONS="$(MOCK_OPTIONS)" \ RPM_BUILD_OPTIONS='$(RPM_BUILD_OPTIONS)' \ + DISTRO_REPOS='$(DISTRO_REPOS)' \ TARGET="$<" \ packaging/rpm_chrootbuild endif @@ -383,9 +395,6 @@ test: $(call install_repos,$(REPO_NAME)@$(BRANCH_NAME):$(BUILD_NUMBER)) yum -y install $(TEST_PACKAGES) -show_VARIABLES: - $(foreach v, $(.VARIABLES), $(info $(v) = $($(v)))) - show_spec: @echo '$(SPEC)' diff --git a/packaging/rpm_chrootbuild b/packaging/rpm_chrootbuild index fd77e6f..8e8b5bc 100755 --- a/packaging/rpm_chrootbuild +++ b/packaging/rpm_chrootbuild @@ -4,46 +4,54 @@ set -uex # shellcheck disable=SC2153 IFS=\| read -r -a distro_base_local_repos <<< "$DISTRO_BASE_LOCAL_REPOS" +repo_adds=() +repo_dels=() -if [ -w /etc/mock/"$CHROOT_NAME".cfg ]; then - # Remove the distro repos - if [[ $CHROOT_NAME =~ opensuse-leap-* ]]; then - ed /etc/mock/"$CHROOT_NAME".cfg << EOF -/^# repos$/+2;/^"""$/-1d -wq -EOF - elif [[ $CHROOT_NAME =~ epel-8-* ]]; then - echo -e "config_opts['module_enable'] = ['javapackages-tools']\n" >> /etc/mock/"$CHROOT_NAME".cfg - fi - echo -e "config_opts['yum.conf'] += \"\"\"\n" >> /etc/mock/"$CHROOT_NAME".cfg - for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do - branch="master" - build_number="lastSuccessfulBuild" - if [[ $repo = *@* ]]; then - branch="${repo#*@}" - repo="${repo%@*}" - if [[ $branch = *:* ]]; then - build_number="${branch#*:}" - branch="${branch%:*}" - fi +if [ -n "$REPOSITORY_URL" ] && [ -n "$DISTRO_REPOS" ]; then + repo_dels+=("--disablerepo=\*") +fi + +: "${WORKSPACE:=$PWD}" +mock_config_dir="$WORKSPACE/mock" +original_cfg_file="/etc/mock/${CHROOT_NAME}.cfg" +cfg_file="$mock_config_dir/${CHROOT_NAME}_daos.cfg" +mkdir -p "$mock_config_dir" +ln -sf /etc/mock/templates "$mock_config_dir/" +ln -sf /etc/mock/logging.ini "$mock_config_dir/" + +cp "$original_cfg_file" "$cfg_file" + +echo -e "config_opts['yum.conf'] += \"\"\"\n" >> "$cfg_file" +for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do + branch="master" + build_number="lastSuccessfulBuild" + if [[ $repo = *@* ]]; then + branch="${repo#*@}" + repo="${repo%@*}" + if [[ $branch = *:* ]]; then + build_number="${branch#*:}" + branch="${branch%:*}" fi - echo -e "[$repo:$branch:$build_number]\n\ + fi + repo_adds+=("--enablerepo $repo:$branch:$build_number") + echo -e "[$repo:$branch:$build_number]\n\ name=$repo:$branch:$build_number\n\ baseurl=${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/$branch/$build_number/artifact/artifacts/$DISTRO/\n\ enabled=1\n\ -gpgcheck=False\n" >> /etc/mock/"$CHROOT_NAME".cfg - done - for repo in $JOB_REPOS "${distro_base_local_repos[@]}"; do - repo_name=${repo##*://} - repo_name=${repo_name//\//_} - echo -e "[${repo_name//@/_}]\n\ +gpgcheck=False\n" >> "$cfg_file" +done +for repo in $JOB_REPOS "${distro_base_local_repos[@]}"; do + repo_name=${repo##*://} + repo_name=${repo_name//\//_} + repo_adds+=("--enablerepo $repo_name") + echo -e "[${repo_name//@/_}]\n\ name=${repo_name}\n\ baseurl=${repo}\n\ -enabled=1\n" >> /etc/mock/"$CHROOT_NAME".cfg - done - echo "\"\"\"" >> /etc/mock/"$CHROOT_NAME".cfg -else - echo "Unable to update /etc/mock/$CHROOT_NAME.cfg." - echo "You need to make sure it has the needed repos in it yourself." -fi -eval mock -r "$CHROOT_NAME" $MOCK_OPTIONS $RPM_BUILD_OPTIONS "$TARGET" +enabled=1\n" >> "$cfg_file" +done +echo "\"\"\"" >> "$cfg_file" + +# shellcheck disable=SC2086 +eval mock --configdir "$mock_config_dir" -r "${CHROOT_NAME}_daos" \ + "${repo_dels[*]}" "${repo_adds[*]}" \ + $MOCK_OPTIONS $RPM_BUILD_OPTIONS "$TARGET" From 973e4974b1a311cbb5b4181d9ec46913f4ae577d Mon Sep 17 00:00:00 2001 From: Phillip Henderson Date: Wed, 7 Jul 2021 14:36:06 -0400 Subject: [PATCH 4/8] Trying this. Signed-off-by: Phillip Henderson --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 953f1da..5d69e48 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -39,6 +39,6 @@ // To use a test branch (i.e. PR) until it lands to master // I.e. for testing library changes //@Library(value="pipeline-lib@your_branch") _ -@Library(value="daos@PR-6127") _ +//@Library(value="daos@PR-6127") _ packageBuildingPipeline(['distros' : ['centos7', 'centos8', 'leap15']]) \ No newline at end of file From 8bdd770ea49af2948717a855a779ebfcf33fd1d1 Mon Sep 17 00:00:00 2001 From: Phillip Henderson Date: Wed, 7 Jul 2021 14:40:19 -0400 Subject: [PATCH 5/8] Trying with a pragma. PR-repos: daos@PR-6127 Signed-off-by: Phillip Henderson --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5d69e48..a3410ea 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -39,6 +39,5 @@ // To use a test branch (i.e. PR) until it lands to master // I.e. for testing library changes //@Library(value="pipeline-lib@your_branch") _ -//@Library(value="daos@PR-6127") _ packageBuildingPipeline(['distros' : ['centos7', 'centos8', 'leap15']]) \ No newline at end of file From 497b3497a35a9dd8cccfc48a341d6d85eb838802 Mon Sep 17 00:00:00 2001 From: Phillip Henderson Date: Wed, 7 Jul 2021 14:47:47 -0400 Subject: [PATCH 6/8] Fixing pragma. PR-repos: daos@PR-6127:1 Signed-off-by: Phillip Henderson --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index a3410ea..dc0f353 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,4 +40,5 @@ // I.e. for testing library changes //@Library(value="pipeline-lib@your_branch") _ + packageBuildingPipeline(['distros' : ['centos7', 'centos8', 'leap15']]) \ No newline at end of file From 04b7298a7871243fc60374e1691e840f44e1a667 Mon Sep 17 00:00:00 2001 From: Phillip Henderson Date: Wed, 7 Jul 2021 15:24:25 -0400 Subject: [PATCH 7/8] Attempting to fix Leap 15 build failure. PR-repos: daos@PR-6127:1 Signed-off-by: Phillip Henderson --- Jenkinsfile | 1 - hdf5-vol-daos.spec | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index dc0f353..a3410ea 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,5 +40,4 @@ // I.e. for testing library changes //@Library(value="pipeline-lib@your_branch") _ - packageBuildingPipeline(['distros' : ['centos7', 'centos8', 'leap15']]) \ No newline at end of file diff --git a/hdf5-vol-daos.spec b/hdf5-vol-daos.spec index ba40942..e5d8900 100644 --- a/hdf5-vol-daos.spec +++ b/hdf5-vol-daos.spec @@ -63,10 +63,10 @@ BuildRequires: libuuid-devel BuildRequires: boost-devel BuildRequires: gcc, gcc-c++ %if (0%{?suse_version} >= 1500) -BuildRequires: cmake >= 3.1 +BuildRequires: cmake >= 3.1, cmake < 3.15 BuildRequires: lua-lmod %else -BuildRequires: cmake3 >= 3.1 +BuildRequires: cmake3 >= 3.1, cmake3 < 3.15 BuildRequires: Lmod %endif BuildRequires: hdf5-devel%{?_isa} From d222030ae79fd715c14466626fc3e4a83200d157 Mon Sep 17 00:00:00 2001 From: Phillip Henderson Date: Wed, 7 Jul 2021 15:54:57 -0400 Subject: [PATCH 8/8] Leap 15 fix take 2. PR-repos: daos@PR-6127:1 Signed-off-by: Phillip Henderson --- hdf5-vol-daos.spec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hdf5-vol-daos.spec b/hdf5-vol-daos.spec index e5d8900..8b9bd02 100644 --- a/hdf5-vol-daos.spec +++ b/hdf5-vol-daos.spec @@ -63,10 +63,10 @@ BuildRequires: libuuid-devel BuildRequires: boost-devel BuildRequires: gcc, gcc-c++ %if (0%{?suse_version} >= 1500) -BuildRequires: cmake >= 3.1, cmake < 3.15 +BuildRequires: cmake >= 3.15.1 BuildRequires: lua-lmod %else -BuildRequires: cmake3 >= 3.1, cmake3 < 3.15 +BuildRequires: cmake3 >= 3.1 BuildRequires: Lmod %endif BuildRequires: hdf5-devel%{?_isa}