From b8047b8d2b01ac29d7e867be7d2c5b7fb9c4d37b Mon Sep 17 00:00:00 2001 From: "John E. Malmberg" Date: Wed, 26 Feb 2025 12:53:55 -0600 Subject: [PATCH 1/5] SRE-2832 Improve portability Update the build procedure to be more portable to other Jenkins environments. .dockerignore: New file Jenkinsfile: Minor fixes packaging.Dockerfile*: Support building behind a proxy. packaging.rpm_chrootbuild: Support building behind a proxy. packaging/scripts: Move code from Dockerfiles for easier maintenance. Signed-off-by: John E. Malmberg --- .dockerignore | 7 ++ Jenkinsfile | 6 +- packaging/Dockerfile.mockbuild | 23 +++--- packaging/Dockerfile.ubuntu | 43 +++++++++++ packaging/Dockerfile.ubuntu.20.04 | 43 ++++------- packaging/rpm_chrootbuild | 17 ++++- packaging/scripts/repo-helper-fedora.sh | 75 ++++++++++++++++++++ packaging/scripts/repo-helper-ubuntu.sh | 94 +++++++++++++++++++++++++ 8 files changed, 264 insertions(+), 44 deletions(-) create mode 100644 .dockerignore create mode 100644 packaging/Dockerfile.ubuntu create mode 100644 packaging/scripts/repo-helper-fedora.sh create mode 100644 packaging/scripts/repo-helper-ubuntu.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..0c205de6b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +# .dockerignore file for DAOS. + +# Firstly deny everything and then allow only directories and files that we're +# interested in. Other files will not be required for the build and they +# just generate noise and extra work for docker. +* +!packaging/scripts diff --git a/Jenkinsfile b/Jenkinsfile index 4d9c383c6..d0e0667e0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,6 @@ #!/usr/bin/groovy -/* Copyright (C) 2019 Intel Corporation +/* Copyright (C) 2019-2024 Intel Corporation + * Copyright (C) 2025 Hewlett Packard Enterprise Development LP * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -39,5 +40,6 @@ // To use a test branch (i.e. PR) until it lands to master // I.e. for testing library changes //@Library(value="pipeline-lib@your_branch") _ +@Library(value='pipeline-lib@malmberg/sre-2832') _ packageBuildingPipelineDAOSTest(['distros': ['el8', 'el9', 'leap15', 'ubuntu20.04'], - 'test-tag': 'dfuse']) + 'test-tag': 'dfuse']) diff --git a/packaging/Dockerfile.mockbuild b/packaging/Dockerfile.mockbuild index 6bad37d48..51cbaf7ab 100644 --- a/packaging/Dockerfile.mockbuild +++ b/packaging/Dockerfile.mockbuild @@ -1,5 +1,6 @@ # # Copyright 2018-2024 Intel Corporation +# Copyright 2025 Hewlett Packard Enterprise Development LP # # 'recipe' for Docker to build an RPM # @@ -13,15 +14,15 @@ LABEL maintainer="daos@daos.groups.io" # Use local repo server if present ARG REPO_FILE_URL -RUN if [ -n "$REPO_FILE_URL" ]; then \ - cd /etc/yum.repos.d/ && \ - curl -k -f -o daos_ci-fedora-artifactory.repo.tmp \ - "$REPO_FILE_URL"daos_ci-fedora-artifactory.repo && \ - for file in *.repo; do \ - true > $file; \ - done; \ - mv daos_ci-fedora-artifactory.repo{.tmp,}; \ - fi +ARG DAOS_LAB_CA_FILE_URL +# script to install OS updates basic tools and daos dependencies +# COPY ./utils/scripts/install-fedora.sh /tmp/install.sh +# script to setup local repo if available +COPY ./packaging/scripts/repo-helper-fedora.sh /tmp/repo-helper.sh + +RUN chmod +x /tmp/repo-helper.sh && \ + /tmp/repo-helper.sh && \ + rm -f /tmp/repo-helper.sh # Install basic tools RUN dnf -y install mock make \ @@ -33,8 +34,8 @@ RUN dnf -y install mock make \ ARG UID=1000 # Add build user (to keep rpmbuild happy) -ENV USER build -ENV PASSWD build +ENV USER=build +ENV PASSWD=build # add the user to the mock group so it can run mock RUN if [ $UID != 0 ]; then \ useradd -u $UID -ms /bin/bash $USER; \ diff --git a/packaging/Dockerfile.ubuntu b/packaging/Dockerfile.ubuntu new file mode 100644 index 000000000..bd42da40c --- /dev/null +++ b/packaging/Dockerfile.ubuntu @@ -0,0 +1,43 @@ +# +# Copyright 2019-2021, Intel Corporation +# Copyright 2025 Hewlett Packard Enterprise Development LP +# +# 'recipe' for Docker to build an Debian package +# +# Pull base image +ARG BASE_DISTRO=ubuntu:20.04 +FROM $BASE_DISTRO +LABEL org.opencontainers.image.authors="daos@daos.groups.io" +# Needed for later use of BASE_DISTRO +ARG BASE_DISTRO + +ARG REPO_FILE_URL +ARG DAOS_LAB_CA_FILE_URL +# script to setup local repo if available +COPY ./scripts/repo-helper-ubuntu.sh /tmp/repo-helper.sh + +# Install basic tools +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + autoconf bash ca-certificates curl debhelper dh-make \ + dpkg-dev dh-python doxygen gcc git git-buildpackage \ + javahelper locales make patch pbuilder pkg-config \ + python3-dev python3-distro python3-distutils rpm scons wget \ + cmake valgrind rpmdevtools + +# use same UID as host and default value of 1000 if not specified +ARG UID=1000 + +# Add build user (to keep chrootbuild happy) +ENV USER=build +RUN useradd -u $UID -ms /bin/bash $USER + +# need to run the build command as root, as it needs to chroot +RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then \ + echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \ + fi; \ + echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build; \ + chmod 0440 /etc/sudoers.d/build; \ + visudo -c; \ + sudo -l -U build diff --git a/packaging/Dockerfile.ubuntu.20.04 b/packaging/Dockerfile.ubuntu.20.04 index 4d0546448..2838cb133 100644 --- a/packaging/Dockerfile.ubuntu.20.04 +++ b/packaging/Dockerfile.ubuntu.20.04 @@ -1,40 +1,23 @@ +# Keep Dockerfile.ubuntu the same as this file until all packaging +# jobs are fixed to have a Dockerfile.ubuntu, and then the common +# Jenkinsfile will be changed to use Dockerfile.ubuntu. # # Copyright 2019-2021, Intel Corporation +# Copyright 2025 Hewlett Packard Enterprise Development LP # # 'recipe' for Docker to build an Debian package # # Pull base image -FROM ubuntu:20.04 +ARG BASE_DISTRO=ubuntu:20.04 +FROM $BASE_DISTRO LABEL org.opencontainers.image.authors="daos@daos.groups.io" - -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - curl gpg +# Needed for later use of BASE_DISTRO +ARG BASE_DISTRO ARG REPO_FILE_URL -RUN if [ -n "$REPO_FILE_URL" ]; then \ - cd /etc/apt/sources.list.d && \ - curl -f -o daos_ci-ubuntu20.04-artifactory.list.tmp \ - "$REPO_FILE_URL"daos_ci-ubuntu20.04-artifactory.list && \ - true > ../sources.list && \ - mv daos_ci-ubuntu20.04-artifactory.list.tmp \ - daos_ci-ubuntu20.04-artifactory.list; \ - url="${REPO_FILE_URL%/*/}/hpe-ilorest-ubuntu-bionic-proxy/"; \ - else \ - url="https://downloads.linux.hpe.com/SDR/repo/ilorest/"; \ - fi; \ - cd -; \ - mkdir -p /usr/local/share/keyrings/; \ - curl -f -O "$url"GPG-KEY-hprest; \ - gpg --no-default-keyring --keyring ./temp-keyring.gpg \ - --import GPG-KEY-hprest; \ - gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \ - --output /usr/local/share/keyrings/hpe-sdr-public.gpg; \ - rm ./temp-keyring.gpg; \ - curl -f -O "$REPO_FILE_URL"esad_repo.key; \ - gpg --no-default-keyring --keyring ./temp-keyring.gpg \ - --import esad_repo.key; \ - gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \ - --output /usr/local/share/keyrings/daos-stack-public.gpg +ARG DAOS_LAB_CA_FILE_URL +# script to setup local repo if available +COPY ./scripts/repo-helper-ubuntu.sh /tmp/repo-helper.sh # Install basic tools RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ @@ -48,7 +31,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ ARG UID=1000 # Add build user (to keep chrootbuild happy) -ENV USER build +ENV USER=build RUN useradd -u $UID -ms /bin/bash $USER # need to run the build command as root, as it needs to chroot @@ -56,7 +39,7 @@ RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \ fi; \ echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build; \ - echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \ echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build; \ chmod 0440 /etc/sudoers.d/build; \ visudo -c; \ diff --git a/packaging/rpm_chrootbuild b/packaging/rpm_chrootbuild index d6443b735..5174c0ee4 100755 --- a/packaging/rpm_chrootbuild +++ b/packaging/rpm_chrootbuild @@ -2,6 +2,9 @@ set -uex +: "${HTTPS_PROXY:=}" +: "${REPO_FILE_URL:=}" + cp /etc/mock/"$CHROOT_NAME".cfg mock.cfg # Enable mock ccache plugin @@ -10,6 +13,18 @@ config_opts['plugin_conf']['ccache_enable'] = True config_opts['plugin_conf']['ccache_opts']['dir'] = "%(cache_topdir)s/%(root)s/ccache/" EOF +# Optionally add a proxy to mock +if [ -n "$HTTPS_PROXY" ];then + yum_proxy="http://${HTTPS_PROXY##*//}" + echo "config_opts['https_proxy'] = '$yum_proxy'" >> mock.cfg +fi + +# No proxy for local mirrors +if [ -n "$REPO_FILE_URL" ]; then + direct="${REPO_FILE_URL##*//}" + direct="${direct%%/*}" + echo "config_opts['no_proxy'] = '${direct}'" >> mock.cfg +fi if [[ $CHROOT_NAME == *epel-8-x86_64 ]]; then cat <> mock.cfg @@ -127,7 +142,7 @@ if ! eval time mock -r mock.cfg ${repo_dels[*]} ${repo_adds[*]} --no-clean \ fi # Save the ccache -if [ -d /scratch/ ]; then +if [ -d /scratch/mock ]; then mkdir -p "$bs_dir"/ if ! flock "$bs_dir" -c "tar -czf $bs_dir/ccache-$CHROOT_NAME-$PACKAGE.tar.gz /var/cache/mock/${CHROOT_NAME}/ccache"; then echo "Failed to save ccache. Plowing onward." diff --git a/packaging/scripts/repo-helper-fedora.sh b/packaging/scripts/repo-helper-fedora.sh new file mode 100644 index 000000000..baa702664 --- /dev/null +++ b/packaging/scripts/repo-helper-fedora.sh @@ -0,0 +1,75 @@ +#!/bin/bash +set -uex + +# This script is used by Dockerfiles to optionally use +# a local repository instead of a distro provided repository. + +: "${REPO_FILE_URL:=}" +: "${DAOS_LAB_CA_FILE_URL:=}" +: "${FVERSION:=latest}" +: "${REPOSITORY_NAME:=artifactory}" +: "${archive:=}" +if [ "$FVERSION" != "latest" ]; then + archive="-archive" +fi + +# shellcheck disable=SC2120 +disable_repos () { + local repos_dir="$1" + shift + local save_repos + IFS=" " read -r -a save_repos <<< "${*:-} daos_ci-fedora${archive}-${REPOSITORY_NAME}" + if [ -n "$REPO_FILE_URL" ]; then + pushd "$repos_dir" + local repo + for repo in "${save_repos[@]}"; do + mv "$repo".repo{,.tmp} + done + for file in *.repo; do + true > "$file" + done + for repo in "${save_repos[@]}"; do + mv "$repo".repo{.tmp,} + done + popd + fi +} + +# Use local repo server if present +install_curl() { + : +} + +# Use local repo server if present +install_optional_ca() { + ca_storage="/etc/pki/ca-trust/source/anchors/" + if [ -n "$DAOS_LAB_CA_FILE_URL" ]; then + curl -k --noproxy '*' -sSf -o "${ca_storage}lab_ca_file.crt" \ + "$DAOS_LAB_CA_FILE_URL" + update-ca-trust + fi +} + +# Use local repo server if present +# if a local repo server is present and the distro repo server can not +# be reached, have to bootstrap in an environment to get curl installed +# to then install the pre-built repo file. + +if [ -n "$REPO_FILE_URL" ]; then + install_curl + install_optional_ca + mkdir -p /etc/yum.repos.d + pushd /etc/yum.repos.d/ + curl -k --noproxy '*' -sSf \ + -o "daos_ci-fedora${archive}-${REPOSITORY_NAME}.repo" \ + "{$REPO_FILE_URL}daos_ci-fedora${archive}-${REPOSITORY_NAME}.repo" + disable_repos /etc/yum.repos.d/ + popd +fi +dnf -y install dnf-plugins-core +# This does not work in fedora/41 anymore -- needs investigation +# dnf -y config-manager --save --setopt=assumeyes=True +# dnf config-manager --save --setopt=install_weak_deps=False +dnf clean all + +disable_repos /etc/yum.repos.d/ "${save_repos[@]}" diff --git a/packaging/scripts/repo-helper-ubuntu.sh b/packaging/scripts/repo-helper-ubuntu.sh new file mode 100644 index 000000000..04c8faeda --- /dev/null +++ b/packaging/scripts/repo-helper-ubuntu.sh @@ -0,0 +1,94 @@ +#!/bin/bash +set -uex + +# This script is used by dockerfiles to optionally use +# a local repository instead of a distro provided repository. +# It will also optionally allow running a /tmp/install script +# for custom packages if present. + +: "${REPO_FILE_URL:=}" +: "${HTTPS_PROXY:=}" +: "${DAOS_LAB_CA_FILE_UR:=}" +: "${REPOSITORY_NAME:=artifactory}" + +disable_repos () { + if [ -e /etc/apt/sources.list.d/ubuntu.sources ];then + mv /etc/apt/sources.list.d/ubuntu.sources \ + etc/apt/sources.list.d/ubuntu.sources.disabled + elif [ -e /etc/apt/sources.list ];then + mv /etc/apt/sources.list \ + etc/apt/sources.list.disabled + fi +} + +# Use local repo server if present +install_curl() { + + if command -v curl; then + echo "found curl!" + return + else + apt-get update + apt-get install curl ca-certificates gpg gpg-agent + fi + + if command -v wget; then + echo "found wget!" + return + fi + # If we don't find one of these, we are basically sunk for using + # a local repository mirror. +} + +# Use local repo server if present +install_optional_ca() { + ca_storage="/usr/local/share/ca-certificates/" + if [ -n "$DAOS_LAB_CA_FILE_URL" ]; then + curl -k --noproxy '*' -sSf -o "${ca_storage}lab_ca_file.crt" \ + "$DAOS_LAB_CA_FILE_URL" + update-ca-certificates + fi +} + +echo "APT::Get::Assume-Yes \"true\";" > /etc/apt/apt.conf.d/no-prompt +echo "APT::Install-Recommends \"false\";" > /etc/apt/apt.conf.d/no-recommends +if [ -n "$HTTPS_PROXY" ];then + apt_proxy="http://${HTTPS_PROXY##*//}" + echo "Acquire::http::Proxy \"$apt_proxy\";" > \ + /etc/apt/apt.conf.d/local_proxy + if [ -n "$REPO_FILE_URL" ]; then + direct="${REPO_FILE_URL##*//}" + direct="${direct%%/*}" + echo "Acquire::http::Proxy { $direct DIRECT; };" >> \ + /etc/apt/apt.conf.d/local_proxy + fi +fi + +# Use local repo server if present +# if a local repo server is present and the distro repo server can not +# be reached, have to bootstrap in an environment to get curl installed +# to then install the pre-built repo file. +DISTRO_VERSION="${BASE_DISTRO##*:}" +if [ -n "$REPO_FILE_URL" ]; then + install_curl + install_optional_ca + curl -k --noproxy '*' -sSf \ + -o "daos_ci-ubuntu${DISTRO_VERSION}-${REPOSITORY_NAME}.list" \ + "${REPO_FILE_URL}daos_ci-ubuntu${DISTRO_VERSION}-${REPOSITORY_NAME}.list" + disable_repos + mkdir -p /usr/local/share/keyrings/ + curl --noproxy '*' -sSf -O "${REPO_FILE_URL}esad_repo.key" + gpg --no-default-keyring --keyring ./temp-keyring.gpg \ + --import esad_repo.key + gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \ + --output /usr/local/share/keyrings/daos-stack-public.gpg +fi + +apt-get update +apt-get upgrade +apt-get install gpg-agent software-properties-common +add-apt-repository ppa:longsleep/golang-backports +apt-get update +chmod +x /tmp/install.sh +/tmp/install.sh +apt-get clean all From f28048b77f120f1dc22b859db9a16098627c0360 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 4 Jul 2025 12:58:26 +0200 Subject: [PATCH 2/5] The library changes have already landed to master Signed-off-by: Tomasz Gromadzki --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d0e0667e0..b6c89ffca 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,6 +40,5 @@ // To use a test branch (i.e. PR) until it lands to master // I.e. for testing library changes //@Library(value="pipeline-lib@your_branch") _ -@Library(value='pipeline-lib@malmberg/sre-2832') _ packageBuildingPipelineDAOSTest(['distros': ['el8', 'el9', 'leap15', 'ubuntu20.04'], 'test-tag': 'dfuse']) From 8532d2fb5a6c77bdece66a01692abf5bd4cbddf3 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 4 Jul 2025 13:13:48 +0200 Subject: [PATCH 3/5] Disable Ubuntu Signed-off-by: Tomasz Gromadzki --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index b6c89ffca..06f572fbf 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,5 +40,5 @@ // To use a test branch (i.e. PR) until it lands to master // I.e. for testing library changes //@Library(value="pipeline-lib@your_branch") _ -packageBuildingPipelineDAOSTest(['distros': ['el8', 'el9', 'leap15', 'ubuntu20.04'], +packageBuildingPipelineDAOSTest(['distros': ['el8', 'el9', 'leap15'], 'test-tag': 'dfuse']) From ae6d6dd74bf94c248927f3db4c2f6f155354d459 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 4 Jul 2025 14:04:19 +0200 Subject: [PATCH 4/5] Update packaging Signed-off-by: Tomasz Gromadzki --- packaging/Dockerfile.coverity | 3 +- packaging/Dockerfile.mockbuild | 3 + packaging/Dockerfile.ubuntu | 10 +++- packaging/Dockerfile.ubuntu.20.04 | 13 +++-- packaging/Dockerfile.ubuntu.rolling | 3 +- packaging/debian_chrootbuild | 75 ++++++++++++++++--------- packaging/rpm_chrootbuild | 9 ++- packaging/scripts/repo-helper-ubuntu.sh | 31 ++++++---- 8 files changed, 99 insertions(+), 48 deletions(-) diff --git a/packaging/Dockerfile.coverity b/packaging/Dockerfile.coverity index 7eed2c35d..f8171c92a 100755 --- a/packaging/Dockerfile.coverity +++ b/packaging/Dockerfile.coverity @@ -1,12 +1,13 @@ # # Copyright 2018-2020, Intel Corporation +# Copyright 2025 Hewlett Packard Enterprise Development LP # # 'recipe' for Docker to build for a Coverity scan. # # Pull base image FROM fedora:latest -MAINTAINER daos-stack +LABEL maintainer="daos-stack "" # use same UID as host and default value of 1000 if not specified ARG UID=1000 diff --git a/packaging/Dockerfile.mockbuild b/packaging/Dockerfile.mockbuild index 51cbaf7ab..d8d86ac98 100644 --- a/packaging/Dockerfile.mockbuild +++ b/packaging/Dockerfile.mockbuild @@ -15,6 +15,7 @@ LABEL maintainer="daos@daos.groups.io" # Use local repo server if present ARG REPO_FILE_URL ARG DAOS_LAB_CA_FILE_URL +ARG REPOSITORY_NAME # script to install OS updates basic tools and daos dependencies # COPY ./utils/scripts/install-fedora.sh /tmp/install.sh # script to setup local repo if available @@ -41,6 +42,8 @@ RUN if [ $UID != 0 ]; then \ useradd -u $UID -ms /bin/bash $USER; \ echo "$USER:$PASSWD" | chpasswd; \ usermod -a -G mock $USER; \ + mkdir -p /var/cache/mock; \ + chown $USER:root /var/cache/mock; \ fi ARG CB0 diff --git a/packaging/Dockerfile.ubuntu b/packaging/Dockerfile.ubuntu index bd42da40c..1e3813d58 100644 --- a/packaging/Dockerfile.ubuntu +++ b/packaging/Dockerfile.ubuntu @@ -1,3 +1,6 @@ +# Keep Dockerfile.ubuntu the same as this file until all packaging +# jobs are fixed to have a Dockerfile.ubuntu, and then the common +# Jenkinsfile will be changed to use Dockerfile.ubuntu. # # Copyright 2019-2021, Intel Corporation # Copyright 2025 Hewlett Packard Enterprise Development LP @@ -13,9 +16,14 @@ ARG BASE_DISTRO ARG REPO_FILE_URL ARG DAOS_LAB_CA_FILE_URL +ARG REPOSITORY_NAME # script to setup local repo if available COPY ./scripts/repo-helper-ubuntu.sh /tmp/repo-helper.sh +RUN chmod +x /tmp/repo-helper.sh && \ + /tmp/repo-helper.sh && \ + rm -f /tmp/repo-helper.sh + # Install basic tools RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ autoconf bash ca-certificates curl debhelper dh-make \ @@ -36,7 +44,7 @@ RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \ fi; \ echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build; \ - echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \ echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build; \ chmod 0440 /etc/sudoers.d/build; \ visudo -c; \ diff --git a/packaging/Dockerfile.ubuntu.20.04 b/packaging/Dockerfile.ubuntu.20.04 index 2838cb133..bd5363b59 100644 --- a/packaging/Dockerfile.ubuntu.20.04 +++ b/packaging/Dockerfile.ubuntu.20.04 @@ -16,16 +16,21 @@ ARG BASE_DISTRO ARG REPO_FILE_URL ARG DAOS_LAB_CA_FILE_URL +ARG REPOSITORY_NAME # script to setup local repo if available -COPY ./scripts/repo-helper-ubuntu.sh /tmp/repo-helper.sh +COPY ./packaging/scripts/repo-helper-ubuntu.sh /tmp/repo-helper.sh -# Install basic tools +RUN chmod +x /tmp/repo-helper.sh && \ + /tmp/repo-helper.sh && \ + rm -f /tmp/repo-helper.sh + +# Install basic tools - rpmdevtools temporary commented out. RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ autoconf bash ca-certificates curl debhelper dh-make \ dpkg-dev dh-python doxygen gcc git git-buildpackage \ javahelper locales make patch pbuilder pkg-config \ - python3-dev python3-distro python3-distutils rpm scons wget \ - cmake valgrind rpmdevtools + python3-dev python3-distro python3-distutils rpm scons sudo \ + wget cmake valgrind # rpmdevtools # use same UID as host and default value of 1000 if not specified ARG UID=1000 diff --git a/packaging/Dockerfile.ubuntu.rolling b/packaging/Dockerfile.ubuntu.rolling index 02aca4584..ed5d3633c 100644 --- a/packaging/Dockerfile.ubuntu.rolling +++ b/packaging/Dockerfile.ubuntu.rolling @@ -1,11 +1,12 @@ # # Copyright 2019, Intel Corporation +# Copyright 2025 Hewlett Packard Enterprise Development LP # # 'recipe' for Docker to build an Debian package # # Pull base image FROM ubuntu:rolling -Maintainer daos-stack +LABEL org.opencontainers.image.authors="daos@daos.groups.io" # use same UID as host and default value of 1000 if not specified ARG UID=1000 diff --git a/packaging/debian_chrootbuild b/packaging/debian_chrootbuild index cc2cc96d8..5cb2943f2 100755 --- a/packaging/debian_chrootbuild +++ b/packaging/debian_chrootbuild @@ -2,17 +2,35 @@ set -uex +: "${REPO_FILE_URL:=}" +: "${HTTPS_PROXY:=}" + +# Currently not fully working behind a proxy if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then - echo "MIRRORSITE=${ARTIFACTORY_URL}artifactory/ubuntu-proxy" | sudo tee /root/.pbuilderrc + pbuilderrc="./pbuilder_rc.txt" + rm -f "$pbuilderrc" + if [ -n "${HTTPS_PROXY}" ]; then + echo "export http_proxy=\"${HTTPS_PROXY}\"" >> "$pbuilderrc" + else + echo "MIRRORSITE=${ARTIFACTORY_URL}/ubuntu-proxy/ubuntu" > "$pbuilderrc" + fi + #if [ -n "$REPO_FILE_URL" ]; then + # direct="${REPO_FILE_URL##*//}" + # direct="${direct%%/*}" + # echo "no_proxy=\"${direct}\"" >> "$pbuilderrc" + #fi + # shellcheck disable=SC2002 + cat "$pbuilderrc" | sudo tee /root/.pbuilderrc fi # shellcheck disable=SC2086 sudo pbuilder create \ --extrapackages "gnupg ca-certificates" \ - $DISTRO_ID_OPT + $DISTRO_ID_OPT || true # Ignore error status for now. repo_args="" repos_added=() +# currently a bit broken, pbuilder will not accept user provided CAs. for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do branch="master" build_number="lastSuccessfulBuild" @@ -32,31 +50,34 @@ for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do repo_args="$repo_args|deb [trusted=yes] ${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/$branch/$build_number/artifact/artifacts/$DISTRO/ ./" done -repo_args+="|$(curl -sSf "$REPO_FILE_URL"daos_ci-"$DISTRO"-artifactory.list | - sed -e 's/#.*//' -e '/ubuntu-proxy/d' -e '/^$/d' -e '/^$/d' \ - -e 's/signed-by=.*\.gpg/trusted=yes/' | - sed -e ':a; N; $!ba; s/\n/|/g')" -for repo in $JOB_REPOS; do - repo_name=${repo##*://} - repo_name=${repo_name//\//_} - if [[ " ${repos_added[*]} " = *\ ${repo_name}\ * ]]; then - # don't add duplicates, first found wins - continue - fi - repos_added+=("$repo_name") - repo_args+="|deb ${repo} $VERSION_CODENAME main" -done -# NB: This PPA is needed to support modern go toolchains on ubuntu 20.04. -# After the build is updated to use 22.04, which supports go >= 1.18, it -# should no longer be needed. -repo_args="$repo_args|deb [trusted=yes] https://ppa.launchpadcontent.net/longsleep/golang-backports/ubuntu $VERSION_CODENAME main" -echo "$repo_args" -if [ "$repo_args" = "|" ]; then - repo_args="" -else - #repo_args="--othermirror"${repo_args#|}\"" - repo_args="${repo_args#|}" -fi +# currently broken, builder will not accept internal certs. +# repo_args+="|$(curl -sSf "$REPO_FILE_URL"daos_ci-"$DISTRO"-artifactory.list | +# sed -e 's/#.*//' -e '/ubuntu-proxy/d' -e '/^$/d' -e '/^$/d' \ +# -e 's/signed-by=.*\.gpg/trusted=yes/' | +# sed -e ':a; N; $!ba; s/\n/|/g')" +#for repo in $JOB_REPOS; do +# repo_name=${repo##*://} +# repo_name=${repo_name//\//_} +# if [[ " ${repos_added[*]} " = *\ ${repo_name}\ * ]]; then +# # don't add duplicates, first found wins +# continue +# fi +# repos_added+=("$repo_name") +# repo_args+="|deb ${repo} $VERSION_CODENAME main" +#done + +## NB: This PPA is needed to support modern go toolchains on ubuntu 20.04. +## After the build is updated to use 22.04, which supports go >= 1.18, it +## should no longer be needed. +# currently broken - claim is public key not available. +#repo_args="$repo_args|deb [trusted=yes] https://ppa.launchpadcontent.net/longsleep/golang-backports/ubuntu $VERSION_CODENAME main" +#echo "$repo_args" +#if [ "$repo_args" = "|" ]; then +# repo_args="" +#else +# #repo_args="--othermirror"${repo_args#|}\"" +# repo_args="${repo_args#|}" +#fi cd "$DEB_TOP" # shellcheck disable=SC2086 sudo pbuilder update --override-config $DISTRO_ID_OPT ${repo_args:+--othermirror "$repo_args"} diff --git a/packaging/rpm_chrootbuild b/packaging/rpm_chrootbuild index 5174c0ee4..8d37abc7d 100755 --- a/packaging/rpm_chrootbuild +++ b/packaging/rpm_chrootbuild @@ -4,6 +4,8 @@ set -uex : "${HTTPS_PROXY:=}" : "${REPO_FILE_URL:=}" +: "${ARCH:=$(arch)}" +: "${REPOSITORY_NAME:=artifactory}" cp /etc/mock/"$CHROOT_NAME".cfg mock.cfg @@ -13,6 +15,7 @@ config_opts['plugin_conf']['ccache_enable'] = True config_opts['plugin_conf']['ccache_opts']['dir'] = "%(cache_topdir)s/%(root)s/ccache/" EOF + # Optionally add a proxy to mock if [ -n "$HTTPS_PROXY" ];then yum_proxy="http://${HTTPS_PROXY##*//}" @@ -26,7 +29,7 @@ if [ -n "$REPO_FILE_URL" ]; then echo "config_opts['no_proxy'] = '${direct}'" >> mock.cfg fi -if [[ $CHROOT_NAME == *epel-8-x86_64 ]]; then +if [[ $CHROOT_NAME == *"epel-8-${ARCH}" ]]; then cat <> mock.cfg config_opts['module_setup_commands'] = [ ('enable', 'javapackages-tools:201801'), @@ -36,7 +39,7 @@ EOF fi # Use dnf on CentOS 7 -if [[ $CHROOT_NAME == *epel-7-x86_64 ]]; then +if [[ $CHROOT_NAME == *"epel-7-$ARCH" ]]; then MOCK_OPTIONS="--dnf --no-bootstrap-chroot${MOCK_OPTIONS:+ }$MOCK_OPTIONS" fi @@ -76,7 +79,7 @@ if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then fi fi curl -sSf "$REPO_FILE_URL"daos_ci-"${CHROOT_NAME%-*}".repo >> mock.cfg - repo_adds+=("--enablerepo *-artifactory") + repo_adds+=("--enablerepo *-${REPOSITORY_NAME}") fi fi diff --git a/packaging/scripts/repo-helper-ubuntu.sh b/packaging/scripts/repo-helper-ubuntu.sh index 04c8faeda..20220e982 100644 --- a/packaging/scripts/repo-helper-ubuntu.sh +++ b/packaging/scripts/repo-helper-ubuntu.sh @@ -1,14 +1,14 @@ #!/bin/bash set -uex -# This script is used by dockerfiles to optionally use +# This script is used by Dockerfiles to optionally use # a local repository instead of a distro provided repository. # It will also optionally allow running a /tmp/install script # for custom packages if present. : "${REPO_FILE_URL:=}" : "${HTTPS_PROXY:=}" -: "${DAOS_LAB_CA_FILE_UR:=}" +: "${DAOS_LAB_CA_FILE_URL:=}" : "${REPOSITORY_NAME:=artifactory}" disable_repos () { @@ -29,7 +29,8 @@ install_curl() { return else apt-get update - apt-get install curl ca-certificates gpg gpg-agent + apt-get install curl ca-certificates gpg gpg-agent \ + software-properties-common fi if command -v wget; then @@ -72,10 +73,17 @@ DISTRO_VERSION="${BASE_DISTRO##*:}" if [ -n "$REPO_FILE_URL" ]; then install_curl install_optional_ca - curl -k --noproxy '*' -sSf \ - -o "daos_ci-ubuntu${DISTRO_VERSION}-${REPOSITORY_NAME}.list" \ - "${REPO_FILE_URL}daos_ci-ubuntu${DISTRO_VERSION}-${REPOSITORY_NAME}.list" - disable_repos + # Ubuntu local repo mirror is not working + # curl -k --noproxy '*' -sSf \ + # -o "daos_ci-ubuntu${DISTRO_VERSION}-${REPOSITORY_NAME}.list" \ + # "${REPO_FILE_URL}daos_ci-ubuntu${DISTRO_VERSION}-${REPOSITORY_NAME}.list" + # disable_repos + # temp hack until we can debug the Ubuntu repos. + REPO_BASE="${REPO_FILE_URL%repo-files/}" + curl -k --noproxy '*' -sSf \ + -o "rpmdevtools_8.10-10_amd64.deb" \ + "${REPO_BASE}daos-stack-deps-ubuntu-${DISTRO_VERSION}-x86_64-stable-local/pool/rpmdevtools_8.10-10_amd64.deb" + apt-get install "./rpmdevtools_8.10-10_amd64.deb" mkdir -p /usr/local/share/keyrings/ curl --noproxy '*' -sSf -O "${REPO_FILE_URL}esad_repo.key" gpg --no-default-keyring --keyring ./temp-keyring.gpg \ @@ -86,9 +94,10 @@ fi apt-get update apt-get upgrade -apt-get install gpg-agent software-properties-common -add-apt-repository ppa:longsleep/golang-backports +# add-apt-repository ppa:longsleep/golang-backports apt-get update -chmod +x /tmp/install.sh -/tmp/install.sh +if [ -e /tmp/install.sh ]; then + chmod +x /tmp/install.sh + /tmp/install.sh +fi apt-get clean all From 3d709cf834ff878a93bb6a35e2d73227fea024e8 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 17 Jul 2025 09:58:47 +0200 Subject: [PATCH 5/5] SRE-3205 ci: Update for Fedora 42 - redhat-lsb-core no longer supported by Fedora 42 - Fedora 41 and 42 are not available in archive repo, archive repo is only for EOL releases - new test to verify that packaging works for all supported version of Fedora: latest (default), 41, 42 Signed-off-by: Tomasz Gromadzki --- packaging/Dockerfile.mockbuild | 6 +++--- packaging/scripts/repo-helper-fedora.sh | 8 ++++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/packaging/Dockerfile.mockbuild b/packaging/Dockerfile.mockbuild index d8d86ac98..4d8f243dd 100644 --- a/packaging/Dockerfile.mockbuild +++ b/packaging/Dockerfile.mockbuild @@ -26,9 +26,9 @@ RUN chmod +x /tmp/repo-helper.sh && \ rm -f /tmp/repo-helper.sh # Install basic tools -RUN dnf -y install mock make \ - rpm-build createrepo rpmlint redhat-lsb-core git \ - python-srpm-macros rpmdevtools && \ +RUN dnf -y install mock make \ + rpm-build createrepo rpmlint git \ + python-srpm-macros rpmdevtools && \ dnf -y clean all # use same UID as host and default value of 1000 if not specified diff --git a/packaging/scripts/repo-helper-fedora.sh b/packaging/scripts/repo-helper-fedora.sh index baa702664..6c8375e38 100644 --- a/packaging/scripts/repo-helper-fedora.sh +++ b/packaging/scripts/repo-helper-fedora.sh @@ -10,7 +10,11 @@ set -uex : "${REPOSITORY_NAME:=artifactory}" : "${archive:=}" if [ "$FVERSION" != "latest" ]; then - archive="-archive" + if [ "$FVERSION" != "42" ]; then + if [ "$FVERSION" != "41" ]; then + archive="-archive" + fi + fi fi # shellcheck disable=SC2120 @@ -62,7 +66,7 @@ if [ -n "$REPO_FILE_URL" ]; then pushd /etc/yum.repos.d/ curl -k --noproxy '*' -sSf \ -o "daos_ci-fedora${archive}-${REPOSITORY_NAME}.repo" \ - "{$REPO_FILE_URL}daos_ci-fedora${archive}-${REPOSITORY_NAME}.repo" + "${REPO_FILE_URL}daos_ci-fedora${archive}-${REPOSITORY_NAME}.repo" disable_repos /etc/yum.repos.d/ popd fi