From e326286521f3a3f416210d43f55b99bc0fc11c59 Mon Sep 17 00:00:00 2001 From: Kenneth Cain Date: Wed, 9 Dec 2020 11:02:26 -0500 Subject: [PATCH 1/2] Update packaging to build with libdaos.so.1 And update packaging/ subdir contents based on latest daos packaging repo contents. Temporary (to be resolved before landing): romio.spec Source refers to mpich job/PR-44 rather than the usual job/daos_adio-rpm branch. PR-repos: daos@PR-3935:32 PR-repos: mpich@PR-44 Signed-off-by: Kenneth Cain --- Makefile | 7 +- packaged-runtests-centos7.patch | 92 ++++------ packaged-runtests-leap15.patch | 92 ++++------ packaging/Dockerfile.centos.7 | 4 +- packaging/Dockerfile.mockbuild | 11 +- packaging/Dockerfile.ubuntu.18.04 | 29 --- packaging/Dockerfile.ubuntu.20.04 | 39 ++++ packaging/Makefile_distro_vars.mk | 58 ++++-- packaging/Makefile_packaging.mk | 291 +++++++++++++++--------------- packaging/debian_chrootbuild | 43 +++++ packaging/rpm_chrootbuild | 40 ++++ romio.spec | 18 +- 12 files changed, 402 insertions(+), 322 deletions(-) delete mode 100644 packaging/Dockerfile.ubuntu.18.04 create mode 100644 packaging/Dockerfile.ubuntu.20.04 create mode 100755 packaging/debian_chrootbuild create mode 100755 packaging/rpm_chrootbuild diff --git a/Makefile b/Makefile index 90e5df7..5239915 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ -NAME := romio -SRC_EXT := gz +NAME := romio +SRC_EXT := gz +TEST_PACKAGES := romio-tests include packaging/Makefile_packaging.mk @@ -9,4 +10,4 @@ romio-3.3.tar.gz: FORCE $(SRPM): FORCE debug: - echo $(SRPM) \ No newline at end of file + echo $(SRPM) diff --git a/packaged-runtests-centos7.patch b/packaged-runtests-centos7.patch index 9734135..d27f7fe 100644 --- a/packaged-runtests-centos7.patch +++ b/packaged-runtests-centos7.patch @@ -1,6 +1,6 @@ ---- a/test/runtests 2019-12-12 17:06:15.443041123 -0500 -+++ b/test/runtests 2019-12-12 16:05:49.079353484 -0500 -@@ -2,6 +2,7 @@ +--- a/test/runtests 2020-12-16 10:12:27.093075638 -0500 ++++ b/test/runtests 2020-12-16 10:15:04.924143621 -0500 +@@ -7,6 +7,7 @@ prefix=/usr exec_prefix=/usr bindir=/usr/lib64/mpich/bin @@ -8,7 +8,7 @@ mpirun="${bindir}/mpiexec" STOPFILE=${MPITEST_STOPTEST:-"$HOME/.stopmpichtests"} # -@@ -25,7 +26,7 @@ +@@ -29,7 +30,7 @@ ;; -margs=*) shift @@ -18,7 +18,7 @@ ;; -echo) @@ -52,7 +53,7 @@ - daos_pool=1 + subset_only=1 ;; -fname=*) - FILENAME=`echo $arg|sed 's/-*fname=//'` @@ -26,7 +26,7 @@ ;; -atend) shift -@@ -89,7 +90,7 @@ +@@ -87,7 +88,7 @@ echo "Found stopfile; exiting" exit 0 fi @@ -35,7 +35,7 @@ $MAKE $1 if [ ! -x $1 ] ; then echo "Could not build executable $1; aborting tests" -@@ -107,9 +108,9 @@ +@@ -105,9 +106,9 @@ OutTime() { if [ $quiet = 0 ] ; then if [ -z "$hostname" ] ; then @@ -47,7 +47,7 @@ echo "$hostname : $d" fi } -@@ -156,20 +157,20 @@ +@@ -154,13 +155,13 @@ MakeExe simple rm -rf $FILENAME* echo "**** Testing ${1}.c ****" @@ -63,24 +63,7 @@ $MAKE default fi # - - if [ $daos_pool = 1 ] ; then - echo '**** Creating DAOS Pool ****' --pool=`dmg_old create --size=1GB` -+pool=$(dmg_old create --size=1GB) - value=$pool - echo $value - OLDIFS=$IFS -@@ -178,7 +179,7 @@ - IFS=$OLDIFS - export DAOS_POOL=${array[0]} - export DAOS_SVCL=${array[1]} --export DAOS_CONT=`uuidgen` -+export DAOS_CONT=$(uuidgen) - echo '**** Creating DAOS Container ****' - daos cont create --pool=$DAOS_POOL --svc=$DAOS_SVCL --cont=$DAOS_CONT --type=POSIX - fi -@@ -193,7 +194,7 @@ +@@ -175,7 +176,7 @@ MakeExe simple \rm -f $FILENAME* echo '**** Testing simple.c ****' @@ -89,7 +72,7 @@ # CheckOutput simple CleanExe simple # -@@ -203,7 +204,7 @@ +@@ -185,7 +186,7 @@ MakeExe async \rm -f $FILENAME* echo '**** Testing async.c ****' @@ -98,7 +81,7 @@ # CheckOutput async CleanExe async # -@@ -213,7 +214,7 @@ +@@ -195,7 +196,7 @@ MakeExe async-multiple \rm -f $FILENAME* echo '**** Testing async-multiple.c ****' @@ -107,7 +90,7 @@ # CheckOutput async-multiple CleanExe async-multiple # -@@ -226,7 +227,7 @@ +@@ -208,7 +209,7 @@ echo '**** Testing atomicity.c ****' # Atomicity test recommends at least 8 processes (separate processors # even better) @@ -116,7 +99,7 @@ # CheckOutput atomicity CleanExe atomicity # -@@ -237,7 +238,7 @@ +@@ -219,7 +220,7 @@ MakeExe coll_test \rm -f $FILENAME* echo '**** Testing coll_test.c ****' @@ -125,7 +108,7 @@ # CheckOutput coll_test CleanExe coll_test # -@@ -247,7 +248,7 @@ +@@ -229,7 +230,7 @@ MakeExe excl \rm -f $FILENAME* echo '**** Testing excl.c ****' @@ -134,7 +117,7 @@ # CheckOutput excl CleanExe excl # -@@ -257,7 +258,7 @@ +@@ -239,7 +240,7 @@ MakeExe file_info \rm -f $FILENAME* echo '**** Testing file_info.c ****' @@ -143,7 +126,7 @@ # CheckOutput file_info CleanExe file_info # -@@ -267,7 +268,7 @@ +@@ -249,7 +250,7 @@ MakeExe i_noncontig \rm -f $FILENAME* echo '**** Testing i_noncontig.c ****' @@ -152,7 +135,7 @@ # CheckOutput i_noncontig CleanExe i_noncontig # -@@ -277,7 +278,7 @@ +@@ -259,7 +260,7 @@ MakeExe noncontig \rm -f $FILENAME* echo '**** Testing noncontig.c ****' @@ -161,7 +144,7 @@ # CheckOutput noncontig CleanExe noncontig # -@@ -287,7 +288,7 @@ +@@ -269,7 +270,7 @@ MakeExe noncontig_coll \rm -f $FILENAME* echo '**** Testing noncontig_coll.c ****' @@ -170,7 +153,7 @@ # CheckOutput noncontig_coll CleanExe noncontig_coll # -@@ -297,30 +298,30 @@ +@@ -279,30 +280,30 @@ MakeExe noncontig_coll2 \rm -f $FILENAME* echo '**** Testing noncontig_coll2.c ****' @@ -207,7 +190,7 @@ # -offm 4 -hints romio_cb_write enable -fname $FILENAME #CleanExe write_all_test OutTime -@@ -330,7 +331,7 @@ +@@ -312,7 +313,7 @@ MakeExe misc \rm -f $FILENAME* echo '**** Testing misc.c ****' @@ -216,7 +199,7 @@ # CheckOutput misc CleanExe misc # -@@ -342,7 +343,7 @@ +@@ -324,7 +325,7 @@ MakeExe shared_fp \rm -f $FILENAME* echo '**** Testing shared_fp.c ****' @@ -225,7 +208,7 @@ # CheckOutput shared_fp CleanExe shared_fp # -@@ -352,7 +353,7 @@ +@@ -334,7 +335,7 @@ MakeExe ordered_fp \rm -f $FILENAME* echo '**** Testing ordered_fp.c ****' @@ -234,7 +217,7 @@ CleanExe ordered_fp # OutTime -@@ -362,7 +363,7 @@ +@@ -344,7 +345,7 @@ MakeExe split_coll \rm -f $FILENAME* echo '**** Testing split_coll.c ****' @@ -243,7 +226,7 @@ # CheckOutput split_coll CleanExe split_coll # -@@ -372,7 +373,7 @@ +@@ -354,7 +355,7 @@ MakeExe psimple \rm -f $FILENAME* echo '**** Testing psimple.c ****' @@ -252,7 +235,7 @@ # CheckOutput psimple CleanExe psimple # -@@ -382,7 +383,7 @@ +@@ -364,7 +365,7 @@ MakeExe error \rm -f $FILENAME* echo '**** Testing error.c ****' @@ -261,7 +244,7 @@ # CheckOutput error CleanExe error # -@@ -393,7 +394,7 @@ +@@ -375,7 +376,7 @@ \rm -f $FILENAME* echo '**** Testing status.c ****' # Some systems have a status program. @@ -270,7 +253,7 @@ # CheckOutput status CleanExe status testfiles="$testfiles types_with_zeros.out" -@@ -401,18 +402,18 @@ +@@ -383,18 +384,18 @@ MakeExe types_with_zeros \rm -f $FILENAME* echo '**** Testing types_with_zeros ****' @@ -292,7 +275,7 @@ CleanExe syshints # -@@ -426,7 +427,7 @@ +@@ -408,7 +409,7 @@ MakeExe fmisc \rm -f $FILENAME* echo '**** Testing fmisc.f ****' @@ -301,7 +284,7 @@ # CheckOutput fmisc CleanExe fmisc # -@@ -437,7 +438,7 @@ +@@ -419,7 +420,7 @@ MakeExe fcoll_test \rm -f $FILENAME* echo '**** Testing fcoll_test.f ****' @@ -310,7 +293,7 @@ # CheckOutput fcoll_test CleanExe fcoll_test # -@@ -447,7 +448,7 @@ +@@ -429,7 +430,7 @@ MakeExe pfcoll_test \rm -f $FILENAME* echo '**** Testing pfcoll_test.f ****' @@ -319,7 +302,7 @@ # CheckOutput pfcoll_test CleanExe pfcoll_test # -@@ -456,7 +457,7 @@ +@@ -438,7 +439,7 @@ # else # Just run checks @@ -328,16 +311,7 @@ if test "$testfiles" = "*.out" ; then echo "No output files remain from previous test!" exit 1 -@@ -465,7 +466,7 @@ - - if [ $daos_pool = 1 ] ; then - echo '**** Destroying DAOS Pool ****' --pool=`mpirun -np 1 dmg_old destroy --force --pool=${array[0]}` -+pool=$($mpirun -np 1 dmg_old destroy --force --pool=${array[0]}) - fi - - # -@@ -473,14 +474,14 @@ +@@ -450,14 +451,14 @@ # rm -f iotests.diff # nodiff=1 # for file in $testfiles ; do @@ -354,7 +328,7 @@ # rm -f ${bfile} # grep -v 'FORTRAN STOP' ${file} > ${bfile} # if diff -b $bfile $stdfile > /dev/null ; then -@@ -490,18 +491,18 @@ +@@ -467,18 +468,18 @@ # if diff -b $bfile $stdfile2 > /dev/null ; then # true # else diff --git a/packaged-runtests-leap15.patch b/packaged-runtests-leap15.patch index b372e0a..c1b0db6 100644 --- a/packaged-runtests-leap15.patch +++ b/packaged-runtests-leap15.patch @@ -1,6 +1,6 @@ ---- a/test/runtests 2019-12-12 17:06:15.443041123 -0500 -+++ b/test/runtests 2019-12-12 16:05:49.079353484 -0500 -@@ -2,6 +2,7 @@ +--- a/test/runtests 2020-12-04 23:29:23.000000000 -0500 ++++ b/test/runtests 2020-12-16 10:57:27.347672767 -0500 +@@ -7,6 +7,7 @@ prefix=/usr/lib64/mpi/gcc/mpich exec_prefix=/usr/lib64/mpi/gcc/mpich bindir=/usr/lib64/mpi/gcc/mpich/bin @@ -8,7 +8,7 @@ mpirun="${bindir}/mpiexec" STOPFILE=${MPITEST_STOPTEST:-"$HOME/.stopmpichtests"} # -@@ -25,7 +26,7 @@ +@@ -29,7 +30,7 @@ ;; -margs=*) shift @@ -18,7 +18,7 @@ ;; -echo) @@ -52,7 +53,7 @@ - daos_pool=1 + subset_only=1 ;; -fname=*) - FILENAME=`echo $arg|sed 's/-*fname=//'` @@ -26,7 +26,7 @@ ;; -atend) shift -@@ -89,7 +90,7 @@ +@@ -87,7 +88,7 @@ echo "Found stopfile; exiting" exit 0 fi @@ -35,7 +35,7 @@ $MAKE $1 if [ ! -x $1 ] ; then echo "Could not build executable $1; aborting tests" -@@ -107,9 +108,9 @@ +@@ -105,9 +106,9 @@ OutTime() { if [ $quiet = 0 ] ; then if [ -z "$hostname" ] ; then @@ -47,7 +47,7 @@ echo "$hostname : $d" fi } -@@ -156,20 +157,20 @@ +@@ -154,13 +155,13 @@ MakeExe simple rm -rf $FILENAME* echo "**** Testing ${1}.c ****" @@ -63,24 +63,7 @@ $MAKE default fi # - - if [ $daos_pool = 1 ] ; then - echo '**** Creating DAOS Pool ****' --pool=`dmg_old create --size=1GB` -+pool=$(dmg_old create --size=1GB) - value=$pool - echo $value - OLDIFS=$IFS -@@ -178,7 +179,7 @@ - IFS=$OLDIFS - export DAOS_POOL=${array[0]} - export DAOS_SVCL=${array[1]} --export DAOS_CONT=`uuidgen` -+export DAOS_CONT=$(uuidgen) - echo '**** Creating DAOS Container ****' - daos cont create --pool=$DAOS_POOL --svc=$DAOS_SVCL --cont=$DAOS_CONT --type=POSIX - fi -@@ -193,7 +194,7 @@ +@@ -175,7 +176,7 @@ MakeExe simple \rm -f $FILENAME* echo '**** Testing simple.c ****' @@ -89,7 +72,7 @@ # CheckOutput simple CleanExe simple # -@@ -203,7 +204,7 @@ +@@ -185,7 +186,7 @@ MakeExe async \rm -f $FILENAME* echo '**** Testing async.c ****' @@ -98,7 +81,7 @@ # CheckOutput async CleanExe async # -@@ -213,7 +214,7 @@ +@@ -195,7 +196,7 @@ MakeExe async-multiple \rm -f $FILENAME* echo '**** Testing async-multiple.c ****' @@ -107,7 +90,7 @@ # CheckOutput async-multiple CleanExe async-multiple # -@@ -226,7 +227,7 @@ +@@ -208,7 +209,7 @@ echo '**** Testing atomicity.c ****' # Atomicity test recommends at least 8 processes (separate processors # even better) @@ -116,7 +99,7 @@ # CheckOutput atomicity CleanExe atomicity # -@@ -237,7 +238,7 @@ +@@ -219,7 +220,7 @@ MakeExe coll_test \rm -f $FILENAME* echo '**** Testing coll_test.c ****' @@ -125,7 +108,7 @@ # CheckOutput coll_test CleanExe coll_test # -@@ -247,7 +248,7 @@ +@@ -229,7 +230,7 @@ MakeExe excl \rm -f $FILENAME* echo '**** Testing excl.c ****' @@ -134,7 +117,7 @@ # CheckOutput excl CleanExe excl # -@@ -257,7 +258,7 @@ +@@ -239,7 +240,7 @@ MakeExe file_info \rm -f $FILENAME* echo '**** Testing file_info.c ****' @@ -143,7 +126,7 @@ # CheckOutput file_info CleanExe file_info # -@@ -267,7 +268,7 @@ +@@ -249,7 +250,7 @@ MakeExe i_noncontig \rm -f $FILENAME* echo '**** Testing i_noncontig.c ****' @@ -152,7 +135,7 @@ # CheckOutput i_noncontig CleanExe i_noncontig # -@@ -277,7 +278,7 @@ +@@ -259,7 +260,7 @@ MakeExe noncontig \rm -f $FILENAME* echo '**** Testing noncontig.c ****' @@ -161,7 +144,7 @@ # CheckOutput noncontig CleanExe noncontig # -@@ -287,7 +288,7 @@ +@@ -269,7 +270,7 @@ MakeExe noncontig_coll \rm -f $FILENAME* echo '**** Testing noncontig_coll.c ****' @@ -170,7 +153,7 @@ # CheckOutput noncontig_coll CleanExe noncontig_coll # -@@ -297,30 +298,30 @@ +@@ -279,30 +280,30 @@ MakeExe noncontig_coll2 \rm -f $FILENAME* echo '**** Testing noncontig_coll2.c ****' @@ -207,7 +190,7 @@ # -offm 4 -hints romio_cb_write enable -fname $FILENAME #CleanExe write_all_test OutTime -@@ -330,7 +331,7 @@ +@@ -312,7 +313,7 @@ MakeExe misc \rm -f $FILENAME* echo '**** Testing misc.c ****' @@ -216,7 +199,7 @@ # CheckOutput misc CleanExe misc # -@@ -342,7 +343,7 @@ +@@ -324,7 +325,7 @@ MakeExe shared_fp \rm -f $FILENAME* echo '**** Testing shared_fp.c ****' @@ -225,7 +208,7 @@ # CheckOutput shared_fp CleanExe shared_fp # -@@ -352,7 +353,7 @@ +@@ -334,7 +335,7 @@ MakeExe ordered_fp \rm -f $FILENAME* echo '**** Testing ordered_fp.c ****' @@ -234,7 +217,7 @@ CleanExe ordered_fp # OutTime -@@ -362,7 +363,7 @@ +@@ -344,7 +345,7 @@ MakeExe split_coll \rm -f $FILENAME* echo '**** Testing split_coll.c ****' @@ -243,7 +226,7 @@ # CheckOutput split_coll CleanExe split_coll # -@@ -372,7 +373,7 @@ +@@ -354,7 +355,7 @@ MakeExe psimple \rm -f $FILENAME* echo '**** Testing psimple.c ****' @@ -252,7 +235,7 @@ # CheckOutput psimple CleanExe psimple # -@@ -382,7 +383,7 @@ +@@ -364,7 +365,7 @@ MakeExe error \rm -f $FILENAME* echo '**** Testing error.c ****' @@ -261,7 +244,7 @@ # CheckOutput error CleanExe error # -@@ -393,7 +394,7 @@ +@@ -375,7 +376,7 @@ \rm -f $FILENAME* echo '**** Testing status.c ****' # Some systems have a status program. @@ -270,7 +253,7 @@ # CheckOutput status CleanExe status testfiles="$testfiles types_with_zeros.out" -@@ -401,18 +402,18 @@ +@@ -383,18 +384,18 @@ MakeExe types_with_zeros \rm -f $FILENAME* echo '**** Testing types_with_zeros ****' @@ -292,7 +275,7 @@ CleanExe syshints # -@@ -426,7 +427,7 @@ +@@ -408,7 +409,7 @@ MakeExe fmisc \rm -f $FILENAME* echo '**** Testing fmisc.f ****' @@ -301,7 +284,7 @@ # CheckOutput fmisc CleanExe fmisc # -@@ -437,7 +438,7 @@ +@@ -419,7 +420,7 @@ MakeExe fcoll_test \rm -f $FILENAME* echo '**** Testing fcoll_test.f ****' @@ -310,7 +293,7 @@ # CheckOutput fcoll_test CleanExe fcoll_test # -@@ -447,7 +448,7 @@ +@@ -429,7 +430,7 @@ MakeExe pfcoll_test \rm -f $FILENAME* echo '**** Testing pfcoll_test.f ****' @@ -319,7 +302,7 @@ # CheckOutput pfcoll_test CleanExe pfcoll_test # -@@ -456,7 +457,7 @@ +@@ -438,7 +439,7 @@ # else # Just run checks @@ -328,16 +311,7 @@ if test "$testfiles" = "*.out" ; then echo "No output files remain from previous test!" exit 1 -@@ -465,7 +466,7 @@ - - if [ $daos_pool = 1 ] ; then - echo '**** Destroying DAOS Pool ****' --pool=`mpirun -np 1 dmg_old destroy --force --pool=${array[0]}` -+pool=$($mpirun -np 1 dmg_old destroy --force --pool=${array[0]}) - fi - - # -@@ -473,14 +474,14 @@ +@@ -450,14 +451,14 @@ # rm -f iotests.diff # nodiff=1 # for file in $testfiles ; do @@ -354,7 +328,7 @@ # rm -f ${bfile} # grep -v 'FORTRAN STOP' ${file} > ${bfile} # if diff -b $bfile $stdfile > /dev/null ; then -@@ -490,18 +491,18 @@ +@@ -467,18 +468,18 @@ # if diff -b $bfile $stdfile2 > /dev/null ; then # true # else diff --git a/packaging/Dockerfile.centos.7 b/packaging/Dockerfile.centos.7 index 70ba804..2514d84 100644 --- a/packaging/Dockerfile.centos.7 +++ b/packaging/Dockerfile.centos.7 @@ -1,5 +1,5 @@ # -# Copyright 2018-2019, Intel Corporation +# Copyright 2018-2020, Intel Corporation # # 'recipe' for Docker to build an RPM # @@ -28,7 +28,7 @@ RUN echo "$USER:$PASSWD" | chpasswd RUN usermod -a -G mock $USER # mock in Docker needs to use the old-chroot option -RUN grep use_nspawn || \ +RUN grep use_nspawn /etc/mock/site-defaults.cfg || \ echo "config_opts['use_nspawn'] = False" >> /etc/mock/site-defaults.cfg RUN chmod g+w /etc/mock/default.cfg diff --git a/packaging/Dockerfile.mockbuild b/packaging/Dockerfile.mockbuild index 36cc35e..a55b2be 100644 --- a/packaging/Dockerfile.mockbuild +++ b/packaging/Dockerfile.mockbuild @@ -1,5 +1,5 @@ # -# Copyright 2018-2019, Intel Corporation +# Copyright 2018-2020, Intel Corporation # # 'recipe' for Docker to build an RPM # @@ -12,8 +12,9 @@ MAINTAINER daos-stack ARG UID=1000 # Install basic tools -RUN dnf -y install mock make rpm-build curl createrepo rpmlint redhat-lsb-core \ - git python-srpm-macros rpmdevtools +RUN dnf -y install mock make \ + rpm-build curl createrepo rpmlint redhat-lsb-core git \ + python-srpm-macros rpmdevtools # Add build user (to keep rpmbuild happy) ENV USER build @@ -24,7 +25,7 @@ RUN echo "$USER:$PASSWD" | chpasswd RUN usermod -a -G mock $USER # mock in Docker needs to use the old-chroot option -RUN grep use_nspawn || \ +RUN grep use_nspawn /etc/mock/site-defaults.cfg || \ echo "config_opts['use_nspawn'] = False" >> /etc/mock/site-defaults.cfg -RUN chmod g+w /etc/mock/* \ No newline at end of file +RUN chmod g+w /etc/mock/* diff --git a/packaging/Dockerfile.ubuntu.18.04 b/packaging/Dockerfile.ubuntu.18.04 deleted file mode 100644 index 2180c9a..0000000 --- a/packaging/Dockerfile.ubuntu.18.04 +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright 2019, Intel Corporation -# -# 'recipe' for Docker to build an Debian package -# -# Pull base image -FROM ubuntu:18.04 -Maintainer daos-stack - -# use same UID as host and default value of 1000 if not specified -ARG UID=1000 - -# Install basic tools -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - autoconf bash curl debhelper dh-make dpkg-dev doxygen gcc \ - git git-buildpackage locales make patch pbuilder rpm wget - -# Add build user (to keep chrootbuild happy) -ENV USER build -RUN useradd -u $UID -ms /bin/bash $USER - -# need to run the build command as root, as it needs to chroot -RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then \ - echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \ - fi; \ - echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" > /etc/sudoers.d/build; \ - chmod 0440 /etc/sudoers.d/build; \ - visudo -c; \ - sudo -l -U build diff --git a/packaging/Dockerfile.ubuntu.20.04 b/packaging/Dockerfile.ubuntu.20.04 new file mode 100644 index 0000000..cf16bce --- /dev/null +++ b/packaging/Dockerfile.ubuntu.20.04 @@ -0,0 +1,39 @@ +# +# Copyright 2019-2020, Intel Corporation +# +# 'recipe' for Docker to build an Debian package +# +# Pull base image +FROM ubuntu:20.04 +MAINTAINER daos-stack + +# use same UID as host and default value of 1000 if not specified +ARG UID=1000 +ARG REPO_URL="" +ARG REPO_UBUNTU_20_04="" + +# Install basic tools +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + autoconf bash curl debhelper dh-make dpkg-dev doxygen gcc \ + git git-buildpackage locales make patch pbuilder rpm wget \ + ca-certificates scons python3-distutils pkg-config \ + python3-dev python3-distro + +# rpmdevtools +RUN echo "deb [trusted=yes] ${REPO_URL}${REPO_UBUNTU_20_04} focal main" > /etc/apt/sources.list.d/daos-stack-ubuntu-stable-local.list +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + rpmdevtools + +# Add build user (to keep chrootbuild happy) +ENV USER build +RUN useradd -u $UID -ms /bin/bash $USER + +# need to run the build command as root, as it needs to chroot +RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then \ + echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \ + fi; \ + echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build; \ + chmod 0440 /etc/sudoers.d/build; \ + visudo -c; \ + sudo -l -U build diff --git a/packaging/Makefile_distro_vars.mk b/packaging/Makefile_distro_vars.mk index 6044fdb..c126c81 100644 --- a/packaging/Makefile_distro_vars.mk +++ b/packaging/Makefile_distro_vars.mk @@ -1,19 +1,25 @@ +DOT := . + +DOCKER := docker + # Find out what we are -ID_LIKE := $(shell . /etc/os-release; echo $$ID_LIKE) +ID_LIKE := $(shell . /etc/os-release; echo $$ID_LIKE) # Of course that does not work for SLES-12 -ID := $(shell . /etc/os-release; echo $$ID) -VERSION_ID := $(shell . /etc/os-release; echo $$VERSION_ID) +ID := $(shell . /etc/os-release; echo $$ID) +VERSION_ID := $(shell . /etc/os-release; echo $$VERSION_ID) +VERSION_CODENAME := $(shell . /etc/os-release; echo $$VERSION_CODENAME) + ifeq ($(ID_LIKE),debian) -UBUNTU_VERS := $(shell . /etc/os-release; echo $$VERSION) -ifeq ($(VERSION_ID),19.04) -# Bug - distribution is set to "devel" -DISTRO_ID_OPT = --distribution disco -endif -DISTRO_ID := ubuntu$(VERSION_ID) -DISTRO_BASE = $(basename UBUNTU_$(VERSION_ID)) +SPECTOOL := spectool +UBUNTU_VERS := $(shell . /etc/os-release; echo $$VERSION) +DISTRO_ID_OPT := --distribution $(VERSION_CODENAME) +DISTRO_ID := ubuntu$(VERSION_ID) VERSION_ID_STR := $(subst $(DOT),_,$(VERSION_ID)) +DISTRO_BASE := UBUNTU_$(VERSION_ID_STR) endif ifeq ($(ID),fedora) +DOCKER := podman +SPECTOOL := spectool # a Fedora-based mock builder # derive the the values of: # VERSION_ID (i.e. 7) @@ -27,12 +33,25 @@ DISTRO_ID := el7 DISTRO_BASE := EL_7 SED_EXPR := 1s/$(DIST)//p endif +ifeq ($(CHROOT_NAME),epel-8-x86_64) +DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) +VERSION_ID := 8 +DISTRO_ID := el8 +DISTRO_BASE := EL_8 +SED_EXPR := 1s/$(DIST)//p +endif ifeq ($(CHROOT_NAME),opensuse-leap-15.1-x86_64) VERSION_ID := 15.1 DISTRO_ID := sl15.1 DISTRO_BASE := LEAP_15 SED_EXPR := 1p endif +ifeq ($(CHROOT_NAME),opensuse-leap-15.2-x86_64) +VERSION_ID := 15.2 +DISTRO_ID := sl15.2 +DISTRO_BASE := LEAP_15 +SED_EXPR := 1p +endif ifeq ($(CHROOT_NAME),leap-42.3-x86_64) # TBD if support is ever resurrected endif @@ -45,6 +64,7 @@ DISTRO_ID := el$(VERSION_ID) DISTRO_BASE := $(basename EL_$(VERSION_ID)) DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) SED_EXPR := 1s/$(DIST)//p +SPECTOOL := spectool define install_repo if yum-config-manager --add-repo=$(1); then \ repo_file=$$(ls -tar /etc/yum.repos.d/*.repo | tail -1); \ @@ -66,7 +86,23 @@ DISTRO_ID := sle$(VERSION_ID) DISTRO_BASE := $(basename SLES_$(VERSION_ID)) endif ifeq ($(ID_LIKE),suse) +SPECTOOL := rpmdev-spectool define install_repo zypper --non-interactive ar $(1) endef -endif \ No newline at end of file +endif +ifeq ($(ID_LIKE),debian) +ifndef LANG +export LANG = C.UTF-8 +endif +ifndef LC_ALL +export LC_ALL = C.UTF-8 +endif +else +ifndef LANG +export LANG = C.utf8 +endif +ifndef LC_ALL +export LC_ALL = C.utf8 +endif +endif diff --git a/packaging/Makefile_packaging.mk b/packaging/Makefile_packaging.mk index 08166b4..a15009d 100644 --- a/packaging/Makefile_packaging.mk +++ b/packaging/Makefile_packaging.mk @@ -1,11 +1,16 @@ # Common Makefile for including -# Needs the following variables set at a minium: +# Needs the following variables set at a minimum: # NAME := # SRC_EXT := +# force bash (looking at you Ubuntu) +SHELL=/bin/bash + # Put site overrides (i.e. REPOSITORY_URL, DAOS_STACK_*_LOCAL_REPO) in here -include Makefile.local +# default to Leap 15 distro for chrootbuild +CHROOT_NAME ?= opensuse-leap-15.2-x86_64 include packaging/Makefile_distro_vars.mk ifeq ($(DEB_NAME),) @@ -14,69 +19,75 @@ endif CALLING_MAKEFILE := $(word 1, $(MAKEFILE_LIST)) +TOPDIR ?= $(CURDIR) +BUILD_PREFIX ?= . + DOT := . RPM_BUILD_OPTIONS += $(EXTERNAL_RPM_BUILD_OPTIONS) # some defaults the caller can override -BUILD_OS ?= leap.15 -CHROOT_NAME ?= opensuse-leap-15-x86_64 PACKAGING_CHECK_DIR ?= ../packaging LOCAL_REPOS ?= true +TEST_PACKAGES ?= ${NAME} + +# unfortunately we cannot always name the repo the same as the project +REPO_NAME ?= $(NAME) + +PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos: *\(.*\)/\1/p') +LEAP_15_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-leap15: *\(.*\)/\1/p') +EL_7_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el7: *\(.*\)/\1/p') +UBUNTU_20_04_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-ubuntu20: *\(.*\)/\1/p') -COMMON_RPM_ARGS := --define "%_topdir $$PWD/_topdir" $(BUILD_DEFINES) +COMMON_RPM_ARGS := --define "_topdir $$PWD/_topdir" $(BUILD_DEFINES) SPEC := $(shell if [ -f $(NAME)-$(DISTRO_BASE).spec ]; then echo $(NAME)-$(DISTRO_BASE).spec; else echo $(NAME).spec; fi) VERSION = $(eval VERSION := $(shell rpm $(COMMON_RPM_ARGS) --specfile --qf '%{version}\n' $(SPEC) | sed -n '1p'))$(VERSION) -DEB_VERS := $(subst rc,~rc,$(VERSION)) -DEB_RVERS := $(subst $(DOT),\$(DOT),$(DEB_VERS)) -DEB_BVERS := $(basename $(subst ~rc,$(DOT)rc,$(DEB_VERS))) +DEB_RVERS := $(subst $(DOT),\$(DOT),$(VERSION)) +DEB_BVERS := $(basename $(subst ~rc,$(DOT)rc,$(VERSION))) RELEASE = $(eval RELEASE := $(shell rpm $(COMMON_RPM_ARGS) --specfile --qf '%{release}\n' $(SPEC) | sed -n '$(SED_EXPR)'))$(RELEASE) SRPM = _topdir/SRPMS/$(NAME)-$(VERSION)-$(RELEASE)$(DIST).src.rpm RPMS = $(eval RPMS := $(addsuffix .rpm,$(addprefix _topdir/RPMS/x86_64/,$(shell rpm --specfile $(SPEC)))))$(RPMS) DEB_TOP := _topdir/BUILD -DEB_BUILD := $(DEB_TOP)/$(NAME)-$(DEB_VERS) -DEB_TARBASE := $(DEB_TOP)/$(DEB_NAME)_$(DEB_VERS) -SOURCE = $(eval SOURCE := $(shell CHROOT_NAME=$(CHROOT_NAME) spectool -S -l $(SPEC) | sed -e 2,\$$d -e 's/.*: *//'))$(SOURCE) -PATCHES = $(eval PATCHES := $(shell CHROOT_NAME=$(CHROOT_NAME) spectool -l $(SPEC) | sed -e 1d -e 's/.*: *//' -e 's/.*\///'))$(PATCHES) -SOURCES := $(addprefix _topdir/SOURCES/,$(notdir $(SOURCE)) $(PATCHES)) +DEB_BUILD := $(DEB_TOP)/$(NAME)-$(VERSION) +DEB_TARBASE := $(DEB_TOP)/$(DEB_NAME)_$(VERSION) +SOURCE ?= $(eval SOURCE := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -S -l $(SPEC) | sed -e 2,\$$d -e 's/\\\#/\\\\\#/g' -e 's/.*: *//'))$(SOURCE) +PATCHES ?= $(eval PATCHES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e 's/.*: *//' -e 's/.*\///' -e '/\.patch/p'))$(PATCHES) +OTHER_SOURCES := $(eval OTHER_SOURCES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e 's/.*: *//' -e 's/.*\///' -e '/\.patch/d' -e p))$(OTHER_SOURCES) +SOURCES := $(addprefix _topdir/SOURCES/,$(notdir $(SOURCE)) $(PATCHES) $(OTHER_SOURCES)) ifeq ($(ID_LIKE),debian) -DEBS := $(addsuffix _$(DEB_VERS)-1_amd64.deb,$(shell sed -n '/-udeb/b; s,^Package:[[:blank:]],$(DEB_TOP)/,p' debian/control)) -DEB_PREV_RELEASE := $(shell dpkg-parsechangelog -S version) -DEB_DSC := $(DEB_NAME)_$(DEB_PREV_RELEASE)$(GIT_INFO).dsc -#Ubuntu Containers do not set a UTF-8 environment by default. -ifndef LANG -export LANG = C.UTF-8 -endif -ifndef LC_ALL -export LC_ALL = C.UTF-8 +DEBS := $(addsuffix _$(VERSION)-1_amd64.deb,$(shell sed -n '/-udeb/b; s,^Package:[[:blank:]],$(DEB_TOP)/,p' $(TOPDIR)/debian/control)) +DEB_PREV_RELEASE := $(shell cd $(TOPDIR) && dpkg-parsechangelog -S version) +ifneq ($(GIT_SHORT),) +GIT_INFO ?= .$(GIT_NUM_COMMITS).g$(GIT_SHORT) endif +DEB_DSC := $(DEB_NAME)_$(DEB_PREV_RELEASE)$(GIT_INFO).dsc TARGETS := $(DEBS) else -# CentOS/Suse packages that want a locale set need this. -ifndef LANG -export LANG = en_US.utf8 -endif -ifndef LC_ALL -export LC_ALL = en_US.utf8 -endif TARGETS := $(RPMS) $(SRPM) endif define distro_map - case $(DISTRO_ID) in \ - el7) distro="centos7" \ - ;; \ - el8) distro="centos8" \ - ;; \ - sle12.3) distro="sles12.3" \ - ;; \ - sl42.3) distro="leap42.3" \ - ;; \ - sl15.1) distro="leap15" \ - ;; \ - esac; + case $(DISTRO_ID) in \ + el7) distro="centos7" \ + ;; \ + el8) distro="centos8" \ + ;; \ + sle12.3) distro="sles12.3" \ + ;; \ + sl42.3) distro="leap42.3" \ + ;; \ + sl15.*) distro="leap15" \ + ;; \ + ubuntu*) distro="$(DISTRO_ID)" \ + ;; \ + esac; endef define install_repos + IFS='|' read -ra BASES <<< "$($(DISTRO_BASE)_LOCAL_REPOS)"; \ + for baseurl in "$${BASES[@]}"; do \ + baseurl="$${baseurl# *}"; \ + $(call install_repo,$$baseurl); \ + done for repo in $($(DISTRO_BASE)_PR_REPOS) \ $(PR_REPOS) $(1); do \ branch="master"; \ @@ -122,20 +133,27 @@ _topdir/SOURCES/%: % | _topdir/SOURCES/ ifeq ($(DL_VERSION),) DL_VERSION = $(VERSION) endif +ifeq ($(DL_NAME),) +DL_NAME = $(NAME) +endif -$(NAME)-$(DL_VERSION).tar.$(SRC_EXT).asc: - rm -f ./$(NAME)-*.tar.{gz,bz*,xz}.asc +$(DL_NAME)-$(DL_VERSION).tar.$(SRC_EXT).asc: $(SPEC) $(CALLING_MAKEFILE) + rm -f ./$(DL_NAME)-*.tar.{gz,bz*,xz}.asc curl -f -L -O '$(SOURCE).asc' -$(NAME)-$(DL_VERSION).tar.$(SRC_EXT): - rm -f ./$(NAME)-*.tar.{gz,bz*,xz} +$(DL_NAME)-$(DL_VERSION).tar.$(SRC_EXT).sig: $(SPEC) $(CALLING_MAKEFILE) + rm -f ./$(DL_NAME)-*.tar.{gz,bz*,xz}.sig + curl -f -L -O '$(SOURCE).sig' + +$(DL_NAME)-$(DL_VERSION).tar.$(SRC_EXT): $(SPEC) $(CALLING_MAKEFILE) + rm -f ./$(DL_NAME)-*.tar.{gz,bz*,xz} curl -f -L -O '$(SOURCE)' -v$(DL_VERSION).tar.$(SRC_EXT): +v$(DL_VERSION).tar.$(SRC_EXT): $(SPEC) $(CALLING_MAKEFILE) rm -f ./v*.tar.{gz,bz*,xz} curl -f -L -O '$(SOURCE)' -$(DL_VERSION).tar.$(SRC_EXT): +$(DL_VERSION).tar.$(SRC_EXT): $(SPEC) $(CALLING_MAKEFILE) rm -f ./*.tar.{gz,bz*,xz} curl -f -L -O '$(SOURCE)' @@ -146,7 +164,7 @@ $(DEB_BUILD)/%: % | $(DEB_BUILD)/ $(DEB_BUILD).tar.$(SRC_EXT): $(notdir $(SOURCE)) | $(DEB_TOP)/ ln -f $< $@ -$(DEB_TARBASE).orig.tar.$(SRC_EXT) : $(DEB_BUILD).tar.$(SRC_EXT) +$(DEB_TARBASE).orig.tar.$(SRC_EXT): $(DEB_BUILD).tar.$(SRC_EXT) rm -f $(DEB_TOP)/*.orig.tar.* ln -f $< $@ @@ -189,19 +207,20 @@ $(DEB_TOP)/.patched: $(PATCHES) check-env deb_detar | \ # Move the debian files into the Debian directory. ifeq ($(ID_LIKE),debian) -$(DEB_TOP)/.deb_files : $(shell find debian -type f) deb_detar | \ +$(DEB_TOP)/.deb_files: $(shell find $(TOPDIR)/debian -type f) deb_detar | \ $(DEB_BUILD)/debian/ - find debian -maxdepth 1 -type f -exec cp '{}' '$(DEB_BUILD)/{}' ';' - if [ -e debian/source ]; then \ - cp -r debian/source $(DEB_BUILD)/debian; fi - if [ -e debian/local ]; then \ - cp -r debian/local $(DEB_BUILD)/debian; fi - if [ -e debian/examples ]; then \ - cp -r debian/examples $(DEB_BUILD)/debian; fi - if [ -e debian/upstream ]; then \ - cp -r debian/upstream $(DEB_BUILD)/debian; fi - if [ -e debian/tests ]; then \ - cp -r debian/tests $(DEB_BUILD)/debian; fi + cd $(TOPDIR)/ && \ + find debian -maxdepth 1 -type f -exec cp '{}' '$(BUILD_PREFIX)/$(DEB_BUILD)/{}' ';' + if [ -e $(TOPDIR)/debian/source ]; then \ + cp -r $(TOPDIR)/debian/source $(DEB_BUILD)/debian; fi + if [ -e $(TOPDIR)/debian/local ]; then \ + cp -r $(TOPDIR)/debian/local $(DEB_BUILD)/debian; fi + if [ -e $(TOPDIR)/debian/examples ]; then \ + cp -r $(TOPDIR)/debian/examples $(DEB_BUILD)/debian; fi + if [ -e $(TOPDIR)/debian/upstream ]; then \ + cp -r $(TOPDIR)/debian/upstream $(DEB_BUILD)/debian; fi + if [ -e $(TOPDIR)/debian/tests ]; then \ + cp -r $(TOPDIR)/debian/tests $(DEB_BUILD)/debian; fi rm -f $(DEB_BUILD)/debian/*.ex $(DEB_BUILD)/debian/*.EX rm -f $(DEB_BUILD)/debian/*.orig ifneq ($(GIT_INFO),) @@ -228,7 +247,7 @@ $(subst deb,%,$(DEBS)): $(DEB_BUILD).tar.$(SRC_EXT) \ cd $(DEB_BUILD); debuild -- clean git status rm -rf $(DEB_TOP)/$(NAME)-tmp - lfile1=$(shell echo $(DEB_TOP)/$(NAME)[0-9]*_$(DEB_VERS)-1_amd64.deb);\ + lfile1=$(shell echo $(DEB_TOP)/$(NAME)[0-9]*_$(VERSION)-1_amd64.deb);\ lfile=$$(ls $${lfile1}); \ lfile2=$${lfile##*/}; lname=$${lfile2%%_*}; \ dpkg-deb -R $${lfile} \ @@ -269,95 +288,58 @@ ls: $(TARGETS) # *_LOCAL_* repos are locally built packages. # *_GROUP_* repos are a local mirror of a group of upstream repos. # *_GROUP_* repos may not supply a repomd.xml.key. +ifeq ($(LOCAL_REPOS),true) ifneq ($(REPOSITORY_URL),) ifneq ($(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO),) -$(DISTRO_BASE)_LOCAL_REPOS := $(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)/ -endif -ifneq ($(DAOS_STACK_$(DISTRO_BASE)_GROUP_REPO),) -$(DISTRO_BASE)_LOCAL_REPOS += $(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_GROUP_REPO)/ -endif -endif - ifeq ($(ID_LIKE),debian) -ifneq ($(DAOS_STACK_REPO_SUPPORT),) -TEST_STR := $(DAOS_STACK_REPO_UBUNTU_$(VERSION_ID_STR)_LIST) -ifneq ($(TEST_STR),) -UBUNTU_REPOS := $(shell curl $(DAOS_STACK_REPO_SUPPORT)$(TEST_STR)) -# Additional repos can be added but must be separated by a | character. -UBUNTU_ADD_REPOS = --othermirror "$(UBUNTU_REPOS)" -else -ifneq ($(DAOS_STACK_REPO_UBUNTU_ROLLING_LIST),) -UBUNTU_REPOS := $(shell curl $(DAOS_STACK_REPO_SUPPORT)$(DAOS_STACK_REPO_UBUNTU_ROLLING_LIST)) -# Additional repos can be added but must be separated by a | character. -UBUNTU_ADD_REPOS = --othermirror "$(UBUNTU_REPOS)" +# $(DISTRO_BASE)_LOCAL_REPOS is a list separated by | because you cannot pass lists +# of values with spaces as environment variables +$(DISTRO_BASE)_LOCAL_REPOS := [trusted=yes] endif +$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS) $(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)/ endif -# Need to figure out how to support multiple keys, such as for IPMCTL -ifneq ($(DAOS_STACK_REPO_PUB_KEY),) -HAVE_DAOS_STACK_KEY := TRUE - -$(DAOS_STACK_REPO_PUB_KEY): - curl -f -L -O '$(DAOS_STACK_REPO_SUPPORT)$(DAOS_STACK_REPO_PUB_KEY)' +$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)| +ifneq ($(DAOS_STACK_$(DISTRO_BASE)_GROUP_REPO),) +$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)$(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_GROUP_REPO)/| endif endif - -chrootbuild: $(DEB_TOP)/$(DEB_DSC) $(DAOS_STACK_REPO_PUB_KEY) - sudo pbuilder create \ - --extrapackages "gnupg ca-certificates" $(DISTRO_ID_OPT) -ifneq ($(HAVE_DAOS_STACK_KEY),) - printf "apt-key add - <> /etc/mock/$(CHROOT_NAME).cfg; \ - $(call distro_map) \ - for repo in $($(DISTRO_BASE)_PR_REPOS) $(PR_REPOS); do \ - branch="master"; \ - build_number="lastSuccessfulBuild"; \ - if [[ $$repo = *@* ]]; then \ - branch="$${repo#*@}"; \ - repo="$${repo%@*}"; \ - if [[ $$branch = *:* ]]; then \ - build_number="$${branch#*:}"; \ - branch="$${branch%:*}"; \ - fi; \ - fi; \ - echo -e "[$$repo:$$branch:$$build_number]\n\ -name=$$repo:$$branch:$$build_number\n\ -baseurl=$${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$$repo/job/$$branch/$$build_number/artifact/artifacts/$$distro/\n\ -enabled=1\n\ -gpgcheck=False\n" >> /etc/mock/$(CHROOT_NAME).cfg; \ - done; \ - if ! $(LOCAL_REPOS); then \ - LOCAL_REPOS=""; \ - else \ - LOCAL_REPOS="$($(DISTRO_BASE)_LOCAL_REPOS)"; \ - fi; \ - for repo in $$LOCAL_REPOS $($(DISTRO_BASE)_REPOS); do \ - repo_name=$${repo##*://}; \ - repo_name=$${repo_name//\//_}; \ - echo -e "[$$repo_name]\n\ -name=$${repo_name}\n\ -baseurl=$${repo}\n\ -enabled=1\n" >> /etc/mock/$(CHROOT_NAME).cfg; \ - done; \ - echo "\"\"\"" >> /etc/mock/$(CHROOT_NAME).cfg; \ - else \ - echo "Unable to update /etc/mock/$(CHROOT_NAME).cfg."; \ - echo "You need to make sure it has the needed repos in it yourself."; \ - fi - mock -r $(CHROOT_NAME) $(MOCK_OPTIONS) $(RPM_BUILD_OPTIONS) $< + $(call distro_map) \ + DISTRO="$$distro" \ + CHROOT_NAME="$(CHROOT_NAME)" \ + PR_REPOS="$(PR_REPOS)" \ + DISTRO_BASE_PR_REPOS="$($(DISTRO_BASE)_PR_REPOS)" \ + JENKINS_URL="$${JENKINS_URL}" \ + JOB_REPOS="$(JOB_REPOS)" \ + DISTRO_BASE_LOCAL_REPOS="$($(DISTRO_BASE)_LOCAL_REPOS)" \ + MOCK_OPTIONS="$(MOCK_OPTIONS)" \ + RPM_BUILD_OPTIONS='$(RPM_BUILD_OPTIONS)' \ + TARGET="$<" \ + packaging/rpm_chrootbuild endif docker_chrootbuild: - docker build --build-arg UID=$$(id -u) -t $(BUILD_OS)-chrootbuild \ - -f packaging/Dockerfile.$(BUILD_OS) . - docker run --privileged=true -w $$PWD -v=$$PWD:$$PWD \ - -it $(BUILD_OS)-chrootbuild bash -c "make chrootbuild" + $(DOCKER) build --build-arg UID=$$(id -u) -t chrootbuild \ + -f packaging/Dockerfile.mockbuild . + $(DOCKER) run --privileged=true -w $(TOPDIR) -v=$(TOPDIR):$(TOPDIR) \ + -it chrootbuild bash -c "make -C $(CURDIR) \ + CHROOT_NAME=$(CHROOT_NAME) chrootbuild" rpmlint: $(SPEC) rpmlint $< @@ -392,34 +374,51 @@ ifndef DEBFULLNAME endif test: - @echo "No test defined for this module" + # Test the rpmbuild by installing the built RPM + $(call install_repos,$(REPO_NAME)@$(BRANCH_NAME):$(BUILD_NUMBER)) + yum -y install $(TEST_PACKAGES) + +show_spec: + @echo '$(SPEC)' + +show_build_defines: + @echo '$(BUILD_DEFINES)' + +show_common_rpm_args: + @echo '$(COMMON_RPM_ARGS)' show_version: - @echo $(VERSION) + @echo '$(VERSION)' show_release: - @echo $(RELEASE) + @echo '$(RELEASE)' show_rpms: - @echo $(RPMS) + @echo '$(RPMS)' show_source: - @echo $(SOURCE) + @echo '$(SOURCE)' + +show_patches: + @echo '$(PATCHES)' show_sources: - @echo $(SOURCES) + @echo '$(SOURCES)' + +show_other_sources: + @echo '$(OTHER_SOURCES)' show_targets: - @echo $(TARGETS) + @echo '$(TARGETS)' show_makefiles: - @echo $(MAKEFILE_LIST) + @echo '$(MAKEFILE_LIST)' show_calling_makefile: - @echo $(CALLING_MAKEFILE) + @echo '$(CALLING_MAKEFILE)' show_git_metadata: - @echo $(GIT_SHA1):$(GIT_SHORT):$(GIT_NUM_COMMITS) + @echo '$(GIT_SHA1):$(GIT_SHORT):$(GIT_NUM_COMMITS)' .PHONY: srpm rpms debs deb_detar ls chrootbuild rpmlint FORCE \ show_version show_release show_rpms show_source show_sources \ diff --git a/packaging/debian_chrootbuild b/packaging/debian_chrootbuild new file mode 100755 index 0000000..5ea4ead --- /dev/null +++ b/packaging/debian_chrootbuild @@ -0,0 +1,43 @@ +#!/bin/bash + +set -uex + +# shellcheck disable=SC2153 +IFS=\| read -r -a distro_base_local_repos <<< "$DISTRO_BASE_LOCAL_REPOS" + +# shellcheck disable=SC2086 +sudo pbuilder create \ + --extrapackages "gnupg ca-certificates" \ + --othermirror \ + "deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ $VERSION_CODENAME universe|deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ $VERSION_CODENAME-updates main universe" \ + $DISTRO_ID_OPT + +repo_args="" +for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do + branch="master" + build_number="lastSuccessfulBuild" + if [[ $repo = *@* ]]; then + branch="${repo#*@}" + repo="${repo%@*}" + if [[ $branch = *:* ]]; then + build_number="${branch#*:}" + branch="${branch%:*}" + fi + fi + repo_args="$repo_args|deb [trusted=yes] ${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/$branch/$build_number/artifact/artifacts/$DISTRO/ ./" +done +for repo in $JOB_REPOS "${distro_base_local_repos[@]}"; do + repo_args="$repo_args|deb ${repo} $VERSION_CODENAME main" +done +echo "$repo_args" +if [ "$repo_args" = "|" ]; then + repo_args="" +else + #repo_args="--othermirror"${repo_args#|}\"" + repo_args="${repo_args#|}" +fi +cd "$DEB_TOP" +# shellcheck disable=SC2086 +sudo pbuilder update --override-config $DISTRO_ID_OPT ${repo_args:+--othermirror "$repo_args"} +# fail the build if the *.symbols file(s) need updating +sudo DPKG_GENSYMBOLS_CHECK_LEVEL="${DPKG_GENSYMBOLS_CHECK_LEVEL:-4}" pbuilder build "$DEB_DSC" diff --git a/packaging/rpm_chrootbuild b/packaging/rpm_chrootbuild new file mode 100755 index 0000000..321d22f --- /dev/null +++ b/packaging/rpm_chrootbuild @@ -0,0 +1,40 @@ +#!/bin/bash + +set -uex + +# shellcheck disable=SC2153 +IFS=\| read -r -a distro_base_local_repos <<< "$DISTRO_BASE_LOCAL_REPOS" + +if [ -w /etc/mock/"$CHROOT_NAME".cfg ]; then + echo -e "config_opts['yum.conf'] += \"\"\"\n" >> /etc/mock/"$CHROOT_NAME".cfg + for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do + branch="master" + build_number="lastSuccessfulBuild" + if [[ $repo = *@* ]]; then + branch="${repo#*@}" + repo="${repo%@*}" + if [[ $branch = *:* ]]; then + build_number="${branch#*:}" + branch="${branch%:*}" + fi + fi + echo -e "[$repo:$branch:$build_number]\n\ +name=$repo:$branch:$build_number\n\ +baseurl=${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/$branch/$build_number/artifact/artifacts/$DISTRO/\n\ +enabled=1\n\ +gpgcheck=False\n" >> /etc/mock/"$CHROOT_NAME".cfg + done + for repo in $JOB_REPOS "${distro_base_local_repos[@]}"; do + repo_name=${repo##*://} + repo_name=${repo_name//\//_} + echo -e "[${repo_name//@/_}]\n\ +name=${repo_name}\n\ +baseurl=${repo}\n\ +enabled=1\n" >> /etc/mock/"$CHROOT_NAME".cfg + done + echo "\"\"\"" >> /etc/mock/"$CHROOT_NAME".cfg +else + echo "Unable to update /etc/mock/$CHROOT_NAME.cfg." + echo "You need to make sure it has the needed repos in it yourself." +fi +eval mock -r "$CHROOT_NAME" $MOCK_OPTIONS $RPM_BUILD_OPTIONS "$TARGET" diff --git a/romio.spec b/romio.spec index 78f29bd..fbf277e 100644 --- a/romio.spec +++ b/romio.spec @@ -1,10 +1,9 @@ %global romio_home %{_libdir}/romio -%global cart_major 4 -%global daos_major 0 +%global daos_major 1 Name: romio -Version: 3.3 -Release: 3%{?dist} +Version: 3.4~a2 +Release: 1%{?dist} Summary: ROMIO License: MIT @@ -15,7 +14,7 @@ URL: http://www.mpich.org/ %if ("%{?chroot_name}" == "epel-7-x86_64") %define distro centos7 %else -%if ("%{?chroot_name}" == "opensuse-leap-15.1-x86_64") +%if "%{?chroot_name}" == "opensuse-leap-15.1-x86_64" || "%{?chroot_name}" == "opensuse-leap-15.2-x86_64" %define distro leap15 %else %if (0%{?suse_version} >= 1500) && (0%{?suse_version} < 1600) @@ -25,7 +24,7 @@ URL: http://www.mpich.org/ %endif %endif %endif -Source0: https://build.hpdd.intel.com/job/daos-stack/job/mpich/job/daos_adio-rpm/lastSuccessfulBuild/artifact/artifacts/%{distro}/%{name}-%{version}.tar.gz +Source0: https://build.hpdd.intel.com/job/daos-stack/job/mpich/job/PR-44/lastSuccessfulBuild/artifact/artifacts/%{distro}/%{name}-%{version}.tar.gz Patch0: packaged-runtests-%{distro}.patch BuildRequires: mpich-devel @@ -34,14 +33,14 @@ BuildRequires: daos-devel %if (0%{?suse_version} >= 1500) BuildRequires: gcc-fortran %endif -Provides: %{name}-cart-%{cart_major}-daos-%{daos_major} +Provides: %{name}-daos-%{daos_major} %description ROMIO %package tests Summary: ROMIO tests -Provides: %{name}-tests-cart-%{cart_major}-daos-%{daos_major} +Provides: %{name}-tests-daos-%{daos_major} %description tests ROMIO tests @@ -75,6 +74,9 @@ done %license %changelog +* Tue Dec 08 2020 Kenneth Cain - 3.4~a2-1 +- Update packaging to build with libdaos.so.1 + * Tue Jan 21 2020 Brian J. Murrell - 3.3-3 - Add Leap 15.1 support From 6229b7f82a1e99edd0beb3c77932d9da3cc27acb Mon Sep 17 00:00:00 2001 From: Kenneth Cain Date: Tue, 19 Jan 2021 15:27:11 -0500 Subject: [PATCH 2/2] restore Source URL to daos_adio-rpm branch successful build And update packaging to latest daos-stack/packaging repo contents. Signed-off-by: Kenneth Cain --- packaging/rpm_chrootbuild | 7 +++++++ romio.spec | 7 +++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/packaging/rpm_chrootbuild b/packaging/rpm_chrootbuild index 321d22f..e2b201d 100755 --- a/packaging/rpm_chrootbuild +++ b/packaging/rpm_chrootbuild @@ -6,6 +6,13 @@ set -uex IFS=\| read -r -a distro_base_local_repos <<< "$DISTRO_BASE_LOCAL_REPOS" if [ -w /etc/mock/"$CHROOT_NAME".cfg ]; then + # Remove the distro repos + if [[ $CHROOT_NAME =~ opensuse-leap-* ]]; then + ed /etc/mock/"$CHROOT_NAME".cfg << EOF +/^# repos$/+2;/^"""$/-1d +wq +EOF + fi echo -e "config_opts['yum.conf'] += \"\"\"\n" >> /etc/mock/"$CHROOT_NAME".cfg for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do branch="master" diff --git a/romio.spec b/romio.spec index fbf277e..9c6e65b 100644 --- a/romio.spec +++ b/romio.spec @@ -3,7 +3,7 @@ Name: romio Version: 3.4~a2 -Release: 1%{?dist} +Release: 2%{?dist} Summary: ROMIO License: MIT @@ -24,7 +24,7 @@ URL: http://www.mpich.org/ %endif %endif %endif -Source0: https://build.hpdd.intel.com/job/daos-stack/job/mpich/job/PR-44/lastSuccessfulBuild/artifact/artifacts/%{distro}/%{name}-%{version}.tar.gz +Source0: https://build.hpdd.intel.com/job/daos-stack/job/mpich/job/daos_adio-rpm/lastSuccessfulBuild/artifact/artifacts/%{distro}/%{name}-%{version}.tar.gz Patch0: packaged-runtests-%{distro}.patch BuildRequires: mpich-devel @@ -74,6 +74,9 @@ done %license %changelog +* Tue Jan 19 2021 Kenneth Cain - 3.4~a2-2 +- Undo temporary Source URL setting from version 3.4~a2-1 + * Tue Dec 08 2020 Kenneth Cain - 3.4~a2-1 - Update packaging to build with libdaos.so.1