From 5ad5ff2e310bc2e55bd7c47a610354c1db9e996f Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Wed, 1 Jun 2022 21:56:34 +0000 Subject: [PATCH 01/40] DAOS-10686 API: allow empty container lables Signed-off-by: Lei Huang --- ...b83d36a9243e30fe8a1bbc83e81fc46d2be2.patch | 44 --- Makefile | 2 +- mpifileutils.spec | 5 +- ...ed293c69f818aac5a47a0ce3665fc3fafba6.patch | 279 ++++++++++++++++++ 4 files changed, 284 insertions(+), 46 deletions(-) delete mode 100644 0.11.1..356cb83d36a9243e30fe8a1bbc83e81fc46d2be2.patch create mode 100644 v0.11.1..6230ed293c69f818aac5a47a0ce3665fc3fafba6.patch diff --git a/0.11.1..356cb83d36a9243e30fe8a1bbc83e81fc46d2be2.patch b/0.11.1..356cb83d36a9243e30fe8a1bbc83e81fc46d2be2.patch deleted file mode 100644 index a1152fc..0000000 --- a/0.11.1..356cb83d36a9243e30fe8a1bbc83e81fc46d2be2.patch +++ /dev/null @@ -1,44 +0,0 @@ -diff --git a/src/common/mfu_daos.c b/src/common/mfu_daos.c -index 8f53133..b870978 100644 ---- a/src/common/mfu_daos.c -+++ b/src/common/mfu_daos.c -@@ -1182,7 +1182,9 @@ int daos_connect( - * If nothing is passed in for destination a uuid is always generated - * unless user passed one in, because destination container labels are - * not generated */ -- rc = daos_cont_open(*poh, da->dst_cont_uuid, DAOS_COO_RW, coh, &co_info, NULL); -+ char cont_str[130]; -+ uuid_unparse(da->dst_cont_uuid, cont_str); -+ rc = daos_cont_open(*poh, cont_str, DAOS_COO_RW, coh, &co_info, NULL); - } else { - rc = daos_cont_open(*poh, *cont, DAOS_COO_RW, coh, &co_info, NULL); - } -@@ -1273,14 +1275,12 @@ int daos_connect( - } - - /* try to open it again */ -- if (dst_cont_passed) { -- if (is_uuid) { -- rc = daos_cont_open(*poh, da->dst_cont_uuid, DAOS_COO_RW, coh, &co_info, NULL); -- } else { -- rc = daos_cont_open(*poh, *cont, DAOS_COO_RW, coh, &co_info, NULL); -- } -+ if (dst_cont_passed && !is_uuid) { -+ rc = daos_cont_open(*poh, *cont, DAOS_COO_RW, coh, &co_info, NULL); - } else { -- rc = daos_cont_open(*poh, da->dst_cont_uuid, DAOS_COO_RW, coh, &co_info, NULL); -+ char cont_str[130]; -+ uuid_unparse(da->dst_cont_uuid, cont_str); -+ rc = daos_cont_open(*poh, cont_str, DAOS_COO_RW, coh, &co_info, NULL); - } - if (rc != 0) { - MFU_LOG(MFU_LOG_ERR, "Failed to open container: "DF_RC, DP_RC(rc)); -@@ -5617,7 +5617,7 @@ int daos_cont_deserialize_connect(daos_args_t *daos_args, - } else { - uuid_unparse(daos_args->dst_cont_uuid, cont_str); - MFU_LOG(MFU_LOG_INFO, "Successfully created container %s", cont_str); -- rc = daos_cont_open(daos_args->src_poh, daos_args->dst_cont_uuid, -+ rc = daos_cont_open(daos_args->src_poh, cont_str, - DAOS_COO_RW, &daos_args->src_coh, &co_info, NULL); - } - if (rc != 0) { diff --git a/Makefile b/Makefile index e191cc9..a6f53c7 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ NAME := mpifileutils SRC_EXT := gz REPO_NAME := mpifileutils-pkg -PKG_GIT_COMMIT := 356cb83d36a9243e30fe8a1bbc83e81fc46d2be2 +PKG_GIT_COMMIT := 6230ed293c69f818aac5a47a0ce3665fc3fafba6 GITHUB_PROJECT := hpc/$(NAME) TEST_PACKAGES := $(NAME)-mpich-devel $(NAME)-openmpi3-devel diff --git a/mpifileutils.spec b/mpifileutils.spec index 0f7e60f..b243ee5 100644 --- a/mpifileutils.spec +++ b/mpifileutils.spec @@ -50,7 +50,7 @@ Name: mpifileutils Version: 0.11.1 -Release: 2%{?commit:.g%{shortcommit}}%{?dist} +Release: 3%{?commit:.g%{shortcommit}}%{?dist} Summary: File utilities designed for scalability and performance. Group: System Environment/Libraries @@ -250,6 +250,9 @@ done %endif %changelog +*Wed Jun 01 2022 Lei Huang - 0.11.1-3 +- Update to patch 6230ed2 to support empty container lable + * Mon Mar 14 2022 Mohamad Chaarawi - 0.11.1-2 - Update to build with HDF5 1.13.1 diff --git a/v0.11.1..6230ed293c69f818aac5a47a0ce3665fc3fafba6.patch b/v0.11.1..6230ed293c69f818aac5a47a0ce3665fc3fafba6.patch new file mode 100644 index 0000000..306941a --- /dev/null +++ b/v0.11.1..6230ed293c69f818aac5a47a0ce3665fc3fafba6.patch @@ -0,0 +1,279 @@ +diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md +index d7c9a01..4ffcf16 100644 +--- a/CONTRIBUTING.md ++++ b/CONTRIBUTING.md +@@ -3,17 +3,12 @@ + *Thank you for taking the time to contribute!* + + ## Table of Contents +-[Resources](#resources) +- + [How To Contribute](#how-to-contribute) + * [Reporting an Issue & Feature Suggestions](#reporting-an-issue--feature-suggestions) + * [Pull Requests](#pull-requests) + * [License](#license) + * [Contributor's Declaration](#contributors-declaration) + +-## Resources +-[mpiFileUtils Google Group](https://groups.google.com/forum/#!forum/mpifileutils) +- + ## How To Contribute + + ### Reporting an Issue & Feature Suggestions +diff --git a/doc/rst/build.rst b/doc/rst/build.rst +index 047ba87..aedf67c 100644 +--- a/doc/rst/build.rst ++++ b/doc/rst/build.rst +@@ -34,7 +34,7 @@ To be certain of compatibility, it is recommended that one install libarchive-3. + mkdir install + installdir=`pwd`/install + +- wget https://github.com/libarchive/libarchive/releases/download/3.5.1/libarchive-3.5.1.tar.gz ++ wget https://github.com/libarchive/libarchive/releases/download/v3.5.1/libarchive-3.5.1.tar.gz + tar -zxf libarchive-3.5.1.tar.gz + cd libarchive-3.5.1 + ./configure --prefix=$installdir +@@ -135,7 +135,7 @@ which can be done with the following commands: + wget https://github.com/hpc/libcircle/releases/download/v0.3/libcircle-0.3.0.tar.gz + wget https://github.com/llnl/lwgrp/releases/download/v1.0.4/lwgrp-1.0.4.tar.gz + wget https://github.com/llnl/dtcmp/releases/download/v1.1.4/dtcmp-1.1.4.tar.gz +- wget https://github.com/libarchive/libarchive/releases/download/3.5.1/libarchive-3.5.1.tar.gz ++ wget https://github.com/libarchive/libarchive/releases/download/v3.5.1/libarchive-3.5.1.tar.gz + + tar -zxf libcircle-0.3.0.tar.gz + cd libcircle-0.3.0 +diff --git a/src/common/mfu_daos.c b/src/common/mfu_daos.c +index 8f53133..8981371 100644 +--- a/src/common/mfu_daos.c ++++ b/src/common/mfu_daos.c +@@ -1182,7 +1182,9 @@ int daos_connect( + * If nothing is passed in for destination a uuid is always generated + * unless user passed one in, because destination container labels are + * not generated */ +- rc = daos_cont_open(*poh, da->dst_cont_uuid, DAOS_COO_RW, coh, &co_info, NULL); ++ char cont_str[130]; ++ uuid_unparse(da->dst_cont_uuid, cont_str); ++ rc = daos_cont_open(*poh, cont_str, DAOS_COO_RW, coh, &co_info, NULL); + } else { + rc = daos_cont_open(*poh, *cont, DAOS_COO_RW, coh, &co_info, NULL); + } +@@ -1273,14 +1275,12 @@ int daos_connect( + } + + /* try to open it again */ +- if (dst_cont_passed) { +- if (is_uuid) { +- rc = daos_cont_open(*poh, da->dst_cont_uuid, DAOS_COO_RW, coh, &co_info, NULL); +- } else { +- rc = daos_cont_open(*poh, *cont, DAOS_COO_RW, coh, &co_info, NULL); +- } ++ if (dst_cont_passed && !is_uuid) { ++ rc = daos_cont_open(*poh, *cont, DAOS_COO_RW, coh, &co_info, NULL); + } else { +- rc = daos_cont_open(*poh, da->dst_cont_uuid, DAOS_COO_RW, coh, &co_info, NULL); ++ char cont_str[130]; ++ uuid_unparse(da->dst_cont_uuid, cont_str); ++ rc = daos_cont_open(*poh, cont_str, DAOS_COO_RW, coh, &co_info, NULL); + } + if (rc != 0) { + MFU_LOG(MFU_LOG_ERR, "Failed to open container: "DF_RC, DP_RC(rc)); +@@ -3857,7 +3857,7 @@ static int cont_serialize_prop_str(struct hdf5_args* hdf5, + hid_t attr_dspace; + hid_t usr_attr; + +- if (entry == NULL || entry->dpe_str == NULL) { ++ if (entry == NULL) { + MFU_LOG(MFU_LOG_ERR, "Property %s not found", prop_str); + rc = 1; + goto out; +@@ -3870,7 +3870,7 @@ static int cont_serialize_prop_str(struct hdf5_args* hdf5, + rc = 1; + goto out; + } +- status = H5Tset_size(attr_dtype, strlen(entry->dpe_str) + 1); ++ status = H5Tset_size(attr_dtype, (entry->dpe_str ? strlen(entry->dpe_str) : 0) + 1); + if (status < 0) { + MFU_LOG(MFU_LOG_ERR, "failed to set dtype size"); + rc = 1; +@@ -3895,7 +3895,7 @@ static int cont_serialize_prop_str(struct hdf5_args* hdf5, + rc = 1; + goto out; + } +- status = H5Awrite(usr_attr, attr_dtype, entry->dpe_str); ++ status = H5Awrite(usr_attr, attr_dtype, entry->dpe_str ? entry->dpe_str : ""); + if (status < 0) { + MFU_LOG(MFU_LOG_ERR, "failed to write attribute"); + rc = 1; +@@ -5394,22 +5394,23 @@ int cont_deserialize_all_props(struct hdf5_args *hdf5, + goto out; + } + +- rc = daos_cont_open(poh, label_entry->dpe_str, DAOS_COO_RW, &coh, &cont_info, NULL); +- if (rc == -DER_NONEXIST) { +- /* doesn't exist so ok to deserialize this container label */ +- deserialize_label = true; +- } else if (rc != 0) { +- MFU_LOG(MFU_LOG_ERR, "daos_cont_open failed: "DF_RC, DP_RC(rc)); +- goto out; +- } else { +- /* if this succeeds then label already exists, close container after +- * checking */ +- rc = daos_cont_close(coh, NULL); +- if (rc != 0) { ++ if (label_entry->dpe_str[0]) { ++ rc = daos_cont_open(poh, label_entry->dpe_str, DAOS_COO_RW, &coh, &cont_info, NULL); ++ if (rc == -DER_NONEXIST) { ++ /* doesn't exist so ok to deserialize this container label */ ++ deserialize_label = true; ++ } else if (rc != 0) { ++ MFU_LOG(MFU_LOG_ERR, "daos_cont_open failed: "DF_RC, DP_RC(rc)); + goto out; ++ } else { ++ /* if this succeeds then label already exists, close container after ++ * checking */ ++ rc = daos_cont_close(coh, NULL); ++ if (rc != 0) { ++ goto out; ++ } + } + } +- + if (deserialize_label) { + num_props++; + } +@@ -5617,7 +5618,7 @@ int daos_cont_deserialize_connect(daos_args_t *daos_args, + } else { + uuid_unparse(daos_args->dst_cont_uuid, cont_str); + MFU_LOG(MFU_LOG_INFO, "Successfully created container %s", cont_str); +- rc = daos_cont_open(daos_args->src_poh, daos_args->dst_cont_uuid, ++ rc = daos_cont_open(daos_args->src_poh, cont_str, + DAOS_COO_RW, &daos_args->src_coh, &co_info, NULL); + } + if (rc != 0) { +diff --git a/src/common/mfu_flist_copy.c b/src/common/mfu_flist_copy.c +index f5acaab..0e9c78f 100644 +--- a/src/common/mfu_flist_copy.c ++++ b/src/common/mfu_flist_copy.c +@@ -1680,12 +1680,20 @@ static int mfu_copy_file_normal( + /* If we're using O_DIRECT, deal with short reads. + * Retry with same buffer and offset since those must + * be aligned at block boundaries. */ ++ int retries = 0; + while (copy_opts->direct && /* using O_DIRECT */ + bytes_read > 0 && /* read was not an error or eof */ + bytes_read < left_to_read && /* shorter than requested */ + (off + bytes_read) < file_size) /* not at end of file */ + { +- /* TODO: probably should retry a limited number of times then abort */ ++ /* try the read a limited number of times then given up with error */ ++ retries++; ++ if (retries == 5) { ++ MFU_LOG(MFU_LOG_ERR, "Source file `%s' exceeded short read limit, maybe shorter than expected size of %llu bytes", ++ src, file_size); ++ return -1; ++ } ++ + bytes_read = mfu_file_pread(src, buf, left_to_read, off, mfu_src_file); + } + +@@ -1698,8 +1706,8 @@ static int mfu_copy_file_normal( + + /* check for early EOF */ + if (bytes_read == 0) { +- MFU_LOG(MFU_LOG_ERR, "Source file `%s' shorter than expected %llu (errno=%d %s)", +- src, file_size, errno, strerror(errno)); ++ MFU_LOG(MFU_LOG_ERR, "Source file `%s' shorter than expected size of %llu bytes", ++ src, file_size); + return -1; + } + +diff --git a/src/daos-gen/daos-gen.c b/src/daos-gen/daos-gen.c +index d96373a..46ee44d 100644 +--- a/src/daos-gen/daos-gen.c ++++ b/src/daos-gen/daos-gen.c +@@ -18,6 +18,14 @@ + #define BUFLEN 80 + #define UUID_STR_LEN 129 + ++#if defined(DAOS_API_VERSION_MAJOR) && defined(DAOS_API_VERSION_MINOR) ++#define CHECK_DAOS_API_VERSION(major, minor) \ ++ ((DAOS_API_VERSION_MAJOR > (major)) \ ++ || (DAOS_API_VERSION_MAJOR == (major) && DAOS_API_VERSION_MINOR >= (minor))) ++#else ++#define CHECK_DAOS_API_VERSION(major, minor) 0 ++#endif ++ + static uint32_t obj_id_gen = 1; + + /** Print a usage message. */ +@@ -176,13 +184,18 @@ int main(int argc, char** argv) + char key[keys_per_object]; + int i,j; + char uuid_str[UUID_STR_LEN]; ++ uuid_t cont_uuid; ++ ++#if CHECK_DAOS_API_VERSION(2, 0) ++ enum daos_otype_t otype; ++ otype = DAOS_OT_KV_HASHED; ++#else + daos_ofeat_t ofeats; ++ ofeats = DAOS_OF_KV_FLAT; ++#endif + +- ofeats = DAOS_OF_DKEY_UINT64 | DAOS_OF_KV_FLAT | DAOS_OF_KV_FLAT; + /* connect to pool/cont then broadcast to rest of ranks */ + if (rank == 0) { +- /* generate container UUID */ +- uuid_generate(daos_args->src_cont); + daos_pool_info_t pool_info = {0}; + daos_cont_info_t co_info = {0}; + #if DAOS_API_VERSION_MAJOR < 1 +@@ -199,14 +212,27 @@ int main(int argc, char** argv) + } + + /* create cont and open */ ++#if CHECK_DAOS_API_VERSION(2, 0) ++ rc = daos_cont_create(daos_args->src_poh, &cont_uuid, NULL, NULL); ++#else ++ /* generate container UUID */ ++ uuid_generate(daos_args->src_cont); ++ uuid_copy(cont_uuid, daos_args->src_cont); + rc = daos_cont_create(daos_args->src_poh, daos_args->src_cont, NULL, NULL); ++#endif + if (rc != 0) { + MFU_LOG(MFU_LOG_ERR, "Failed to create cont: "DF_RC, DP_RC(rc)); + rc = 1; + goto err_cont; + } ++ uuid_unparse(cont_uuid, uuid_str); ++#if CHECK_DAOS_API_VERSION(2, 0) ++ rc = daos_cont_open(daos_args->src_poh, uuid_str, ++ DAOS_COO_RW, &daos_args->src_coh, &co_info, NULL); ++#else + rc = daos_cont_open(daos_args->src_poh, daos_args->src_cont, + DAOS_COO_RW, &daos_args->src_coh, &co_info, NULL); ++#endif + if (rc != 0) { + MFU_LOG(MFU_LOG_ERR, "Failed to open container: "DF_RC, DP_RC(rc)); + rc = 1; +@@ -222,8 +248,11 @@ int main(int argc, char** argv) + memset(buf, 'A', BUFLEN); + for (i = 0; i < num_objects; i++) { + oid[i] = dts_oid_gen(0); +- ++#if CHECK_DAOS_API_VERSION(2, 0) ++ rc = daos_obj_generate_oid(daos_args->src_coh, &oid[i], otype, OC_RP_XSF, 0, 0); ++#else + rc = daos_obj_generate_oid(daos_args->src_coh, &oid[i], ofeats, OC_RP_XSF, 0, 0); ++#endif + if (rc != 0) { + MFU_LOG(MFU_LOG_ERR, "Failed to generate oid: "DF_RC, DP_RC(rc)); + rc = 1; +@@ -261,7 +290,6 @@ int main(int argc, char** argv) + * generates same amount */ + if (rank == 0) { + int total_num_objects = size * num_objects; +- uuid_unparse(daos_args->src_cont, uuid_str); + printf("Container UUID: %s\n\ttotal objects:%d\n" + "\tkeys per object:%d\n", uuid_str, total_num_objects, keys_per_object); + } From 1fb9757a23563007d2800b8e049880badd655fff Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Wed, 1 Jun 2022 22:03:39 +0000 Subject: [PATCH 02/40] DAOS-10686 API: replace centos8 with el8 in Jenkinsfile Signed-off-by: Lei Huang --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8ecb222..afd864e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,5 +40,5 @@ // I.e. for testing library changes //@Library(value="pipeline-lib@your_branch") _ -packageBuildingPipeline(['distros' : ['centos7', 'centos8', 'leap15']]) +packageBuildingPipeline(['distros' : ['centos7', 'el8', 'leap15']]) From a89f4ea51ec63548634b481f45c2453ca0847ca1 Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Thu, 2 Jun 2022 03:06:13 +0000 Subject: [PATCH 03/40] DAOS-10686 API: fixed typo and patch name Signed-off-by: Lei Huang --- ...ch => 0.11.1..6230ed293c69f818aac5a47a0ce3665fc3fafba6.patch | 0 mpifileutils.spec | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename v0.11.1..6230ed293c69f818aac5a47a0ce3665fc3fafba6.patch => 0.11.1..6230ed293c69f818aac5a47a0ce3665fc3fafba6.patch (100%) diff --git a/v0.11.1..6230ed293c69f818aac5a47a0ce3665fc3fafba6.patch b/0.11.1..6230ed293c69f818aac5a47a0ce3665fc3fafba6.patch similarity index 100% rename from v0.11.1..6230ed293c69f818aac5a47a0ce3665fc3fafba6.patch rename to 0.11.1..6230ed293c69f818aac5a47a0ce3665fc3fafba6.patch diff --git a/mpifileutils.spec b/mpifileutils.spec index b243ee5..96840b2 100644 --- a/mpifileutils.spec +++ b/mpifileutils.spec @@ -251,7 +251,7 @@ done %changelog *Wed Jun 01 2022 Lei Huang - 0.11.1-3 -- Update to patch 6230ed2 to support empty container lable +- Update to patch 6230ed2 to support empty container label * Mon Mar 14 2022 Mohamad Chaarawi - 0.11.1-2 - Update to build with HDF5 1.13.1 From 266fa99e4bdbd3f598b1826d7de71b5274e7ba44 Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Thu, 2 Jun 2022 14:39:23 +0000 Subject: [PATCH 04/40] DAOS-10686 API: update the packaging submodule Signed-off-by: Lei Huang --- packaging/Dockerfile.centos.7 | 4 +- packaging/Dockerfile.mockbuild | 40 ++++--- packaging/Makefile_distro_vars.mk | 33 ++++-- packaging/Makefile_packaging.mk | 176 ++++++++++++++++-------------- packaging/rpm_chrootbuild | 66 ++++++++--- 5 files changed, 198 insertions(+), 121 deletions(-) diff --git a/packaging/Dockerfile.centos.7 b/packaging/Dockerfile.centos.7 index 55510f7..cdfb7f6 100644 --- a/packaging/Dockerfile.centos.7 +++ b/packaging/Dockerfile.centos.7 @@ -1,12 +1,12 @@ # -# Copyright 2018-2021, Intel Corporation +# Copyright 2018-2022, Intel Corporation # # 'recipe' for Docker to build an RPM # # Pull base image FROM centos:7 -MAINTAINER daos-stack +LABEL maintainer="daos@daos.groups.io" # use same UID as host and default value of 1000 if not specified ARG UID=1000 diff --git a/packaging/Dockerfile.mockbuild b/packaging/Dockerfile.mockbuild index 89d494f..25ff965 100644 --- a/packaging/Dockerfile.mockbuild +++ b/packaging/Dockerfile.mockbuild @@ -1,20 +1,30 @@ # -# Copyright 2018-2022, Intel Corporation +# Copyright 2018-2022 Intel Corporation # # 'recipe' for Docker to build an RPM # # Pull base image -FROM fedora:35 -LABEL maintainer="daos@daos.groups.io>" +FROM fedora:latest +LABEL maintainer="daos@daos.groups.io" -# use same UID as host and default value of 1000 if not specified -ARG UID=1000 +# Use local repo server if present +ARG REPO_FILE_URL +RUN if [ -n "$REPO_FILE_URL" ]; then \ + cd /etc/yum.repos.d/ && \ + curl -f -o daos_ci-fedora-artifactory.repo.tmp \ + "$REPO_FILE_URL"daos_ci-fedora-artifactory.repo && \ + rm -f *.repo && \ + mv daos_ci-fedora-artifactory.repo{.tmp,}; \ + fi # Install basic tools -RUN dnf -y install mock make \ - rpm-build curl createrepo rpmlint redhat-lsb-core git \ - python-srpm-macros rpmdevtools mock-core-configs\ \<\ 37.1 +RUN dnf -y install mock make \ + rpm-build createrepo rpmlint redhat-lsb-core git \ + python-srpm-macros rpmdevtools + +# use same UID as host and default value of 1000 if not specified +ARG UID=1000 # Add build user (to keep rpmbuild happy) ENV USER build @@ -24,10 +34,14 @@ RUN echo "$USER:$PASSWD" | chpasswd # add the user to the mock group so it can run mock RUN usermod -a -G mock $USER -# mock in Docker needs to use the old-chroot option -RUN grep use_nspawn /etc/mock/site-defaults.cfg || \ - echo "config_opts['use_nspawn'] = False" >> /etc/mock/site-defaults.cfg - ARG CACHEBUST -RUN dnf -y upgrade --exclude mock-core-configs && \ +RUN dnf -y upgrade && \ dnf clean all + +# Monkey-patch rpmlint until a new release is made with +# https://github.com/rpm-software-management/rpmlint/pull/795 in it +COPY packaging/rpmlint--ignore-unused-rpmlintrc.patch . +RUN (cd $(python3 -c 'import site; print(site.getsitepackages()[-1])') && \ + patch -p1 && \ + rm -f rpmlint/__pycache__/{cli,lint}.*.pyc) < rpmlint--ignore-unused-rpmlintrc.patch && \ + rm -f rpmlint--ignore-unused-rpmlintrc.patch diff --git a/packaging/Makefile_distro_vars.mk b/packaging/Makefile_distro_vars.mk index 44006e0..3d29ffa 100644 --- a/packaging/Makefile_distro_vars.mk +++ b/packaging/Makefile_distro_vars.mk @@ -26,7 +26,7 @@ SPECTOOL := spectool # DISTRO_ID (i.e. el7) # DISTRO_BASE (i.e. EL_7) # from the CHROOT_NAME -ifeq ($(CHROOT_NAME),epel-7-x86_64) +ifeq ($(patsubst %epel-7-x86_64,,$(lastword $(subst +, ,$(CHROOT_NAME)))),) DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) VERSION_ID := 7 DISTRO_ID := el7 @@ -35,14 +35,17 @@ DISTRO_VERSION ?= $(VERSION_ID) ORIG_TARGET_VER := 7 SED_EXPR := 1s/$(DIST)//p endif -ifeq ($(CHROOT_NAME),epel-8-x86_64) -DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) -VERSION_ID := 8 -DISTRO_ID := el8 -DISTRO_BASE := EL_8 -DISTRO_VERSION ?= $(VERSION_ID) -ORIG_TARGET_VER := 8 -SED_EXPR := 1s/$(DIST)//p +ifeq ($(patsubst %epel-8-x86_64,,$(lastword $(subst +, ,$(CHROOT_NAME)))),) +DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) +VERSION_ID := 8.5 +DISTRO_ID := el8 +DISTRO_BASE := EL_8 +ifneq ($(DISTRO_VERSION_EL8),) +override DISTRO_VERSION := $(DISTRO_VERSION_EL8) +endif +DISTRO_VERSION ?= $(VERSION_ID) +ORIG_TARGET_VER := 8.5 +SED_EXPR := 1s/$(DIST)//p endif ifeq ($(CHROOT_NAME),opensuse-leap-15.2-x86_64) VERSION_ID := 15.2 @@ -62,6 +65,18 @@ SED_EXPR := 1p endif endif ifeq ($(ID),centos) +ID = el +endif +ifeq ($(ID),rocky) +ID = el +endif +ifeq ($(ID),almalinux) +ID = el +endif +ifeq ($(ID),rhel) +ID = el +endif +ifeq ($(ID),el) DISTRO_ID := el$(VERSION_ID) DISTRO_BASE := $(basename EL_$(VERSION_ID)) DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) diff --git a/packaging/Makefile_packaging.mk b/packaging/Makefile_packaging.mk index b59782e..8bd618e 100644 --- a/packaging/Makefile_packaging.mk +++ b/packaging/Makefile_packaging.mk @@ -10,7 +10,7 @@ SHELL=/bin/bash -include Makefile.local # default to Leap 15 distro for chrootbuild -CHROOT_NAME ?= opensuse-leap-15.2-x86_64 +CHROOT_NAME ?= opensuse-leap-15.3-x86_64 include packaging/Makefile_distro_vars.mk ifeq ($(DEB_NAME),) @@ -19,7 +19,9 @@ endif CALLING_MAKEFILE := $(word 1, $(MAKEFILE_LIST)) -TOPDIR ?= $(CURDIR) +# this Makefile should always be executed from it's own dir +TOPDIR ?= $(abspath $(dir $(firstword $(MAKEFILE_LIST)))) + BUILD_PREFIX ?= . DOT := . @@ -28,11 +30,6 @@ RPM_BUILD_OPTIONS += $(EXTERNAL_RPM_BUILD_OPTIONS) # some defaults the caller can override PACKAGING_CHECK_DIR ?= ../packaging LOCAL_REPOS ?= true -ifeq ($(ID_LIKE),debian) -DAOS_REPO_TYPE ?= LOCAL -else -DAOS_REPO_TYPE ?= STABLE -endif TEST_PACKAGES ?= ${NAME} # unfortunately we cannot always name the repo the same as the project @@ -43,6 +40,7 @@ LEAP_15_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-rep EL_7_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el7: *\(.*\)/\1/p') EL_8_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el8: *\(.*\)/\1/p') UBUNTU_20_04_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-ubuntu20: *\(.*\)/\1/p') +REPO_FILES_PR ?= $(shell git show -s --format=%B | sed -ne 's/^Repo-files-PR: *\(.*\)/\1/p') ifneq ($(PKG_GIT_COMMIT),) ifeq ($(GITHUB_PROJECT),) @@ -86,7 +84,7 @@ define distro_map case $(DISTRO_ID) in \ el7) distro="centos7" \ ;; \ - el8) distro="centos8" \ + el8) distro="el8" \ ;; \ sle12.3) distro="sles12.3" \ ;; \ @@ -100,27 +98,29 @@ define distro_map endef define install_repos - IFS='|' read -ra BASES <<< "$($(DISTRO_BASE)_LOCAL_REPOS)"; \ - for baseurl in "$${BASES[@]}"; do \ - baseurl="$${baseurl# *}"; \ - $(call install_repo,$$baseurl); \ - done - for repo in $($(DISTRO_BASE)_PR_REPOS) \ - $(PR_REPOS) $(1); do \ - branch="master"; \ - build_number="lastSuccessfulBuild"; \ - if [[ $$repo = *@* ]]; then \ - branch="$${repo#*@}"; \ - repo="$${repo%@*}"; \ - if [[ $$branch = *:* ]]; then \ - build_number="$${branch#*:}"; \ - branch="$${branch%:*}"; \ - fi; \ - fi; \ - $(call distro_map) \ + if [ "$(ID_LIKE)" = "debian" ]; then \ + IFS='|' read -ra BASES <<< "$($(DISTRO_BASE)_LOCAL_REPOS)"; \ + for baseurl in "$${BASES[@]}"; do \ + baseurl="$${baseurl# *}"; \ + $(call install_repo,$$baseurl) \ + done; \ + fi + for repo in $($(DISTRO_BASE)_PR_REPOS) \ + $(PR_REPOS) $(1); do \ + branch="master"; \ + build_number="lastSuccessfulBuild"; \ + if [[ $$repo = *@* ]]; then \ + branch="$${repo#*@}"; \ + repo="$${repo%@*}"; \ + if [[ $$branch = *:* ]]; then \ + build_number="$${branch#*:}"; \ + branch="$${branch%:*}"; \ + fi; \ + fi; \ + $(call distro_map) \ baseurl=$${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$$repo/job/$$branch/; \ - baseurl+=$$build_number/artifact/artifacts/$$distro/; \ - $(call install_repo,$$baseurl); \ + baseurl+=$$build_number/artifact/artifacts/$$distro/; \ + $(call install_repo,$$baseurl) \ done endef @@ -154,6 +154,11 @@ ifeq ($(DL_NAME),) DL_NAME = $(NAME) endif +# this actually should replace all of the downloaders below +$(notdir $(SOURCE)): $(SPEC) $(CALLING_MAKEFILE) + # TODO: need to clean up old ones + $(SPECTOOL) -g $(SPEC) + $(DL_NAME)$(DL_VERSION).linux-amd64.tar.$(SRC_EXT): $(SPEC) $(CALLING_MAKEFILE) rm -f ./$(DL_NAME)*.tar{gz,bz*,xz} $(SPECTOOL) -g $(SPEC) @@ -334,60 +339,29 @@ patch: endif # *_LOCAL_* repos are locally built packages. -# *_GROUP_* repos are a local mirror of a group of upstream repos. -# *_GROUP_* repos may not supply a repomd.xml.key. ifeq ($(LOCAL_REPOS),true) - ifneq ($(REPOSITORY_URL),) - # group repos are not working in Nexus so we hack in the group members directly below - #ifneq ($(DAOS_STACK_$(DISTRO_BASE)_DOCKER_$(DAOS_REPO_TYPE)_REPO),) - #DISTRO_REPOS = $(DAOS_STACK_$(DISTRO_BASE)_DOCKER_$(DAOS_REPO_TYPE)_REPO) - #$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_DOCKER_$(DAOS_REPO_TYPE)_REPO)/ - #endif - ifneq ($(DAOS_STACK_$(DISTRO_BASE)_$(DAOS_REPO_TYPE)_REPO),) - ifeq ($(ID_LIKE),debian) - # $(DISTRO_BASE)_LOCAL_REPOS is a list separated by | because you cannot pass lists - # of values with spaces as environment variables - $(DISTRO_BASE)_LOCAL_REPOS := [trusted=yes] - else - $(DISTRO_BASE)_LOCAL_REPOS := $(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO) - DISTRO_REPOS = disabled # any non-empty value here works and is not used beyond testing if the value is empty or not - endif # ifeq ($(ID_LIKE),debian) - ifeq ($(DISTRO_BASE), EL_8) - # hack to use 8.3 non-group repos on EL_8 - $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(subst $(ORIG_TARGET_VER),$(DISTRO_VERSION),$(REPOSITORY_URL)repository/centos-8.3-base-x86_64-proxy|$(REPOSITORY_URL)repository/centos-8.3-extras-x86_64-proxy|$(REPOSITORY_URL)repository/epel-el-8-x86_64-proxy) - else ifeq ($(DISTRO_BASE), EL_7) - # hack to use 7.9 non-group repos on EL_7 - $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(subst $(ORIG_TARGET_VER),$(DISTRO_VERSION),$(REPOSITORY_URL)repository/centos-7.9-base-x86_64-proxy|$(REPOSITORY_URL)repository/centos-7.9-extras-x86_64-proxy|$(REPOSITORY_URL)repository/centos-7.9-updates-x86_64-proxy|$(REPOSITORY_URL)repository/epel-el-7-x86_64-proxy) - else ifeq ($(DISTRO_BASE), LEAP_15) - # hack to use 15 non-group repos on LEAP_15 - $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(subst $(ORIG_TARGET_VER),$(DISTRO_VERSION),$(REPOSITORY_URL)repository/opensuse-15.2-oss-x86_64-proxy|$(REPOSITORY_URL)repository/opensuse-15.2-update-oss-x86_64-provo-mirror-proxy|$(REPOSITORY_URL)repository/opensuse-15.2-update-non-oss-x86_64-proxy|$(REPOSITORY_URL)repository/opensuse-15.2-non-oss-x86_64-proxy|$(REPOSITORY_URL)repository/opensuse-15.2-repo-sle-update-proxy|$(REPOSITORY_URL)repository/opensuse-15.2-repo-backports-update-proxy) - else - # debian - $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS) $(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_$(DAOS_REPO_TYPE)_REPO) - endif # ifeq ($(DISTRO_BASE), *) - endif #ifneq ($(DAOS_STACK_$(DISTRO_BASE)_$(DAOS_REPO_TYPE)_REPO),) - ifneq ($(DAOS_STACK_$(DISTRO_BASE)_APPSTREAM_REPO),) - $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_APPSTREAM_REPO) - endif - # group repos are not working in Nexus so we hack in the group members directly above - ifneq ($(DAOS_STACK_$(DISTRO_BASE)_POWERTOOLS_REPO),) - $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_POWERTOOLS_REPO) - endif - ifneq ($(ID_LIKE),debian) - ifneq ($(DAOS_STACK_INTEL_ONEAPI_REPO),) - $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(REPOSITORY_URL)$(DAOS_STACK_INTEL_ONEAPI_REPO) - endif # ifneq ($(DAOS_STACK_INTEL_ONEAPI_REPO),) - endif # ifneq ($(ID_LIKE),debian) - endif # ifneq ($(REPOSITORY_URL),) + ifneq ($(ARTIFACTORY_URL),) + ifneq ($(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO),) + DISTRO_REPOS = disabled # any non-empty value here works and is not used beyond testing if the value is empty or not + # convert to artifactory url + DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO := $(subst reposi,artifac,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)) + # $(DISTRO_BASE)_LOCAL_REPOS is a list separated by | because you cannot pass lists + # of values with spaces as environment variables + $(DISTRO_BASE)_LOCAL_REPOS := [trusted=yes] $(ARTIFACTORY_URL)$(subst stack,stack-daos,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)) + $(DISTRO_BASE)_LOCAL_REPOS += |[trusted=yes] $(ARTIFACTORY_URL)$(subst stack,stack-deps,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)) + endif #ifneq ($(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO),) + endif # ifneq ($(ARTIFACTORY_URL),) endif # ifeq ($(LOCAL_REPOS),true) ifeq ($(ID_LIKE),debian) chrootbuild: $(DEB_TOP)/$(DEB_DSC) $(call distro_map) \ DISTRO="$$distro" \ PR_REPOS="$(PR_REPOS)" \ + REPO_FILES_PR="$(REPO_FILES_PR)" \ DISTRO_BASE_PR_REPOS="$($(DISTRO_BASE)_PR_REPOS)" \ JENKINS_URL="$${JENKINS_URL}" \ JOB_REPOS="$(JOB_REPOS)" \ + REPO_FILE_URL="$(REPO_FILE_URL)" \ DISTRO_BASE_LOCAL_REPOS="$($(DISTRO_BASE)_LOCAL_REPOS)" \ VERSION_CODENAME="$(VERSION_CODENAME)" \ DEB_TOP="$(DEB_TOP)" \ @@ -400,26 +374,61 @@ chrootbuild: $(SRPM) $(CALLING_MAKEFILE) DISTRO="$$distro" \ CHROOT_NAME="$(CHROOT_NAME)" \ PR_REPOS="$(PR_REPOS)" \ + REPO_FILES_PR="$(REPO_FILES_PR)" \ DISTRO_BASE_PR_REPOS="$($(DISTRO_BASE)_PR_REPOS)" \ JENKINS_URL="$${JENKINS_URL}" \ JOB_REPOS="$(JOB_REPOS)" \ - DISTRO_BASE_LOCAL_REPOS="$($(DISTRO_BASE)_LOCAL_REPOS)" \ + REPO_FILE_URL="$(REPO_FILE_URL)" \ MOCK_OPTIONS="$(MOCK_OPTIONS)" \ RPM_BUILD_OPTIONS='$(RPM_BUILD_OPTIONS)' \ DISTRO_REPOS='$(DISTRO_REPOS)' \ + ARTIFACTORY_URL="$(ARTIFACTORY_URL)" \ + REPOSITORY_URL="$(REPOSITORY_URL)" \ + DISTRO_VERSION="$(DISTRO_VERSION)" \ TARGET="$<" \ packaging/rpm_chrootbuild endif +podman_chrootbuild: + if ! podman build --build-arg REPO_FILE_URL=$(REPO_FILE_URL) \ + -t $(subst +,-,$(CHROOT_NAME))-chrootbuild \ + -f packaging/Dockerfile.mockbuild .; then \ + echo "Container build failed"; \ + exit 1; \ + fi + rm -f /var/lib/mock/$(CHROOT_NAME)/result/{root,build}.log + podman run --rm --privileged -w $(TOPDIR) -v=$(TOPDIR)/..:$(TOPDIR)/.. \ + -it $(subst +,-,$(CHROOT_NAME))-chrootbuild \ + bash -c 'if ! DISTRO_REPOS=false \ + REPO_FILE_URL=$(REPO_FILE_URL) \ + REPOSITORY_URL=$(REPOSITORY_URL) \ + make REPO_FILES_PR=$(REPO_FILES_PR) \ + MOCK_OPTIONS=$(MOCK_OPTIONS) \ + CHROOT_NAME=$(CHROOT_NAME) -C $(CURDIR) chrootbuild; then \ + cat /var/lib/mock/$(CHROOT_NAME)/{result/{root,build},root/builddir/build/BUILD/*/config}.log; \ + exit 1; \ + fi; \ + rpmlint $$(ls /var/lib/mock/$(CHROOT_NAME)/result/*.rpm | \ + grep -v -e debuginfo -e debugsource -e src.rpm)' + docker_chrootbuild: - $(DOCKER) build --build-arg UID=$$(id -u) -t chrootbuild \ - -f packaging/Dockerfile.mockbuild . - $(DOCKER) run --privileged=true -w $(TOPDIR) -v=$(TOPDIR):$(TOPDIR) \ - -it chrootbuild bash -c "make -C $(CURDIR) \ - CHROOT_NAME=$(CHROOT_NAME) chrootbuild" + if ! $(DOCKER) build --build-arg UID=$$(id -u) -t chrootbuild \ + --build-arg REPO_FILE_URL=$(REPO_FILE_URL) \ + -f packaging/Dockerfile.mockbuild .; then \ + echo "Container build failed"; \ + exit 1; \ + fi + rm -f /var/lib/mock/$(CHROOT_NAME)/result/{root,build}.log + if ! $(DOCKER) run --user=$$(id -u) --privileged=true -w $(TOPDIR) \ + -v=$(TOPDIR):$(TOPDIR) \ + -it chrootbuild bash -c "make -C $(CURDIR) \ + CHROOT_NAME=$(CHROOT_NAME) chrootbuild"; then \ + cat /var/lib/mock/$(CHROOT_NAME)/result/{root,build}.log; \ + exit 1; \ + fi rpmlint: $(SPEC) - rpmlint $< + rpmlint --ignore-unused-rpmlintrc $< packaging_check: if grep -e --repo $(CALLING_MAKEFILE); then \ @@ -456,6 +465,15 @@ test: $(call install_repos,$(REPO_NAME)@$(BRANCH_NAME):$(BUILD_NUMBER)) dnf -y install $(TEST_PACKAGES) +show_NAME: + @echo '$(NAME)' + +show_DISTRO_ID: + @echo '$(DISTRO_ID)' + +show_distro_map: + @$(call distro_map) echo "$$distro" + show_spec: @echo '$(SPEC)' diff --git a/packaging/rpm_chrootbuild b/packaging/rpm_chrootbuild index 8c1008e..03ab4c5 100755 --- a/packaging/rpm_chrootbuild +++ b/packaging/rpm_chrootbuild @@ -2,17 +2,7 @@ set -uex -# shellcheck disable=SC2153 -IFS=\| read -r -a distro_base_local_repos <<< "$DISTRO_BASE_LOCAL_REPOS" -repo_adds=() -repo_dels=() - -if [ -n "${REPOSITORY_URL:-}" ] && [ -n "$DISTRO_REPOS" ]; then - repo_dels+=("--disablerepo=\*") -fi - -: "${WORKSPACE:=$PWD}" -mock_config_dir="$WORKSPACE/mock" +mock_config_dir="${WORKSPACE:-$PWD}/mock" original_cfg_file="/etc/mock/${CHROOT_NAME}.cfg" cfg_file="$mock_config_dir/${CHROOT_NAME}.cfg" mkdir -p "$mock_config_dir" @@ -21,12 +11,49 @@ ln -sf /etc/mock/logging.ini "$mock_config_dir/" cp "$original_cfg_file" "$cfg_file" -if [ "${CHROOT_NAME}" == "epel-8-x86_64" ]; then - echo -e "config_opts['module_enable'] = ['javapackages-tools:201801']" \ - >> "$cfg_file" +if [[ $CHROOT_NAME == *epel-8-x86_64 ]]; then + cat <> "$cfg_file" +config_opts['module_setup_commands'] = [ + ('enable', 'javapackages-tools:201801'), + ('disable', 'go-toolset') +] +EOF + fi +# shellcheck disable=SC2153 +repo_adds=() +repo_dels=() + echo -e "config_opts['yum.conf'] += \"\"\"\n" >> "$cfg_file" + +if { [ -n "${REPOSITORY_URL:-}" ] || [ -n "${ARTIFACTORY_URL:-}" ]; } \ + && [ -n "$DISTRO_REPOS" ]; then + repo_dels+=("--disablerepo=\*") + + if [ -n "${REPO_FILE_URL:-}" ]; then + if [ -n "$REPO_FILES_PR" ]; then + if [[ $REPO_FILES_PR = PR-* ]]; then + build_number="lastSuccessfulBuild" + branch="$REPO_FILES_PR" + if [[ $branch = *:* ]]; then + build_number="${branch#*:}" + branch="${branch%:*}" + fi + REPO_FILE_URL="${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-do/job/repo-files/job/$branch/$build_number/artifact/" + else + if [ ! -d "$REPO_FILES_PR" ]; then + echo "Could not find your specified directory \"$REPO_FILES_PR\" to read repo-files from" + exit 1 + fi + REPO_FILE_URL="file://$(readlink -e "$REPO_FILES_PR")/" + fi + fi + curl -sSf "$REPO_FILE_URL"daos_ci-"$DISTRO"-mock-artifactory.repo >> "$cfg_file" + repo_adds+=("--enablerepo *-artifactory") + fi +fi + for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do branch="master" build_number="lastSuccessfulBuild" @@ -45,7 +72,7 @@ baseurl=${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/jo enabled=1\n\ gpgcheck=False\n" >> "$cfg_file" done -for repo in $JOB_REPOS "${distro_base_local_repos[@]}"; do +for repo in $JOB_REPOS; do repo_name=${repo##*://} repo_name=${repo_name//\//_} repo_adds+=("--enablerepo $repo_name") @@ -56,7 +83,10 @@ enabled=1\n" >> "$cfg_file" done echo "\"\"\"" >> "$cfg_file" +if [ -n "$DISTRO_VERSION" ]; then + releasever_opt=("--config-opts=releasever=$DISTRO_VERSION") +fi # shellcheck disable=SC2086 -eval mock --configdir "$mock_config_dir" -r "${CHROOT_NAME}" \ - ${repo_dels[*]} ${repo_adds[*]} \ - $MOCK_OPTIONS $RPM_BUILD_OPTIONS "$TARGET" +eval mock --configdir "$mock_config_dir" -r "${CHROOT_NAME}" \ + ${repo_dels[*]} ${repo_adds[*]} --disablerepo=\*-debug* \ + "${releasever_opt[@]}" $MOCK_OPTIONS $RPM_BUILD_OPTIONS "$TARGET" From 54fe555e541234a06beb361fea0435611c352501 Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Thu, 2 Jun 2022 15:09:27 +0000 Subject: [PATCH 05/40] DAOS-10686 API: add packaging/rpmlint--ignore-unused-rpmlintrc.patch Signed-off-by: Lei Huang --- .../rpmlint--ignore-unused-rpmlintrc.patch | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 packaging/rpmlint--ignore-unused-rpmlintrc.patch diff --git a/packaging/rpmlint--ignore-unused-rpmlintrc.patch b/packaging/rpmlint--ignore-unused-rpmlintrc.patch new file mode 100644 index 0000000..f24075f --- /dev/null +++ b/packaging/rpmlint--ignore-unused-rpmlintrc.patch @@ -0,0 +1,41 @@ +From 060fbdd03d03b0b9835b861a72f9aeb25347277d Mon Sep 17 00:00:00 2001 +From: Martin Liska +Date: Tue, 25 Jan 2022 14:29:05 +0100 +Subject: [PATCH] Add new option --ignore-unused-rpmlintrc. + +The option skips reporting of unused-rpmlintrc-filter errors. + +Fixes: #794. +--- + rpmlint/cli.py | 2 ++ + rpmlint/lint.py | 3 ++- + test/test_lint.py | 3 ++- + 3 files changed, 6 insertions(+), 2 deletions(-) + +diff --git a/rpmlint/cli.py b/rpmlint/cli.py +index 81d5b92b..8a7bc09e 100644 +--- a/rpmlint/cli.py ++++ b/rpmlint/cli.py +@@ -82,6 +82,8 @@ def process_lint_args(argv): + parser.add_argument('-i', '--installed', nargs='+', default='', help='installed packages to be validated by rpmlint') + parser.add_argument('-t', '--time-report', action='store_true', help='print time report for run checks') + parser.add_argument('-T', '--profile', action='store_true', help='print cProfile report') ++ parser.add_argument('--ignore-unused-rpmlintrc', action='store_true', ++ help='Do not report "unused-rpmlintrc-filter" errors') + lint_modes_parser = parser.add_mutually_exclusive_group() + lint_modes_parser.add_argument('-s', '--strict', action='store_true', help='treat all messages as errors') + lint_modes_parser.add_argument('-P', '--permissive', action='store_true', help='treat individual errors as non-fatal') +diff --git a/rpmlint/lint.py b/rpmlint/lint.py +index a98d2c04..1d5e0050 100644 +--- a/rpmlint/lint.py ++++ b/rpmlint/lint.py +@@ -277,7 +277,8 @@ + + # validate used filters in rpmlintrc + if is_last: +- self.output.validate_filters(pkg) ++ if not self.options['ignore_unused_rpmlintrc']: ++ self.output.validate_filters(pkg) + + if spec_checks: + self.specfiles_checked += 1 From 7acf648636b11dd0a2c765f34ce66fe182c14377 Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Thu, 2 Jun 2022 18:38:06 +0000 Subject: [PATCH 06/40] DAOS-10686 API: Add "-g" in compiling and linking flags Signed-off-by: Lei Huang --- mpifileutils.spec | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mpifileutils.spec b/mpifileutils.spec index 96840b2..039b7eb 100644 --- a/mpifileutils.spec +++ b/mpifileutils.spec @@ -175,8 +175,9 @@ for mpi in %{?mpi_list}; do mkdir $mpi pushd $mpi %module_load $mpi - %cmake ../ -DCMAKE_C_FLAGS="${RPM_OPT_FLAGS}" \ - -DCMAKE_CXX_FLAGS="${RPM_OPT_FLAGS}" \ + %cmake ../ -DCMAKE_C_FLAGS="${RPM_OPT_FLAGS} -g " \ + -DCMAKE_CXX_FLAGS="${RPM_OPT_FLAGS} -g " \ + -DCMAKE_LD_FLAGS="${RPM_OPT_FLAGS} -g " \ -DENABLE_DAOS=ON \ -DENABLE_LIBARCHIVE=OFF \ -DENABLE_HDF5=ON \ From 175dd2dca371d7c4ba87cf54203f12f1b090ef14 Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Thu, 2 Jun 2022 20:29:36 +0000 Subject: [PATCH 07/40] DAOS-10686 API: Disable static lib by remove libmfu.a during packaging Signed-off-by: Lei Huang --- mpifileutils.spec | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mpifileutils.spec b/mpifileutils.spec index 039b7eb..aa7af9b 100644 --- a/mpifileutils.spec +++ b/mpifileutils.spec @@ -175,9 +175,8 @@ for mpi in %{?mpi_list}; do mkdir $mpi pushd $mpi %module_load $mpi - %cmake ../ -DCMAKE_C_FLAGS="${RPM_OPT_FLAGS} -g " \ - -DCMAKE_CXX_FLAGS="${RPM_OPT_FLAGS} -g " \ - -DCMAKE_LD_FLAGS="${RPM_OPT_FLAGS} -g " \ + %cmake ../ -DCMAKE_C_FLAGS="${RPM_OPT_FLAGS}" \ + -DCMAKE_CXX_FLAGS="${RPM_OPT_FLAGS}" \ -DENABLE_DAOS=ON \ -DENABLE_LIBARCHIVE=OFF \ -DENABLE_HDF5=ON \ @@ -202,6 +201,7 @@ rm -rf %{buildroot} for mpi in %{?mpi_list}; do %module_load $mpi make install -C $mpi DESTDIR=%{buildroot} + rm %{buildroot}%{mpi_libdir}/${mpi}/%{mpi_lib_ext}/libmfu.a module purge done From 1d77888d033cea7d534a429e7fd054ba54271d2f Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Thu, 2 Jun 2022 20:31:47 +0000 Subject: [PATCH 08/40] DAOS-10686 API: fix format in mpifileutils.spec Signed-off-by: Lei Huang --- mpifileutils.spec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mpifileutils.spec b/mpifileutils.spec index aa7af9b..ea323fa 100644 --- a/mpifileutils.spec +++ b/mpifileutils.spec @@ -175,8 +175,8 @@ for mpi in %{?mpi_list}; do mkdir $mpi pushd $mpi %module_load $mpi - %cmake ../ -DCMAKE_C_FLAGS="${RPM_OPT_FLAGS}" \ - -DCMAKE_CXX_FLAGS="${RPM_OPT_FLAGS}" \ + %cmake ../ -DCMAKE_C_FLAGS="${RPM_OPT_FLAGS}" \ + -DCMAKE_CXX_FLAGS="${RPM_OPT_FLAGS}" \ -DENABLE_DAOS=ON \ -DENABLE_LIBARCHIVE=OFF \ -DENABLE_HDF5=ON \ From 4a4c61569e763980771422c13c6c1c4f2de0d051 Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Thu, 2 Jun 2022 20:50:10 +0000 Subject: [PATCH 09/40] DAOS-10686 API: do not install libmfu.a Signed-off-by: Lei Huang --- mpifileutils.spec | 3 --- 1 file changed, 3 deletions(-) diff --git a/mpifileutils.spec b/mpifileutils.spec index ea323fa..19b6411 100644 --- a/mpifileutils.spec +++ b/mpifileutils.spec @@ -215,7 +215,6 @@ done %files openmpi-devel %{mpi_includedir}/openmpi%{mpi_include_ext}/* %{mpi_libdir}/openmpi/%{mpi_lib_ext}/lib*.so -%{mpi_libdir}/openmpi/%{mpi_lib_ext}/lib*.a %endif %if %{with_openmpi3} @@ -231,7 +230,6 @@ done %files openmpi3-devel %{mpi_includedir}/openmpi3%{mpi_include_ext}/* %{mpi_libdir}/openmpi3/%{mpi_lib_ext}/lib*.so -%{mpi_libdir}/openmpi3/%{mpi_lib_ext}/lib*.a %endif %if %{with_mpich} @@ -247,7 +245,6 @@ done %files mpich-devel %{mpi_includedir}/mpich%{mpi_include_ext}/* %{mpi_libdir}/mpich/%{mpi_lib_ext}/lib*.so -%{mpi_libdir}/mpich/%{mpi_lib_ext}/lib*.a %endif %changelog From 310fee2f8dabb62051cbc9bb8484d044d74056ec Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Sun, 5 Jun 2022 02:50:54 +0000 Subject: [PATCH 10/40] DAOS-10686 API: fix rpmlint warnings in .spec Signed-off-by: Lei Huang --- mpifileutils.spec | 41 +++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/mpifileutils.spec b/mpifileutils.spec index 19b6411..15cb6f6 100644 --- a/mpifileutils.spec +++ b/mpifileutils.spec @@ -51,10 +51,10 @@ Name: mpifileutils Version: 0.11.1 Release: 3%{?commit:.g%{shortcommit}}%{?dist} -Summary: File utilities designed for scalability and performance. +Summary: File utilities designed for scalability and performance Group: System Environment/Libraries -License: Copyright and BSD License +License: BSD URL: https://hpc.github.io/mpifileutils Source: https://github.com/hpc/%{name}/archive/v%{version}.tar.gz %if "%{?commit}" != "" @@ -82,22 +82,22 @@ BuildRequires: libfabric1 >= 1.12.0 %endif %description -File utilities designed for scalability and performance. +File utilities designed for scalability and performance %if %{with_openmpi} %package openmpi -Summary: File utilities designed for scalability and performance. +Summary: File utilities designed for scalability and performance BuildRequires: openmpi-devel BuildRequires: dtcmp-openmpi-devel BuildRequires: libcircle-openmpi-devel BuildRequires: hdf5-vol-daos-openmpi-devel %description openmpi -File utilities designed for scalability and performance. +File utilities designed for scalability and performance %package openmpi-devel -Summary: File utilities designed for scalability and performance. +Summary: File utilities designed for scalability and performance Requires: %{name}-openmpi%{_isa} = %version-%release %description openmpi-devel @@ -106,25 +106,25 @@ Development files for %{name}-openmpi. %if %{with_openmpi3} %package openmpi3 -Summary: File utilities designed for scalability and performance. +Summary: File utilities designed for scalability and performance BuildRequires: openmpi3-devel BuildRequires: dtcmp-openmpi3-devel BuildRequires: libcircle-openmpi3-devel BuildRequires: hdf5-vol-daos-openmpi3-devel %description openmpi3 -File utilities designed for scalability and performance. +File utilities designed for scalability and performance %if (0%{?suse_version} >= 1500) %package -n libmfu0-openmpi3 -Summary: File utilities designed for scalability and performance. +Summary: File utilities designed for scalability and performance %description -n libmfu0-openmpi3 Shared libraries for %{name}-openmpi3. %endif %package openmpi3-devel -Summary: File utilities designed for scalability and performance. +Summary: File utilities designed for scalability and performance %if (0%{?suse_version} >= 1500) Requires: libmfu0-openmpi3%{_isa} = %version-%release %else @@ -137,25 +137,25 @@ Development files for %{name}-openmpi3. %if %{with_mpich} %package mpich -Summary: File utilities designed for scalability and performance. +Summary: File utilities designed for scalability and performance BuildRequires: mpich-devel BuildRequires: dtcmp-mpich-devel BuildRequires: libcircle-mpich-devel BuildRequires: hdf5-vol-daos-mpich-devel %description mpich -File utilities designed for scalability and performance. +File utilities designed for scalability and performance %if (0%{?suse_version} >= 1500) %package -n libmfu0-mpich -Summary: File utilities designed for scalability and performance. +Summary: File utilities designed for scalability and performance %description -n libmfu0-mpich Shared libraries for %{name}-mpich. %endif %package mpich-devel -Summary: File utilities designed for scalability and performance. +Summary: File utilities designed for scalability and performance %if (0%{?suse_version} >= 1500) Requires: libmfu0-mpich%{_isa} = %version-%release %else @@ -202,6 +202,7 @@ for mpi in %{?mpi_list}; do %module_load $mpi make install -C $mpi DESTDIR=%{buildroot} rm %{buildroot}%{mpi_libdir}/${mpi}/%{mpi_lib_ext}/libmfu.a + find %{buildroot}%{mpi_libdir}/${mpi}/share/man -type f -exec gzip -9 -n {} \; module purge done @@ -247,6 +248,18 @@ done %{mpi_libdir}/mpich/%{mpi_lib_ext}/lib*.so %endif +%post -n mpifileutils-debuginfo -p /sbin/ldconfig +%post -n mpifileutils-mpich -p /sbin/ldconfig +%post -n mpifileutils-mpich-devel -p /sbin/ldconfig +%post -n mpifileutils-openmpi3 -p /sbin/ldconfig +%post -n mpifileutils-openmpi3-devel -p /sbin/ldconfig + +%postun -n mpifileutils-debuginfo -p /sbin/ldconfig +%postun -n mpifileutils-mpich -p /sbin/ldconfig +%postun -n mpifileutils-mpich-devel -p /sbin/ldconfig +%postun -n mpifileutils-openmpi3 -p /sbin/ldconfig +%postun -n mpifileutils-openmpi3-devel -p /sbin/ldconfig + %changelog *Wed Jun 01 2022 Lei Huang - 0.11.1-3 - Update to patch 6230ed2 to support empty container label From 24e9e30e6e6ad0b5e20c8f24587814b9ff5a0570 Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Sun, 5 Jun 2022 03:49:09 +0000 Subject: [PATCH 11/40] DAOS-10686 API: fix rpmlint warnings in .spec Signed-off-by: Lei Huang --- mpifileutils.spec | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/mpifileutils.spec b/mpifileutils.spec index 15cb6f6..d59a2de 100644 --- a/mpifileutils.spec +++ b/mpifileutils.spec @@ -82,7 +82,9 @@ BuildRequires: libfabric1 >= 1.12.0 %endif %description -File utilities designed for scalability and performance +mpiFileUtils provides both a library called libmfu and a suite of MPI-based +tools to manage large datasets, which may vary from large directory trees +to large files. %if %{with_openmpi} %package openmpi @@ -93,15 +95,17 @@ BuildRequires: libcircle-openmpi-devel BuildRequires: hdf5-vol-daos-openmpi-devel %description openmpi -File utilities designed for scalability and performance - +mpiFileUtils provides both a library called libmfu and a suite of MPI-based +tools to manage large datasets, which may vary from large directory trees +to large files. %package openmpi-devel Summary: File utilities designed for scalability and performance Requires: %{name}-openmpi%{_isa} = %version-%release %description openmpi-devel -Development files for %{name}-openmpi. +Development files for %{name}-openmpi which provides file utilities +designed for scalability and performance. %endif %if %{with_openmpi3} @@ -113,7 +117,9 @@ BuildRequires: libcircle-openmpi3-devel BuildRequires: hdf5-vol-daos-openmpi3-devel %description openmpi3 -File utilities designed for scalability and performance +mpiFileUtils provides both a library called libmfu and a suite of MPI-based +tools to manage large datasets, which may vary from large directory trees +to large files. %if (0%{?suse_version} >= 1500) %package -n libmfu0-openmpi3 @@ -132,7 +138,8 @@ Requires: %{name}-openmpi3%{_isa} = %version-%release %endif %description openmpi3-devel -Development files for %{name}-openmpi3. +Development files for %{name}-openmpi3 which provides file utilities +designed for scalability and performance. %endif %if %{with_mpich} @@ -144,7 +151,9 @@ BuildRequires: libcircle-mpich-devel BuildRequires: hdf5-vol-daos-mpich-devel %description mpich -File utilities designed for scalability and performance +mpiFileUtils provides both a library called libmfu and a suite of MPI-based +tools to manage large datasets, which may vary from large directory trees +to large files. %if (0%{?suse_version} >= 1500) %package -n libmfu0-mpich @@ -163,7 +172,8 @@ Requires: %{name}-mpich%{_isa} = %version-%release %endif %description mpich-devel -Development files for %{name}-mpich. +Development files for %{name}-mpich which provides file utilities +designed for scalability and performance. %endif @@ -175,8 +185,9 @@ for mpi in %{?mpi_list}; do mkdir $mpi pushd $mpi %module_load $mpi - %cmake ../ -DCMAKE_C_FLAGS="${RPM_OPT_FLAGS}" \ - -DCMAKE_CXX_FLAGS="${RPM_OPT_FLAGS}" \ + %cmake ../ -DCMAKE_C_FLAGS="${RPM_OPT_FLAGS} -g -fPIC " \ + -DCMAKE_CXX_FLAGS="${RPM_OPT_FLAGS} -g -fPIC " \ + -DCMAKE_LD_FLAGS="${RPM_OPT_FLAGS} -g -fPIC " \ -DENABLE_DAOS=ON \ -DENABLE_LIBARCHIVE=OFF \ -DENABLE_HDF5=ON \ From a074e227638a462ddf3b19a2dda53ce3510cbd3e Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Mon, 6 Jun 2022 13:38:56 +0000 Subject: [PATCH 12/40] DAOS-10686 API: testing changes for rpmlint warnings in .spec Signed-off-by: Lei Huang --- mpifileutils.spec | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/mpifileutils.spec b/mpifileutils.spec index d59a2de..521cda7 100644 --- a/mpifileutils.spec +++ b/mpifileutils.spec @@ -185,9 +185,9 @@ for mpi in %{?mpi_list}; do mkdir $mpi pushd $mpi %module_load $mpi - %cmake ../ -DCMAKE_C_FLAGS="${RPM_OPT_FLAGS} -g -fPIC " \ - -DCMAKE_CXX_FLAGS="${RPM_OPT_FLAGS} -g -fPIC " \ - -DCMAKE_LD_FLAGS="${RPM_OPT_FLAGS} -g -fPIC " \ + %cmake ../ -DCMAKE_C_FLAGS="${RPM_OPT_FLAGS} -g -fPIE " \ + -DCMAKE_CXX_FLAGS="${RPM_OPT_FLAGS} -g -fPIE " \ + -DCMAKE_LD_FLAGS="${RPM_OPT_FLAGS} -g -fPIE " \ -DENABLE_DAOS=ON \ -DENABLE_LIBARCHIVE=OFF \ -DENABLE_HDF5=ON \ @@ -259,13 +259,11 @@ done %{mpi_libdir}/mpich/%{mpi_lib_ext}/lib*.so %endif -%post -n mpifileutils-debuginfo -p /sbin/ldconfig %post -n mpifileutils-mpich -p /sbin/ldconfig %post -n mpifileutils-mpich-devel -p /sbin/ldconfig %post -n mpifileutils-openmpi3 -p /sbin/ldconfig %post -n mpifileutils-openmpi3-devel -p /sbin/ldconfig -%postun -n mpifileutils-debuginfo -p /sbin/ldconfig %postun -n mpifileutils-mpich -p /sbin/ldconfig %postun -n mpifileutils-mpich-devel -p /sbin/ldconfig %postun -n mpifileutils-openmpi3 -p /sbin/ldconfig From 9122baf186d1a0840baad30f2fcea195febb9b0c Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Mon, 6 Jun 2022 20:16:00 +0000 Subject: [PATCH 13/40] DAOS-10686 API: testing changes for rpmlint warnings in .spec Removed post and postun in .spec. Signed-off-by: Lei Huang --- mpifileutils.spec | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/mpifileutils.spec b/mpifileutils.spec index 521cda7..abe5fb0 100644 --- a/mpifileutils.spec +++ b/mpifileutils.spec @@ -259,16 +259,6 @@ done %{mpi_libdir}/mpich/%{mpi_lib_ext}/lib*.so %endif -%post -n mpifileutils-mpich -p /sbin/ldconfig -%post -n mpifileutils-mpich-devel -p /sbin/ldconfig -%post -n mpifileutils-openmpi3 -p /sbin/ldconfig -%post -n mpifileutils-openmpi3-devel -p /sbin/ldconfig - -%postun -n mpifileutils-mpich -p /sbin/ldconfig -%postun -n mpifileutils-mpich-devel -p /sbin/ldconfig -%postun -n mpifileutils-openmpi3 -p /sbin/ldconfig -%postun -n mpifileutils-openmpi3-devel -p /sbin/ldconfig - %changelog *Wed Jun 01 2022 Lei Huang - 0.11.1-3 - Update to patch 6230ed2 to support empty container label From f92afce437b3015d7c45cacb21b2be562b241e46 Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Mon, 6 Jun 2022 21:10:11 +0000 Subject: [PATCH 14/40] DAOS-10686 API: testing changes for rpmlint warnings in .spec Try "-pie" in LD_FLAGS. Signed-off-by: Lei Huang --- mpifileutils.spec | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mpifileutils.spec b/mpifileutils.spec index abe5fb0..e725052 100644 --- a/mpifileutils.spec +++ b/mpifileutils.spec @@ -126,7 +126,8 @@ to large files. Summary: File utilities designed for scalability and performance %description -n libmfu0-openmpi3 -Shared libraries for %{name}-openmpi3. +Shared libraries for %{name}-openmpi3, file utilities designed for +scalability and performance. %endif %package openmpi3-devel @@ -187,7 +188,7 @@ for mpi in %{?mpi_list}; do %module_load $mpi %cmake ../ -DCMAKE_C_FLAGS="${RPM_OPT_FLAGS} -g -fPIE " \ -DCMAKE_CXX_FLAGS="${RPM_OPT_FLAGS} -g -fPIE " \ - -DCMAKE_LD_FLAGS="${RPM_OPT_FLAGS} -g -fPIE " \ + -DCMAKE_LD_FLAGS="${RPM_OPT_FLAGS} -g -pie " \ -DENABLE_DAOS=ON \ -DENABLE_LIBARCHIVE=OFF \ -DENABLE_HDF5=ON \ From 6fab3821259409899811fbc7bcdd911eab860a7b Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Tue, 7 Jun 2022 02:58:13 +0000 Subject: [PATCH 15/40] DAOS-10686 API: Try installing dependent packages first Try "-pie" in LD_FLAGS. Signed-off-by: Lei Huang --- packaging/Makefile_packaging.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/packaging/Makefile_packaging.mk b/packaging/Makefile_packaging.mk index 8bd618e..d9be453 100644 --- a/packaging/Makefile_packaging.mk +++ b/packaging/Makefile_packaging.mk @@ -463,6 +463,7 @@ endif test: # Test the rpmbuild by installing the built RPM $(call install_repos,$(REPO_NAME)@$(BRANCH_NAME):$(BUILD_NUMBER)) + dnf -y install hdf5-mpich daos daos-client dtcmp-mpich dtcmp-mpich-devel libcircle-mpich libcircle-mpich-devel dnf -y install $(TEST_PACKAGES) show_NAME: From 25abfb25c3799e02acd2137021d77b6f7542802b Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Tue, 7 Jun 2022 03:05:27 +0000 Subject: [PATCH 16/40] DAOS-10686 API: Fixed a typo Signed-off-by: Lei Huang --- packaging/Makefile_packaging.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/Makefile_packaging.mk b/packaging/Makefile_packaging.mk index d9be453..4461eda 100644 --- a/packaging/Makefile_packaging.mk +++ b/packaging/Makefile_packaging.mk @@ -463,7 +463,7 @@ endif test: # Test the rpmbuild by installing the built RPM $(call install_repos,$(REPO_NAME)@$(BRANCH_NAME):$(BUILD_NUMBER)) - dnf -y install hdf5-mpich daos daos-client dtcmp-mpich dtcmp-mpich-devel libcircle-mpich libcircle-mpich-devel + dnf -y install hdf5-mpich daos daos-client dtcmp-mpich dtcmp-mpich-devel libcircle-mpich libcircle-mpich-devel dnf -y install $(TEST_PACKAGES) show_NAME: From 231ff4e0e301e578d8bace5b4823f3f862386741 Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Tue, 7 Jun 2022 03:13:06 +0000 Subject: [PATCH 17/40] DAOS-10686 API: testing installation dependent packages Signed-off-by: Lei Huang --- packaging/Makefile_packaging.mk | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packaging/Makefile_packaging.mk b/packaging/Makefile_packaging.mk index 4461eda..0145f17 100644 --- a/packaging/Makefile_packaging.mk +++ b/packaging/Makefile_packaging.mk @@ -463,8 +463,7 @@ endif test: # Test the rpmbuild by installing the built RPM $(call install_repos,$(REPO_NAME)@$(BRANCH_NAME):$(BUILD_NUMBER)) - dnf -y install hdf5-mpich daos daos-client dtcmp-mpich dtcmp-mpich-devel libcircle-mpich libcircle-mpich-devel - dnf -y install $(TEST_PACKAGES) + dnf -y install hdf5-mpich daos daos-client dtcmp-mpich dtcmp-mpich-devel libcircle-mpich libcircle-mpich-devel $(TEST_PACKAGES) show_NAME: @echo '$(NAME)' From 0686a656b787b477fb349dc14defd34d1c7a98a9 Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Tue, 7 Jun 2022 14:33:00 +0000 Subject: [PATCH 18/40] DAOS-10686 API: Revert dnf -y install in Makefile_packaging.mk Signed-off-by: Lei Huang --- packaging/Makefile_packaging.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/Makefile_packaging.mk b/packaging/Makefile_packaging.mk index 0145f17..8bd618e 100644 --- a/packaging/Makefile_packaging.mk +++ b/packaging/Makefile_packaging.mk @@ -463,7 +463,7 @@ endif test: # Test the rpmbuild by installing the built RPM $(call install_repos,$(REPO_NAME)@$(BRANCH_NAME):$(BUILD_NUMBER)) - dnf -y install hdf5-mpich daos daos-client dtcmp-mpich dtcmp-mpich-devel libcircle-mpich libcircle-mpich-devel $(TEST_PACKAGES) + dnf -y install $(TEST_PACKAGES) show_NAME: @echo '$(NAME)' From b3c6e363c03e9b2dbdd1dbda16224a3e821bbf12 Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Fri, 10 Jun 2022 04:29:08 +0000 Subject: [PATCH 19/40] DAOS-10686 mpifileutils: Add John's patch and verbose for mock Signed-off-by: Lei Huang --- mpifileutils.spec | 7 +++---- packaging/Dockerfile.centos.7 | 4 ++-- packaging/Makefile_packaging.mk | 21 ++++++++++++--------- packaging/rpm_chrootbuild | 2 +- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/mpifileutils.spec b/mpifileutils.spec index e725052..7e456ec 100644 --- a/mpifileutils.spec +++ b/mpifileutils.spec @@ -182,14 +182,13 @@ designed for scalability and performance. %autosetup -p1 %build +export CFLAGS="%{RPM_OPT_FLAGS} -fPIC -pie" +export CXXFLAGS="%{RPM_OPT_FLAGS} -fPIC -pie" for mpi in %{?mpi_list}; do mkdir $mpi pushd $mpi %module_load $mpi - %cmake ../ -DCMAKE_C_FLAGS="${RPM_OPT_FLAGS} -g -fPIE " \ - -DCMAKE_CXX_FLAGS="${RPM_OPT_FLAGS} -g -fPIE " \ - -DCMAKE_LD_FLAGS="${RPM_OPT_FLAGS} -g -pie " \ - -DENABLE_DAOS=ON \ + %cmake ../ -DENABLE_DAOS=ON \ -DENABLE_LIBARCHIVE=OFF \ -DENABLE_HDF5=ON \ -DDTCMP_INCLUDE_DIRS=%{mpi_includedir}/$mpi%{mpi_include_ext} \ diff --git a/packaging/Dockerfile.centos.7 b/packaging/Dockerfile.centos.7 index cdfb7f6..62ed5c6 100644 --- a/packaging/Dockerfile.centos.7 +++ b/packaging/Dockerfile.centos.7 @@ -16,8 +16,8 @@ ARG UID=1000 # Install basic tools RUN yum install -y epel-release -RUN yum install -y mock make rpm-build curl createrepo rpmlint redhat-lsb-core \ - git python-srpm-macros dnf +RUN yum install -y curl createrepo dnf git make mock python-srpm-macros \ + redhat-lsb-core rpm-build rpmdevtools rpmlint # Add build user (to keep rpmbuild happy) ENV USER build diff --git a/packaging/Makefile_packaging.mk b/packaging/Makefile_packaging.mk index 8bd618e..59ce452 100644 --- a/packaging/Makefile_packaging.mk +++ b/packaging/Makefile_packaging.mk @@ -98,13 +98,11 @@ define distro_map endef define install_repos - if [ "$(ID_LIKE)" = "debian" ]; then \ - IFS='|' read -ra BASES <<< "$($(DISTRO_BASE)_LOCAL_REPOS)"; \ - for baseurl in "$${BASES[@]}"; do \ - baseurl="$${baseurl# *}"; \ - $(call install_repo,$$baseurl) \ - done; \ - fi + IFS='|' read -ra BASES <<< "$($(DISTRO_BASE)_LOCAL_REPOS)"; \ + for baseurl in "$${BASES[@]}"; do \ + baseurl="$${baseurl# *}"; \ + $(call install_repo,$$baseurl) \ + done for repo in $($(DISTRO_BASE)_PR_REPOS) \ $(PR_REPOS) $(1); do \ branch="master"; \ @@ -347,8 +345,13 @@ ifeq ($(LOCAL_REPOS),true) DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO := $(subst reposi,artifac,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)) # $(DISTRO_BASE)_LOCAL_REPOS is a list separated by | because you cannot pass lists # of values with spaces as environment variables - $(DISTRO_BASE)_LOCAL_REPOS := [trusted=yes] $(ARTIFACTORY_URL)$(subst stack,stack-daos,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)) - $(DISTRO_BASE)_LOCAL_REPOS += |[trusted=yes] $(ARTIFACTORY_URL)$(subst stack,stack-deps,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)) + ifeq ($(ID_LIKE), debian) + $(DISTRO_BASE)_LOCAL_REPOS := [trusted=yes] $(ARTIFACTORY_URL)$(subst stack,stack-daos,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)) + $(DISTRO_BASE)_LOCAL_REPOS += |[trusted=yes] $(ARTIFACTORY_URL)$(subst stack,stack-deps,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)) + else + $(DISTRO_BASE)_LOCAL_REPOS := $(ARTIFACTORY_URL)$(subst stack,stack-daos,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO))/ + $(DISTRO_BASE)_LOCAL_REPOS += |$(ARTIFACTORY_URL)$(subst stack,stack-deps,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO))/ + endif endif #ifneq ($(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO),) endif # ifneq ($(ARTIFACTORY_URL),) endif # ifeq ($(LOCAL_REPOS),true) diff --git a/packaging/rpm_chrootbuild b/packaging/rpm_chrootbuild index 03ab4c5..89ecdfc 100755 --- a/packaging/rpm_chrootbuild +++ b/packaging/rpm_chrootbuild @@ -89,4 +89,4 @@ fi # shellcheck disable=SC2086 eval mock --configdir "$mock_config_dir" -r "${CHROOT_NAME}" \ ${repo_dels[*]} ${repo_adds[*]} --disablerepo=\*-debug* \ - "${releasever_opt[@]}" $MOCK_OPTIONS $RPM_BUILD_OPTIONS "$TARGET" + "${releasever_opt[@]}" $MOCK_OPTIONS $RPM_BUILD_OPTIONS "$TARGET" -v From 77affbd599a5103ce022e0dd62eceb217291c73f Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Fri, 10 Jun 2022 04:41:50 +0000 Subject: [PATCH 20/40] DAOS-10686 mpifileutils: Use optflags in .spec Signed-off-by: Lei Huang --- mpifileutils.spec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mpifileutils.spec b/mpifileutils.spec index 7e456ec..542f21b 100644 --- a/mpifileutils.spec +++ b/mpifileutils.spec @@ -182,8 +182,8 @@ designed for scalability and performance. %autosetup -p1 %build -export CFLAGS="%{RPM_OPT_FLAGS} -fPIC -pie" -export CXXFLAGS="%{RPM_OPT_FLAGS} -fPIC -pie" +export CFLAGS="%{optflags} -fPIC -pie" +export CXXFLAGS="%{optflags} -fPIC -pie" for mpi in %{?mpi_list}; do mkdir $mpi pushd $mpi From 40571e218de3ee3db6b0a37313a8b04ac102e9f1 Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Mon, 13 Jun 2022 15:10:53 +0000 Subject: [PATCH 21/40] DAOS-10686 API: Add rpm for ubuntu 20 Signed-off-by: Lei Huang --- Jenkinsfile | 2 +- mpifileutils.spec | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index afd864e..143b3ed 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,5 +40,5 @@ // I.e. for testing library changes //@Library(value="pipeline-lib@your_branch") _ -packageBuildingPipeline(['distros' : ['centos7', 'el8', 'leap15']]) +packageBuildingPipeline(['distros' : ['centos7', 'el8', 'leap15', 'ubuntu20.04']]) diff --git a/mpifileutils.spec b/mpifileutils.spec index 542f21b..fd0e4bd 100644 --- a/mpifileutils.spec +++ b/mpifileutils.spec @@ -161,7 +161,8 @@ to large files. Summary: File utilities designed for scalability and performance %description -n libmfu0-mpich -Shared libraries for %{name}-mpich. +Shared libraries for %{name}-mpich, file utilities designed for +scalability and performance. %endif %package mpich-devel From 596442a9a770bce784b56dcbfc326ab11cf9282f Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Sat, 18 Jun 2022 05:09:50 +0000 Subject: [PATCH 22/40] DAOS-10686 mpifileutils: strip binary files and add post Signed-off-by: Lei Huang --- Jenkinsfile | 2 +- mpifileutils.spec | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 143b3ed..afd864e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,5 +40,5 @@ // I.e. for testing library changes //@Library(value="pipeline-lib@your_branch") _ -packageBuildingPipeline(['distros' : ['centos7', 'el8', 'leap15', 'ubuntu20.04']]) +packageBuildingPipeline(['distros' : ['centos7', 'el8', 'leap15']]) diff --git a/mpifileutils.spec b/mpifileutils.spec index fd0e4bd..ed18dea 100644 --- a/mpifileutils.spec +++ b/mpifileutils.spec @@ -214,10 +214,20 @@ for mpi in %{?mpi_list}; do %module_load $mpi make install -C $mpi DESTDIR=%{buildroot} rm %{buildroot}%{mpi_libdir}/${mpi}/%{mpi_lib_ext}/libmfu.a + strip --strip-unneeded %{buildroot}%{mpi_libdir}/${mpi}/%{mpi_lib_ext}/libmfu* + strip --strip-unneeded %{buildroot}%{mpi_libdir}/${mpi}/bin/* find %{buildroot}%{mpi_libdir}/${mpi}/share/man -type f -exec gzip -9 -n {} \; module purge done +%if 0%{?suse_version} >= 01315 +%post -n %{suse_libname} -p /sbin/ldconfig +%postun -n %{suse_libname} -p /sbin/ldconfig +%else +%post -p /sbin/ldconfig +%postun -p /sbin/ldconfig +%endif + %if %{with_openmpi} %files openmpi %defattr(-,root,root,-) From 0586eaa37e5b6603e299b60adc3840a9b6fb0dbd Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Sat, 18 Jun 2022 05:20:51 +0000 Subject: [PATCH 23/40] DAOS-10686 mpifileutils: remove post in .spec Signed-off-by: Lei Huang --- mpifileutils.spec | 8 -------- 1 file changed, 8 deletions(-) diff --git a/mpifileutils.spec b/mpifileutils.spec index ed18dea..e20659f 100644 --- a/mpifileutils.spec +++ b/mpifileutils.spec @@ -220,14 +220,6 @@ for mpi in %{?mpi_list}; do module purge done -%if 0%{?suse_version} >= 01315 -%post -n %{suse_libname} -p /sbin/ldconfig -%postun -n %{suse_libname} -p /sbin/ldconfig -%else -%post -p /sbin/ldconfig -%postun -p /sbin/ldconfig -%endif - %if %{with_openmpi} %files openmpi %defattr(-,root,root,-) From 955cdf935716f70e5cfefb861901aa0c30e1b0e2 Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Sat, 18 Jun 2022 12:33:29 +0000 Subject: [PATCH 24/40] DAOS-10686 mpifileutils: Use Tab to replace space Signed-off-by: Lei Huang --- mpifileutils.spec | 58 +++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/mpifileutils.spec b/mpifileutils.spec index e20659f..201d42b 100644 --- a/mpifileutils.spec +++ b/mpifileutils.spec @@ -186,38 +186,42 @@ designed for scalability and performance. export CFLAGS="%{optflags} -fPIC -pie" export CXXFLAGS="%{optflags} -fPIC -pie" for mpi in %{?mpi_list}; do - mkdir $mpi - pushd $mpi - %module_load $mpi - %cmake ../ -DENABLE_DAOS=ON \ - -DENABLE_LIBARCHIVE=OFF \ - -DENABLE_HDF5=ON \ - -DDTCMP_INCLUDE_DIRS=%{mpi_includedir}/$mpi%{mpi_include_ext} \ - -DDTCMP_LIBRARIES=%{mpi_libdir}/$mpi/%{mpi_lib_ext}/libdtcmp.so \ - -DLibCircle_INCLUDE_DIRS=%{mpi_includedir}/$mpi%{mpi_include_ext} \ - -DLibCircle_LIBRARIES=%{mpi_libdir}/$mpi/%{mpi_lib_ext}/libcircle.so \ - -DHDF5_INCLUDE_DIRS=%{mpi_includedir}/$mpi%{mpi_include_ext} \ - -DHDF5_LIBRARIES=%{mpi_libdir}/$mpi/%{mpi_lib_ext}/libhdf5.so \ - -DWITH_DAOS_PREFIX=/usr \ - -DCMAKE_INSTALL_INCLUDEDIR=%{mpi_includedir}/$mpi%{mpi_include_ext} \ - -DCMAKE_INSTALL_PREFIX=%{mpi_libdir}/$mpi \ - -DCMAKE_INSTALL_LIBDIR=%{mpi_lib_ext} - - make - module purge - popd + mkdir $mpi + pushd $mpi + %module_load $mpi + %cmake ../ -DENABLE_DAOS=ON \ + -DENABLE_LIBARCHIVE=OFF \ + -DENABLE_HDF5=ON \ + -DDTCMP_INCLUDE_DIRS=%{mpi_includedir}/$mpi%{mpi_include_ext} \ + -DDTCMP_LIBRARIES=%{mpi_libdir}/$mpi/%{mpi_lib_ext}/libdtcmp.so \ + -DLibCircle_INCLUDE_DIRS=%{mpi_includedir}/$mpi%{mpi_include_ext} \ + -DLibCircle_LIBRARIES=%{mpi_libdir}/$mpi/%{mpi_lib_ext}/libcircle.so \ + -DHDF5_INCLUDE_DIRS=%{mpi_includedir}/$mpi%{mpi_include_ext} \ + -DHDF5_LIBRARIES=%{mpi_libdir}/$mpi/%{mpi_lib_ext}/libhdf5.so \ + -DWITH_DAOS_PREFIX=/usr \ + -DCMAKE_INSTALL_INCLUDEDIR=%{mpi_includedir}/$mpi%{mpi_include_ext} \ + -DCMAKE_INSTALL_PREFIX=%{mpi_libdir}/$mpi \ + -DCMAKE_INSTALL_LIBDIR=%{mpi_lib_ext} + + make + module purge + popd done %install rm -rf %{buildroot} for mpi in %{?mpi_list}; do - %module_load $mpi - make install -C $mpi DESTDIR=%{buildroot} - rm %{buildroot}%{mpi_libdir}/${mpi}/%{mpi_lib_ext}/libmfu.a - strip --strip-unneeded %{buildroot}%{mpi_libdir}/${mpi}/%{mpi_lib_ext}/libmfu* - strip --strip-unneeded %{buildroot}%{mpi_libdir}/${mpi}/bin/* - find %{buildroot}%{mpi_libdir}/${mpi}/share/man -type f -exec gzip -9 -n {} \; - module purge + %module_load $mpi + make install -C $mpi DESTDIR=%{buildroot} + rm %{buildroot}%{mpi_libdir}/${mpi}/%{mpi_lib_ext}/libmfu.a + +%if (0%{?suse_version} >= 1500) + strip --strip-unneeded %{buildroot}%{mpi_libdir}/${mpi}/%{mpi_lib_ext}/libmfu* + strip --strip-unneeded %{buildroot}%{mpi_libdir}/${mpi}/bin/* +%endif + + find %{buildroot}%{mpi_libdir}/${mpi}/share/man -type f -exec gzip -9 -n {} \; + module purge done %if %{with_openmpi} From 64b4610c2d412e845c3348da66edbcf9a74ea449 Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Wed, 22 Jun 2022 16:28:09 +0000 Subject: [PATCH 25/40] OS-10686 mpifileutils: Building debuginfo packages for Leap 15 Signed-off-by: Lei Huang --- mpifileutils.spec | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/mpifileutils.spec b/mpifileutils.spec index 201d42b..acdae02 100644 --- a/mpifileutils.spec +++ b/mpifileutils.spec @@ -94,6 +94,12 @@ BuildRequires: dtcmp-openmpi-devel BuildRequires: libcircle-openmpi-devel BuildRequires: hdf5-vol-daos-openmpi-devel +%if (0%{?suse_version} > 0) +%global __debug_package 1 +%global _debuginfo_subpackages 0 +%debug_package +%endif + %description openmpi mpiFileUtils provides both a library called libmfu and a suite of MPI-based tools to manage large datasets, which may vary from large directory trees @@ -116,6 +122,12 @@ BuildRequires: dtcmp-openmpi3-devel BuildRequires: libcircle-openmpi3-devel BuildRequires: hdf5-vol-daos-openmpi3-devel +%if (0%{?suse_version} > 0) +%global __debug_package 1 +%global _debuginfo_subpackages 0 +%debug_package +%endif + %description openmpi3 mpiFileUtils provides both a library called libmfu and a suite of MPI-based tools to manage large datasets, which may vary from large directory trees @@ -151,6 +163,12 @@ BuildRequires: dtcmp-mpich-devel BuildRequires: libcircle-mpich-devel BuildRequires: hdf5-vol-daos-mpich-devel +%if (0%{?suse_version} > 0) +%global __debug_package 1 +%global _debuginfo_subpackages 0 +%debug_package +%endif + %description mpich mpiFileUtils provides both a library called libmfu and a suite of MPI-based tools to manage large datasets, which may vary from large directory trees @@ -214,12 +232,6 @@ for mpi in %{?mpi_list}; do %module_load $mpi make install -C $mpi DESTDIR=%{buildroot} rm %{buildroot}%{mpi_libdir}/${mpi}/%{mpi_lib_ext}/libmfu.a - -%if (0%{?suse_version} >= 1500) - strip --strip-unneeded %{buildroot}%{mpi_libdir}/${mpi}/%{mpi_lib_ext}/libmfu* - strip --strip-unneeded %{buildroot}%{mpi_libdir}/${mpi}/bin/* -%endif - find %{buildroot}%{mpi_libdir}/${mpi}/share/man -type f -exec gzip -9 -n {} \; module purge done From 06b213c2aa18c00296f8f77a0351db94011a15ef Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Wed, 22 Jun 2022 19:01:49 +0000 Subject: [PATCH 26/40] DAOS-10686 mpifileutils: remove redundant debug_package Signed-off-by: Lei Huang --- mpifileutils.spec | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/mpifileutils.spec b/mpifileutils.spec index acdae02..23af436 100644 --- a/mpifileutils.spec +++ b/mpifileutils.spec @@ -81,6 +81,12 @@ BuildRequires: libattr-devel BuildRequires: libfabric1 >= 1.12.0 %endif +%if (0%{?suse_version} > 0) +%global __debug_package 1 +%global _debuginfo_subpackages 0 +%debug_package +%endif + %description mpiFileUtils provides both a library called libmfu and a suite of MPI-based tools to manage large datasets, which may vary from large directory trees @@ -94,12 +100,6 @@ BuildRequires: dtcmp-openmpi-devel BuildRequires: libcircle-openmpi-devel BuildRequires: hdf5-vol-daos-openmpi-devel -%if (0%{?suse_version} > 0) -%global __debug_package 1 -%global _debuginfo_subpackages 0 -%debug_package -%endif - %description openmpi mpiFileUtils provides both a library called libmfu and a suite of MPI-based tools to manage large datasets, which may vary from large directory trees @@ -122,12 +122,6 @@ BuildRequires: dtcmp-openmpi3-devel BuildRequires: libcircle-openmpi3-devel BuildRequires: hdf5-vol-daos-openmpi3-devel -%if (0%{?suse_version} > 0) -%global __debug_package 1 -%global _debuginfo_subpackages 0 -%debug_package -%endif - %description openmpi3 mpiFileUtils provides both a library called libmfu and a suite of MPI-based tools to manage large datasets, which may vary from large directory trees @@ -163,12 +157,6 @@ BuildRequires: dtcmp-mpich-devel BuildRequires: libcircle-mpich-devel BuildRequires: hdf5-vol-daos-mpich-devel -%if (0%{?suse_version} > 0) -%global __debug_package 1 -%global _debuginfo_subpackages 0 -%debug_package -%endif - %description mpich mpiFileUtils provides both a library called libmfu and a suite of MPI-based tools to manage large datasets, which may vary from large directory trees From 2fd643a5550ff49a9d01f0517827b55e5e42f37f Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Thu, 23 Jun 2022 15:11:49 +0000 Subject: [PATCH 27/40] DAOS-10686 mpifileutils: suppress rpmlint warnings W: only-non-binary-in-usr-lib W: no-documentation Signed-off-by: Lei Huang --- mpifileutils.rpmlintrc | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 mpifileutils.rpmlintrc diff --git a/mpifileutils.rpmlintrc b/mpifileutils.rpmlintrc new file mode 100644 index 0000000..0936d7c --- /dev/null +++ b/mpifileutils.rpmlintrc @@ -0,0 +1,5 @@ +# This line is mandatory to access the configuration functions +from Config import * + +addFilter(r'mpifileutils-.+-devel\..+: W: only-non-binary-in-usr-lib') +addFilter(r'mpifileutils-.+: W: no-documentation') From d336bdcf49c4ade0a6f11288aa31af383fec6077 Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Thu, 23 Jun 2022 16:11:39 +0000 Subject: [PATCH 28/40] DAOS-10686 mpifileutils: empty commit to trigger rebuild Signed-off-by: Lei Huang From 81313666ea07f708f4de791b1d18a08a52ce6b32 Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Wed, 17 Aug 2022 04:58:42 +0000 Subject: [PATCH 29/40] DAOS-10686 mpifileutils: empty commit to trigger rebuild Signed-off-by: Lei Huang From 8f56795ac54c31da14249763f02c6a1a607d6bd2 Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Mon, 22 Aug 2022 14:27:10 +0000 Subject: [PATCH 30/40] DAOS-10686 mpifileutils: empty commit to trigger rebuild Signed-off-by: Lei Huang From 6588fb5ea33f99fdda4a09269e0e4cbbf3595978 Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Mon, 22 Aug 2022 18:43:06 +0000 Subject: [PATCH 31/40] DAOS-10686 mpifileutils: Remove CentOS 7 in Jenkins Signed-off-by: Lei Huang --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index afd864e..0456cae 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,5 +40,5 @@ // I.e. for testing library changes //@Library(value="pipeline-lib@your_branch") _ -packageBuildingPipeline(['distros' : ['centos7', 'el8', 'leap15']]) +packageBuildingPipeline(['distros' : ['el8', 'leap15']]) From cc146172c613ca1bb6c6c4d768f43953c53df6fc Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Wed, 24 Aug 2022 14:25:35 +0000 Subject: [PATCH 32/40] DAOS-10686 mpifileutils: Set test tags mpifileutils Signed-off-by: Lei Huang --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0456cae..b8e60e5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,5 +40,6 @@ // I.e. for testing library changes //@Library(value="pipeline-lib@your_branch") _ -packageBuildingPipeline(['distros' : ['el8', 'leap15']]) +packageBuildingPipeline(['distros' : ['el8', 'leap15'] + 'test-tag': 'mpifileutils']) From 378d45f3c125e61110def2e7cdb12516ca1cd4c6 Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Wed, 24 Aug 2022 14:29:53 +0000 Subject: [PATCH 33/40] DAOS-10686 mpifileutils: Set test tags mpifileutils Signed-off-by: Lei Huang --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index b8e60e5..8417d10 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,6 +40,6 @@ // I.e. for testing library changes //@Library(value="pipeline-lib@your_branch") _ -packageBuildingPipeline(['distros' : ['el8', 'leap15'] +packageBuildingPipeline(['distros' : ['el8', 'leap15'], 'test-tag': 'mpifileutils']) From 23eb5146e78263b69803714a73f4d5da1b852820 Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Wed, 24 Aug 2022 14:38:13 +0000 Subject: [PATCH 34/40] DAOS-10686 mpifileutils: update packaging Signed-off-by: Lei Huang --- packaging/Dockerfile.centos.7 | 4 ++-- packaging/Dockerfile.ubuntu.20.04 | 2 +- packaging/Makefile_packaging.mk | 21 +++++++++------------ packaging/rpm_chrootbuild | 2 +- 4 files changed, 13 insertions(+), 16 deletions(-) diff --git a/packaging/Dockerfile.centos.7 b/packaging/Dockerfile.centos.7 index 62ed5c6..cdfb7f6 100644 --- a/packaging/Dockerfile.centos.7 +++ b/packaging/Dockerfile.centos.7 @@ -16,8 +16,8 @@ ARG UID=1000 # Install basic tools RUN yum install -y epel-release -RUN yum install -y curl createrepo dnf git make mock python-srpm-macros \ - redhat-lsb-core rpm-build rpmdevtools rpmlint +RUN yum install -y mock make rpm-build curl createrepo rpmlint redhat-lsb-core \ + git python-srpm-macros dnf # Add build user (to keep rpmbuild happy) ENV USER build diff --git a/packaging/Dockerfile.ubuntu.20.04 b/packaging/Dockerfile.ubuntu.20.04 index f956145..0239a1b 100644 --- a/packaging/Dockerfile.ubuntu.20.04 +++ b/packaging/Dockerfile.ubuntu.20.04 @@ -17,7 +17,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ autoconf bash ca-certificates curl debhelper dh-make \ dpkg-dev dh-python doxygen gcc git git-buildpackage locales \ make patch pbuilder pkg-config python3-dev python3-distro \ - python3-distutils rpm scons wget + python3-distutils rpm scons wget cmake # rpmdevtools RUN echo "deb [trusted=yes] ${REPO_URL}${REPO_UBUNTU_20_04} focal main" > /etc/apt/sources.list.d/daos-stack-ubuntu-stable-local.list diff --git a/packaging/Makefile_packaging.mk b/packaging/Makefile_packaging.mk index 59ce452..8bd618e 100644 --- a/packaging/Makefile_packaging.mk +++ b/packaging/Makefile_packaging.mk @@ -98,11 +98,13 @@ define distro_map endef define install_repos - IFS='|' read -ra BASES <<< "$($(DISTRO_BASE)_LOCAL_REPOS)"; \ - for baseurl in "$${BASES[@]}"; do \ - baseurl="$${baseurl# *}"; \ - $(call install_repo,$$baseurl) \ - done + if [ "$(ID_LIKE)" = "debian" ]; then \ + IFS='|' read -ra BASES <<< "$($(DISTRO_BASE)_LOCAL_REPOS)"; \ + for baseurl in "$${BASES[@]}"; do \ + baseurl="$${baseurl# *}"; \ + $(call install_repo,$$baseurl) \ + done; \ + fi for repo in $($(DISTRO_BASE)_PR_REPOS) \ $(PR_REPOS) $(1); do \ branch="master"; \ @@ -345,13 +347,8 @@ ifeq ($(LOCAL_REPOS),true) DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO := $(subst reposi,artifac,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)) # $(DISTRO_BASE)_LOCAL_REPOS is a list separated by | because you cannot pass lists # of values with spaces as environment variables - ifeq ($(ID_LIKE), debian) - $(DISTRO_BASE)_LOCAL_REPOS := [trusted=yes] $(ARTIFACTORY_URL)$(subst stack,stack-daos,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)) - $(DISTRO_BASE)_LOCAL_REPOS += |[trusted=yes] $(ARTIFACTORY_URL)$(subst stack,stack-deps,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)) - else - $(DISTRO_BASE)_LOCAL_REPOS := $(ARTIFACTORY_URL)$(subst stack,stack-daos,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO))/ - $(DISTRO_BASE)_LOCAL_REPOS += |$(ARTIFACTORY_URL)$(subst stack,stack-deps,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO))/ - endif + $(DISTRO_BASE)_LOCAL_REPOS := [trusted=yes] $(ARTIFACTORY_URL)$(subst stack,stack-daos,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)) + $(DISTRO_BASE)_LOCAL_REPOS += |[trusted=yes] $(ARTIFACTORY_URL)$(subst stack,stack-deps,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)) endif #ifneq ($(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO),) endif # ifneq ($(ARTIFACTORY_URL),) endif # ifeq ($(LOCAL_REPOS),true) diff --git a/packaging/rpm_chrootbuild b/packaging/rpm_chrootbuild index 89ecdfc..03ab4c5 100755 --- a/packaging/rpm_chrootbuild +++ b/packaging/rpm_chrootbuild @@ -89,4 +89,4 @@ fi # shellcheck disable=SC2086 eval mock --configdir "$mock_config_dir" -r "${CHROOT_NAME}" \ ${repo_dels[*]} ${repo_adds[*]} --disablerepo=\*-debug* \ - "${releasever_opt[@]}" $MOCK_OPTIONS $RPM_BUILD_OPTIONS "$TARGET" -v + "${releasever_opt[@]}" $MOCK_OPTIONS $RPM_BUILD_OPTIONS "$TARGET" From b1725789bee3cac318c05eaa4ea840b98a24bd33 Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Wed, 24 Aug 2022 16:40:43 +0000 Subject: [PATCH 35/40] DAOS-10686 mpifileutils: packageBuildingPipelineDAOSTest to replace packageBuildingPipeline. Signed-off-by: Lei Huang --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8417d10..f930ffa 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,6 +40,6 @@ // I.e. for testing library changes //@Library(value="pipeline-lib@your_branch") _ -packageBuildingPipeline(['distros' : ['el8', 'leap15'], - 'test-tag': 'mpifileutils']) +packageBuildingPipelineDAOSTest(['distros' : ['el8', 'leap15'], + 'test-tag': 'mpifileutils']) From 0a14cfe9b1247861764076ff5194ac961b5133ab Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Thu, 25 Aug 2022 14:17:09 +0000 Subject: [PATCH 36/40] DAOS-10686 mpifileutils: Set test tags mfu Signed-off-by: Lei Huang --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index f930ffa..64d2256 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -41,5 +41,5 @@ //@Library(value="pipeline-lib@your_branch") _ packageBuildingPipelineDAOSTest(['distros' : ['el8', 'leap15'], - 'test-tag': 'mpifileutils']) + 'test-tag': 'mfu']) From 802e50fd6b18c352fa67601258eb9d4c1a90c247 Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Thu, 25 Aug 2022 14:19:07 +0000 Subject: [PATCH 37/40] DAOS-10686 mpifileutils: fixed format in mpifileutils.spec Signed-off-by: Lei Huang --- mpifileutils.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mpifileutils.spec b/mpifileutils.spec index 23af436..848adf3 100644 --- a/mpifileutils.spec +++ b/mpifileutils.spec @@ -267,7 +267,7 @@ done %endif %changelog -*Wed Jun 01 2022 Lei Huang - 0.11.1-3 +* Wed Jun 01 2022 Lei Huang - 0.11.1-3 - Update to patch 6230ed2 to support empty container label * Mon Mar 14 2022 Mohamad Chaarawi - 0.11.1-2 From 42e117c9cbb05220c81d5f39aab5a733fc92a8b0 Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Fri, 26 Aug 2022 14:16:11 +0000 Subject: [PATCH 38/40] DAOS-10686 mpifileutils: updated test-tag Signed-off-by: Lei Huang --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 64d2256..f74b471 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -41,5 +41,5 @@ //@Library(value="pipeline-lib@your_branch") _ packageBuildingPipelineDAOSTest(['distros' : ['el8', 'leap15'], - 'test-tag': 'mfu']) + 'test-tag': 'mpifileutils copy_procs dcp dsync dserialize']) From 88b95d0908d7f4812c932884bdb79fb9c0f7d68a Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Fri, 26 Aug 2022 14:21:34 +0000 Subject: [PATCH 39/40] DAOS-10686 mpifileutils: update submodule packaging Signed-off-by: Lei Huang --- packaging/Dockerfile.ubuntu.20.04 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/Dockerfile.ubuntu.20.04 b/packaging/Dockerfile.ubuntu.20.04 index 0239a1b..cf49ac8 100644 --- a/packaging/Dockerfile.ubuntu.20.04 +++ b/packaging/Dockerfile.ubuntu.20.04 @@ -17,7 +17,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ autoconf bash ca-certificates curl debhelper dh-make \ dpkg-dev dh-python doxygen gcc git git-buildpackage locales \ make patch pbuilder pkg-config python3-dev python3-distro \ - python3-distutils rpm scons wget cmake + python3-distutils rpm scons wget cmake valgrind # rpmdevtools RUN echo "deb [trusted=yes] ${REPO_URL}${REPO_UBUNTU_20_04} focal main" > /etc/apt/sources.list.d/daos-stack-ubuntu-stable-local.list From 84b95dcf151fc16fa17638da2b44dbf4c4c6e43b Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Thu, 15 Sep 2022 22:23:14 -0400 Subject: [PATCH 40/40] Use repo-files PR To see if that speeds up the build. Repo-files-PR: PR-8 Signed-off-by: Brian J. Murrell