diff --git a/0.11.1..356cb83d36a9243e30fe8a1bbc83e81fc46d2be2.patch b/0.11.1..356cb83d36a9243e30fe8a1bbc83e81fc46d2be2.patch deleted file mode 100644 index a1152fc..0000000 --- a/0.11.1..356cb83d36a9243e30fe8a1bbc83e81fc46d2be2.patch +++ /dev/null @@ -1,44 +0,0 @@ -diff --git a/src/common/mfu_daos.c b/src/common/mfu_daos.c -index 8f53133..b870978 100644 ---- a/src/common/mfu_daos.c -+++ b/src/common/mfu_daos.c -@@ -1182,7 +1182,9 @@ int daos_connect( - * If nothing is passed in for destination a uuid is always generated - * unless user passed one in, because destination container labels are - * not generated */ -- rc = daos_cont_open(*poh, da->dst_cont_uuid, DAOS_COO_RW, coh, &co_info, NULL); -+ char cont_str[130]; -+ uuid_unparse(da->dst_cont_uuid, cont_str); -+ rc = daos_cont_open(*poh, cont_str, DAOS_COO_RW, coh, &co_info, NULL); - } else { - rc = daos_cont_open(*poh, *cont, DAOS_COO_RW, coh, &co_info, NULL); - } -@@ -1273,14 +1275,12 @@ int daos_connect( - } - - /* try to open it again */ -- if (dst_cont_passed) { -- if (is_uuid) { -- rc = daos_cont_open(*poh, da->dst_cont_uuid, DAOS_COO_RW, coh, &co_info, NULL); -- } else { -- rc = daos_cont_open(*poh, *cont, DAOS_COO_RW, coh, &co_info, NULL); -- } -+ if (dst_cont_passed && !is_uuid) { -+ rc = daos_cont_open(*poh, *cont, DAOS_COO_RW, coh, &co_info, NULL); - } else { -- rc = daos_cont_open(*poh, da->dst_cont_uuid, DAOS_COO_RW, coh, &co_info, NULL); -+ char cont_str[130]; -+ uuid_unparse(da->dst_cont_uuid, cont_str); -+ rc = daos_cont_open(*poh, cont_str, DAOS_COO_RW, coh, &co_info, NULL); - } - if (rc != 0) { - MFU_LOG(MFU_LOG_ERR, "Failed to open container: "DF_RC, DP_RC(rc)); -@@ -5617,7 +5617,7 @@ int daos_cont_deserialize_connect(daos_args_t *daos_args, - } else { - uuid_unparse(daos_args->dst_cont_uuid, cont_str); - MFU_LOG(MFU_LOG_INFO, "Successfully created container %s", cont_str); -- rc = daos_cont_open(daos_args->src_poh, daos_args->dst_cont_uuid, -+ rc = daos_cont_open(daos_args->src_poh, cont_str, - DAOS_COO_RW, &daos_args->src_coh, &co_info, NULL); - } - if (rc != 0) { diff --git a/0.11.1..6230ed293c69f818aac5a47a0ce3665fc3fafba6.patch b/0.11.1..6230ed293c69f818aac5a47a0ce3665fc3fafba6.patch new file mode 100644 index 0000000..306941a --- /dev/null +++ b/0.11.1..6230ed293c69f818aac5a47a0ce3665fc3fafba6.patch @@ -0,0 +1,279 @@ +diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md +index d7c9a01..4ffcf16 100644 +--- a/CONTRIBUTING.md ++++ b/CONTRIBUTING.md +@@ -3,17 +3,12 @@ + *Thank you for taking the time to contribute!* + + ## Table of Contents +-[Resources](#resources) +- + [How To Contribute](#how-to-contribute) + * [Reporting an Issue & Feature Suggestions](#reporting-an-issue--feature-suggestions) + * [Pull Requests](#pull-requests) + * [License](#license) + * [Contributor's Declaration](#contributors-declaration) + +-## Resources +-[mpiFileUtils Google Group](https://groups.google.com/forum/#!forum/mpifileutils) +- + ## How To Contribute + + ### Reporting an Issue & Feature Suggestions +diff --git a/doc/rst/build.rst b/doc/rst/build.rst +index 047ba87..aedf67c 100644 +--- a/doc/rst/build.rst ++++ b/doc/rst/build.rst +@@ -34,7 +34,7 @@ To be certain of compatibility, it is recommended that one install libarchive-3. + mkdir install + installdir=`pwd`/install + +- wget https://github.com/libarchive/libarchive/releases/download/3.5.1/libarchive-3.5.1.tar.gz ++ wget https://github.com/libarchive/libarchive/releases/download/v3.5.1/libarchive-3.5.1.tar.gz + tar -zxf libarchive-3.5.1.tar.gz + cd libarchive-3.5.1 + ./configure --prefix=$installdir +@@ -135,7 +135,7 @@ which can be done with the following commands: + wget https://github.com/hpc/libcircle/releases/download/v0.3/libcircle-0.3.0.tar.gz + wget https://github.com/llnl/lwgrp/releases/download/v1.0.4/lwgrp-1.0.4.tar.gz + wget https://github.com/llnl/dtcmp/releases/download/v1.1.4/dtcmp-1.1.4.tar.gz +- wget https://github.com/libarchive/libarchive/releases/download/3.5.1/libarchive-3.5.1.tar.gz ++ wget https://github.com/libarchive/libarchive/releases/download/v3.5.1/libarchive-3.5.1.tar.gz + + tar -zxf libcircle-0.3.0.tar.gz + cd libcircle-0.3.0 +diff --git a/src/common/mfu_daos.c b/src/common/mfu_daos.c +index 8f53133..8981371 100644 +--- a/src/common/mfu_daos.c ++++ b/src/common/mfu_daos.c +@@ -1182,7 +1182,9 @@ int daos_connect( + * If nothing is passed in for destination a uuid is always generated + * unless user passed one in, because destination container labels are + * not generated */ +- rc = daos_cont_open(*poh, da->dst_cont_uuid, DAOS_COO_RW, coh, &co_info, NULL); ++ char cont_str[130]; ++ uuid_unparse(da->dst_cont_uuid, cont_str); ++ rc = daos_cont_open(*poh, cont_str, DAOS_COO_RW, coh, &co_info, NULL); + } else { + rc = daos_cont_open(*poh, *cont, DAOS_COO_RW, coh, &co_info, NULL); + } +@@ -1273,14 +1275,12 @@ int daos_connect( + } + + /* try to open it again */ +- if (dst_cont_passed) { +- if (is_uuid) { +- rc = daos_cont_open(*poh, da->dst_cont_uuid, DAOS_COO_RW, coh, &co_info, NULL); +- } else { +- rc = daos_cont_open(*poh, *cont, DAOS_COO_RW, coh, &co_info, NULL); +- } ++ if (dst_cont_passed && !is_uuid) { ++ rc = daos_cont_open(*poh, *cont, DAOS_COO_RW, coh, &co_info, NULL); + } else { +- rc = daos_cont_open(*poh, da->dst_cont_uuid, DAOS_COO_RW, coh, &co_info, NULL); ++ char cont_str[130]; ++ uuid_unparse(da->dst_cont_uuid, cont_str); ++ rc = daos_cont_open(*poh, cont_str, DAOS_COO_RW, coh, &co_info, NULL); + } + if (rc != 0) { + MFU_LOG(MFU_LOG_ERR, "Failed to open container: "DF_RC, DP_RC(rc)); +@@ -3857,7 +3857,7 @@ static int cont_serialize_prop_str(struct hdf5_args* hdf5, + hid_t attr_dspace; + hid_t usr_attr; + +- if (entry == NULL || entry->dpe_str == NULL) { ++ if (entry == NULL) { + MFU_LOG(MFU_LOG_ERR, "Property %s not found", prop_str); + rc = 1; + goto out; +@@ -3870,7 +3870,7 @@ static int cont_serialize_prop_str(struct hdf5_args* hdf5, + rc = 1; + goto out; + } +- status = H5Tset_size(attr_dtype, strlen(entry->dpe_str) + 1); ++ status = H5Tset_size(attr_dtype, (entry->dpe_str ? strlen(entry->dpe_str) : 0) + 1); + if (status < 0) { + MFU_LOG(MFU_LOG_ERR, "failed to set dtype size"); + rc = 1; +@@ -3895,7 +3895,7 @@ static int cont_serialize_prop_str(struct hdf5_args* hdf5, + rc = 1; + goto out; + } +- status = H5Awrite(usr_attr, attr_dtype, entry->dpe_str); ++ status = H5Awrite(usr_attr, attr_dtype, entry->dpe_str ? entry->dpe_str : ""); + if (status < 0) { + MFU_LOG(MFU_LOG_ERR, "failed to write attribute"); + rc = 1; +@@ -5394,22 +5394,23 @@ int cont_deserialize_all_props(struct hdf5_args *hdf5, + goto out; + } + +- rc = daos_cont_open(poh, label_entry->dpe_str, DAOS_COO_RW, &coh, &cont_info, NULL); +- if (rc == -DER_NONEXIST) { +- /* doesn't exist so ok to deserialize this container label */ +- deserialize_label = true; +- } else if (rc != 0) { +- MFU_LOG(MFU_LOG_ERR, "daos_cont_open failed: "DF_RC, DP_RC(rc)); +- goto out; +- } else { +- /* if this succeeds then label already exists, close container after +- * checking */ +- rc = daos_cont_close(coh, NULL); +- if (rc != 0) { ++ if (label_entry->dpe_str[0]) { ++ rc = daos_cont_open(poh, label_entry->dpe_str, DAOS_COO_RW, &coh, &cont_info, NULL); ++ if (rc == -DER_NONEXIST) { ++ /* doesn't exist so ok to deserialize this container label */ ++ deserialize_label = true; ++ } else if (rc != 0) { ++ MFU_LOG(MFU_LOG_ERR, "daos_cont_open failed: "DF_RC, DP_RC(rc)); + goto out; ++ } else { ++ /* if this succeeds then label already exists, close container after ++ * checking */ ++ rc = daos_cont_close(coh, NULL); ++ if (rc != 0) { ++ goto out; ++ } + } + } +- + if (deserialize_label) { + num_props++; + } +@@ -5617,7 +5618,7 @@ int daos_cont_deserialize_connect(daos_args_t *daos_args, + } else { + uuid_unparse(daos_args->dst_cont_uuid, cont_str); + MFU_LOG(MFU_LOG_INFO, "Successfully created container %s", cont_str); +- rc = daos_cont_open(daos_args->src_poh, daos_args->dst_cont_uuid, ++ rc = daos_cont_open(daos_args->src_poh, cont_str, + DAOS_COO_RW, &daos_args->src_coh, &co_info, NULL); + } + if (rc != 0) { +diff --git a/src/common/mfu_flist_copy.c b/src/common/mfu_flist_copy.c +index f5acaab..0e9c78f 100644 +--- a/src/common/mfu_flist_copy.c ++++ b/src/common/mfu_flist_copy.c +@@ -1680,12 +1680,20 @@ static int mfu_copy_file_normal( + /* If we're using O_DIRECT, deal with short reads. + * Retry with same buffer and offset since those must + * be aligned at block boundaries. */ ++ int retries = 0; + while (copy_opts->direct && /* using O_DIRECT */ + bytes_read > 0 && /* read was not an error or eof */ + bytes_read < left_to_read && /* shorter than requested */ + (off + bytes_read) < file_size) /* not at end of file */ + { +- /* TODO: probably should retry a limited number of times then abort */ ++ /* try the read a limited number of times then given up with error */ ++ retries++; ++ if (retries == 5) { ++ MFU_LOG(MFU_LOG_ERR, "Source file `%s' exceeded short read limit, maybe shorter than expected size of %llu bytes", ++ src, file_size); ++ return -1; ++ } ++ + bytes_read = mfu_file_pread(src, buf, left_to_read, off, mfu_src_file); + } + +@@ -1698,8 +1706,8 @@ static int mfu_copy_file_normal( + + /* check for early EOF */ + if (bytes_read == 0) { +- MFU_LOG(MFU_LOG_ERR, "Source file `%s' shorter than expected %llu (errno=%d %s)", +- src, file_size, errno, strerror(errno)); ++ MFU_LOG(MFU_LOG_ERR, "Source file `%s' shorter than expected size of %llu bytes", ++ src, file_size); + return -1; + } + +diff --git a/src/daos-gen/daos-gen.c b/src/daos-gen/daos-gen.c +index d96373a..46ee44d 100644 +--- a/src/daos-gen/daos-gen.c ++++ b/src/daos-gen/daos-gen.c +@@ -18,6 +18,14 @@ + #define BUFLEN 80 + #define UUID_STR_LEN 129 + ++#if defined(DAOS_API_VERSION_MAJOR) && defined(DAOS_API_VERSION_MINOR) ++#define CHECK_DAOS_API_VERSION(major, minor) \ ++ ((DAOS_API_VERSION_MAJOR > (major)) \ ++ || (DAOS_API_VERSION_MAJOR == (major) && DAOS_API_VERSION_MINOR >= (minor))) ++#else ++#define CHECK_DAOS_API_VERSION(major, minor) 0 ++#endif ++ + static uint32_t obj_id_gen = 1; + + /** Print a usage message. */ +@@ -176,13 +184,18 @@ int main(int argc, char** argv) + char key[keys_per_object]; + int i,j; + char uuid_str[UUID_STR_LEN]; ++ uuid_t cont_uuid; ++ ++#if CHECK_DAOS_API_VERSION(2, 0) ++ enum daos_otype_t otype; ++ otype = DAOS_OT_KV_HASHED; ++#else + daos_ofeat_t ofeats; ++ ofeats = DAOS_OF_KV_FLAT; ++#endif + +- ofeats = DAOS_OF_DKEY_UINT64 | DAOS_OF_KV_FLAT | DAOS_OF_KV_FLAT; + /* connect to pool/cont then broadcast to rest of ranks */ + if (rank == 0) { +- /* generate container UUID */ +- uuid_generate(daos_args->src_cont); + daos_pool_info_t pool_info = {0}; + daos_cont_info_t co_info = {0}; + #if DAOS_API_VERSION_MAJOR < 1 +@@ -199,14 +212,27 @@ int main(int argc, char** argv) + } + + /* create cont and open */ ++#if CHECK_DAOS_API_VERSION(2, 0) ++ rc = daos_cont_create(daos_args->src_poh, &cont_uuid, NULL, NULL); ++#else ++ /* generate container UUID */ ++ uuid_generate(daos_args->src_cont); ++ uuid_copy(cont_uuid, daos_args->src_cont); + rc = daos_cont_create(daos_args->src_poh, daos_args->src_cont, NULL, NULL); ++#endif + if (rc != 0) { + MFU_LOG(MFU_LOG_ERR, "Failed to create cont: "DF_RC, DP_RC(rc)); + rc = 1; + goto err_cont; + } ++ uuid_unparse(cont_uuid, uuid_str); ++#if CHECK_DAOS_API_VERSION(2, 0) ++ rc = daos_cont_open(daos_args->src_poh, uuid_str, ++ DAOS_COO_RW, &daos_args->src_coh, &co_info, NULL); ++#else + rc = daos_cont_open(daos_args->src_poh, daos_args->src_cont, + DAOS_COO_RW, &daos_args->src_coh, &co_info, NULL); ++#endif + if (rc != 0) { + MFU_LOG(MFU_LOG_ERR, "Failed to open container: "DF_RC, DP_RC(rc)); + rc = 1; +@@ -222,8 +248,11 @@ int main(int argc, char** argv) + memset(buf, 'A', BUFLEN); + for (i = 0; i < num_objects; i++) { + oid[i] = dts_oid_gen(0); +- ++#if CHECK_DAOS_API_VERSION(2, 0) ++ rc = daos_obj_generate_oid(daos_args->src_coh, &oid[i], otype, OC_RP_XSF, 0, 0); ++#else + rc = daos_obj_generate_oid(daos_args->src_coh, &oid[i], ofeats, OC_RP_XSF, 0, 0); ++#endif + if (rc != 0) { + MFU_LOG(MFU_LOG_ERR, "Failed to generate oid: "DF_RC, DP_RC(rc)); + rc = 1; +@@ -261,7 +290,6 @@ int main(int argc, char** argv) + * generates same amount */ + if (rank == 0) { + int total_num_objects = size * num_objects; +- uuid_unparse(daos_args->src_cont, uuid_str); + printf("Container UUID: %s\n\ttotal objects:%d\n" + "\tkeys per object:%d\n", uuid_str, total_num_objects, keys_per_object); + } diff --git a/Jenkinsfile b/Jenkinsfile index 8ecb222..f74b471 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,5 +40,6 @@ // I.e. for testing library changes //@Library(value="pipeline-lib@your_branch") _ -packageBuildingPipeline(['distros' : ['centos7', 'centos8', 'leap15']]) +packageBuildingPipelineDAOSTest(['distros' : ['el8', 'leap15'], + 'test-tag': 'mpifileutils copy_procs dcp dsync dserialize']) diff --git a/Makefile b/Makefile index e191cc9..a6f53c7 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ NAME := mpifileutils SRC_EXT := gz REPO_NAME := mpifileutils-pkg -PKG_GIT_COMMIT := 356cb83d36a9243e30fe8a1bbc83e81fc46d2be2 +PKG_GIT_COMMIT := 6230ed293c69f818aac5a47a0ce3665fc3fafba6 GITHUB_PROJECT := hpc/$(NAME) TEST_PACKAGES := $(NAME)-mpich-devel $(NAME)-openmpi3-devel diff --git a/mpifileutils.rpmlintrc b/mpifileutils.rpmlintrc new file mode 100644 index 0000000..0936d7c --- /dev/null +++ b/mpifileutils.rpmlintrc @@ -0,0 +1,5 @@ +# This line is mandatory to access the configuration functions +from Config import * + +addFilter(r'mpifileutils-.+-devel\..+: W: only-non-binary-in-usr-lib') +addFilter(r'mpifileutils-.+: W: no-documentation') diff --git a/mpifileutils.spec b/mpifileutils.spec index 0f7e60f..848adf3 100644 --- a/mpifileutils.spec +++ b/mpifileutils.spec @@ -50,11 +50,11 @@ Name: mpifileutils Version: 0.11.1 -Release: 2%{?commit:.g%{shortcommit}}%{?dist} -Summary: File utilities designed for scalability and performance. +Release: 3%{?commit:.g%{shortcommit}}%{?dist} +Summary: File utilities designed for scalability and performance Group: System Environment/Libraries -License: Copyright and BSD License +License: BSD URL: https://hpc.github.io/mpifileutils Source: https://github.com/hpc/%{name}/archive/v%{version}.tar.gz %if "%{?commit}" != "" @@ -81,50 +81,63 @@ BuildRequires: libattr-devel BuildRequires: libfabric1 >= 1.12.0 %endif +%if (0%{?suse_version} > 0) +%global __debug_package 1 +%global _debuginfo_subpackages 0 +%debug_package +%endif + %description -File utilities designed for scalability and performance. +mpiFileUtils provides both a library called libmfu and a suite of MPI-based +tools to manage large datasets, which may vary from large directory trees +to large files. %if %{with_openmpi} %package openmpi -Summary: File utilities designed for scalability and performance. +Summary: File utilities designed for scalability and performance BuildRequires: openmpi-devel BuildRequires: dtcmp-openmpi-devel BuildRequires: libcircle-openmpi-devel BuildRequires: hdf5-vol-daos-openmpi-devel %description openmpi -File utilities designed for scalability and performance. - +mpiFileUtils provides both a library called libmfu and a suite of MPI-based +tools to manage large datasets, which may vary from large directory trees +to large files. %package openmpi-devel -Summary: File utilities designed for scalability and performance. +Summary: File utilities designed for scalability and performance Requires: %{name}-openmpi%{_isa} = %version-%release %description openmpi-devel -Development files for %{name}-openmpi. +Development files for %{name}-openmpi which provides file utilities +designed for scalability and performance. %endif %if %{with_openmpi3} %package openmpi3 -Summary: File utilities designed for scalability and performance. +Summary: File utilities designed for scalability and performance BuildRequires: openmpi3-devel BuildRequires: dtcmp-openmpi3-devel BuildRequires: libcircle-openmpi3-devel BuildRequires: hdf5-vol-daos-openmpi3-devel %description openmpi3 -File utilities designed for scalability and performance. +mpiFileUtils provides both a library called libmfu and a suite of MPI-based +tools to manage large datasets, which may vary from large directory trees +to large files. %if (0%{?suse_version} >= 1500) %package -n libmfu0-openmpi3 -Summary: File utilities designed for scalability and performance. +Summary: File utilities designed for scalability and performance %description -n libmfu0-openmpi3 -Shared libraries for %{name}-openmpi3. +Shared libraries for %{name}-openmpi3, file utilities designed for +scalability and performance. %endif %package openmpi3-devel -Summary: File utilities designed for scalability and performance. +Summary: File utilities designed for scalability and performance %if (0%{?suse_version} >= 1500) Requires: libmfu0-openmpi3%{_isa} = %version-%release %else @@ -132,30 +145,34 @@ Requires: %{name}-openmpi3%{_isa} = %version-%release %endif %description openmpi3-devel -Development files for %{name}-openmpi3. +Development files for %{name}-openmpi3 which provides file utilities +designed for scalability and performance. %endif %if %{with_mpich} %package mpich -Summary: File utilities designed for scalability and performance. +Summary: File utilities designed for scalability and performance BuildRequires: mpich-devel BuildRequires: dtcmp-mpich-devel BuildRequires: libcircle-mpich-devel BuildRequires: hdf5-vol-daos-mpich-devel %description mpich -File utilities designed for scalability and performance. +mpiFileUtils provides both a library called libmfu and a suite of MPI-based +tools to manage large datasets, which may vary from large directory trees +to large files. %if (0%{?suse_version} >= 1500) %package -n libmfu0-mpich -Summary: File utilities designed for scalability and performance. +Summary: File utilities designed for scalability and performance %description -n libmfu0-mpich -Shared libraries for %{name}-mpich. +Shared libraries for %{name}-mpich, file utilities designed for +scalability and performance. %endif %package mpich-devel -Summary: File utilities designed for scalability and performance. +Summary: File utilities designed for scalability and performance %if (0%{?suse_version} >= 1500) Requires: libmfu0-mpich%{_isa} = %version-%release %else @@ -163,7 +180,8 @@ Requires: %{name}-mpich%{_isa} = %version-%release %endif %description mpich-devel -Development files for %{name}-mpich. +Development files for %{name}-mpich which provides file utilities +designed for scalability and performance. %endif @@ -171,37 +189,39 @@ Development files for %{name}-mpich. %autosetup -p1 %build +export CFLAGS="%{optflags} -fPIC -pie" +export CXXFLAGS="%{optflags} -fPIC -pie" for mpi in %{?mpi_list}; do - mkdir $mpi - pushd $mpi - %module_load $mpi - %cmake ../ -DCMAKE_C_FLAGS="${RPM_OPT_FLAGS}" \ - -DCMAKE_CXX_FLAGS="${RPM_OPT_FLAGS}" \ - -DENABLE_DAOS=ON \ - -DENABLE_LIBARCHIVE=OFF \ - -DENABLE_HDF5=ON \ - -DDTCMP_INCLUDE_DIRS=%{mpi_includedir}/$mpi%{mpi_include_ext} \ - -DDTCMP_LIBRARIES=%{mpi_libdir}/$mpi/%{mpi_lib_ext}/libdtcmp.so \ - -DLibCircle_INCLUDE_DIRS=%{mpi_includedir}/$mpi%{mpi_include_ext} \ - -DLibCircle_LIBRARIES=%{mpi_libdir}/$mpi/%{mpi_lib_ext}/libcircle.so \ - -DHDF5_INCLUDE_DIRS=%{mpi_includedir}/$mpi%{mpi_include_ext} \ - -DHDF5_LIBRARIES=%{mpi_libdir}/$mpi/%{mpi_lib_ext}/libhdf5.so \ - -DWITH_DAOS_PREFIX=/usr \ - -DCMAKE_INSTALL_INCLUDEDIR=%{mpi_includedir}/$mpi%{mpi_include_ext} \ - -DCMAKE_INSTALL_PREFIX=%{mpi_libdir}/$mpi \ - -DCMAKE_INSTALL_LIBDIR=%{mpi_lib_ext} - - make - module purge - popd + mkdir $mpi + pushd $mpi + %module_load $mpi + %cmake ../ -DENABLE_DAOS=ON \ + -DENABLE_LIBARCHIVE=OFF \ + -DENABLE_HDF5=ON \ + -DDTCMP_INCLUDE_DIRS=%{mpi_includedir}/$mpi%{mpi_include_ext} \ + -DDTCMP_LIBRARIES=%{mpi_libdir}/$mpi/%{mpi_lib_ext}/libdtcmp.so \ + -DLibCircle_INCLUDE_DIRS=%{mpi_includedir}/$mpi%{mpi_include_ext} \ + -DLibCircle_LIBRARIES=%{mpi_libdir}/$mpi/%{mpi_lib_ext}/libcircle.so \ + -DHDF5_INCLUDE_DIRS=%{mpi_includedir}/$mpi%{mpi_include_ext} \ + -DHDF5_LIBRARIES=%{mpi_libdir}/$mpi/%{mpi_lib_ext}/libhdf5.so \ + -DWITH_DAOS_PREFIX=/usr \ + -DCMAKE_INSTALL_INCLUDEDIR=%{mpi_includedir}/$mpi%{mpi_include_ext} \ + -DCMAKE_INSTALL_PREFIX=%{mpi_libdir}/$mpi \ + -DCMAKE_INSTALL_LIBDIR=%{mpi_lib_ext} + + make + module purge + popd done %install rm -rf %{buildroot} for mpi in %{?mpi_list}; do - %module_load $mpi - make install -C $mpi DESTDIR=%{buildroot} - module purge + %module_load $mpi + make install -C $mpi DESTDIR=%{buildroot} + rm %{buildroot}%{mpi_libdir}/${mpi}/%{mpi_lib_ext}/libmfu.a + find %{buildroot}%{mpi_libdir}/${mpi}/share/man -type f -exec gzip -9 -n {} \; + module purge done %if %{with_openmpi} @@ -214,7 +234,6 @@ done %files openmpi-devel %{mpi_includedir}/openmpi%{mpi_include_ext}/* %{mpi_libdir}/openmpi/%{mpi_lib_ext}/lib*.so -%{mpi_libdir}/openmpi/%{mpi_lib_ext}/lib*.a %endif %if %{with_openmpi3} @@ -230,7 +249,6 @@ done %files openmpi3-devel %{mpi_includedir}/openmpi3%{mpi_include_ext}/* %{mpi_libdir}/openmpi3/%{mpi_lib_ext}/lib*.so -%{mpi_libdir}/openmpi3/%{mpi_lib_ext}/lib*.a %endif %if %{with_mpich} @@ -246,10 +264,12 @@ done %files mpich-devel %{mpi_includedir}/mpich%{mpi_include_ext}/* %{mpi_libdir}/mpich/%{mpi_lib_ext}/lib*.so -%{mpi_libdir}/mpich/%{mpi_lib_ext}/lib*.a %endif %changelog +* Wed Jun 01 2022 Lei Huang - 0.11.1-3 +- Update to patch 6230ed2 to support empty container label + * Mon Mar 14 2022 Mohamad Chaarawi - 0.11.1-2 - Update to build with HDF5 1.13.1 diff --git a/packaging/Dockerfile.centos.7 b/packaging/Dockerfile.centos.7 index 55510f7..cdfb7f6 100644 --- a/packaging/Dockerfile.centos.7 +++ b/packaging/Dockerfile.centos.7 @@ -1,12 +1,12 @@ # -# Copyright 2018-2021, Intel Corporation +# Copyright 2018-2022, Intel Corporation # # 'recipe' for Docker to build an RPM # # Pull base image FROM centos:7 -MAINTAINER daos-stack +LABEL maintainer="daos@daos.groups.io" # use same UID as host and default value of 1000 if not specified ARG UID=1000 diff --git a/packaging/Dockerfile.mockbuild b/packaging/Dockerfile.mockbuild index 89d494f..25ff965 100644 --- a/packaging/Dockerfile.mockbuild +++ b/packaging/Dockerfile.mockbuild @@ -1,20 +1,30 @@ # -# Copyright 2018-2022, Intel Corporation +# Copyright 2018-2022 Intel Corporation # # 'recipe' for Docker to build an RPM # # Pull base image -FROM fedora:35 -LABEL maintainer="daos@daos.groups.io>" +FROM fedora:latest +LABEL maintainer="daos@daos.groups.io" -# use same UID as host and default value of 1000 if not specified -ARG UID=1000 +# Use local repo server if present +ARG REPO_FILE_URL +RUN if [ -n "$REPO_FILE_URL" ]; then \ + cd /etc/yum.repos.d/ && \ + curl -f -o daos_ci-fedora-artifactory.repo.tmp \ + "$REPO_FILE_URL"daos_ci-fedora-artifactory.repo && \ + rm -f *.repo && \ + mv daos_ci-fedora-artifactory.repo{.tmp,}; \ + fi # Install basic tools -RUN dnf -y install mock make \ - rpm-build curl createrepo rpmlint redhat-lsb-core git \ - python-srpm-macros rpmdevtools mock-core-configs\ \<\ 37.1 +RUN dnf -y install mock make \ + rpm-build createrepo rpmlint redhat-lsb-core git \ + python-srpm-macros rpmdevtools + +# use same UID as host and default value of 1000 if not specified +ARG UID=1000 # Add build user (to keep rpmbuild happy) ENV USER build @@ -24,10 +34,14 @@ RUN echo "$USER:$PASSWD" | chpasswd # add the user to the mock group so it can run mock RUN usermod -a -G mock $USER -# mock in Docker needs to use the old-chroot option -RUN grep use_nspawn /etc/mock/site-defaults.cfg || \ - echo "config_opts['use_nspawn'] = False" >> /etc/mock/site-defaults.cfg - ARG CACHEBUST -RUN dnf -y upgrade --exclude mock-core-configs && \ +RUN dnf -y upgrade && \ dnf clean all + +# Monkey-patch rpmlint until a new release is made with +# https://github.com/rpm-software-management/rpmlint/pull/795 in it +COPY packaging/rpmlint--ignore-unused-rpmlintrc.patch . +RUN (cd $(python3 -c 'import site; print(site.getsitepackages()[-1])') && \ + patch -p1 && \ + rm -f rpmlint/__pycache__/{cli,lint}.*.pyc) < rpmlint--ignore-unused-rpmlintrc.patch && \ + rm -f rpmlint--ignore-unused-rpmlintrc.patch diff --git a/packaging/Dockerfile.ubuntu.20.04 b/packaging/Dockerfile.ubuntu.20.04 index f956145..cf49ac8 100644 --- a/packaging/Dockerfile.ubuntu.20.04 +++ b/packaging/Dockerfile.ubuntu.20.04 @@ -17,7 +17,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ autoconf bash ca-certificates curl debhelper dh-make \ dpkg-dev dh-python doxygen gcc git git-buildpackage locales \ make patch pbuilder pkg-config python3-dev python3-distro \ - python3-distutils rpm scons wget + python3-distutils rpm scons wget cmake valgrind # rpmdevtools RUN echo "deb [trusted=yes] ${REPO_URL}${REPO_UBUNTU_20_04} focal main" > /etc/apt/sources.list.d/daos-stack-ubuntu-stable-local.list diff --git a/packaging/Makefile_distro_vars.mk b/packaging/Makefile_distro_vars.mk index 44006e0..3d29ffa 100644 --- a/packaging/Makefile_distro_vars.mk +++ b/packaging/Makefile_distro_vars.mk @@ -26,7 +26,7 @@ SPECTOOL := spectool # DISTRO_ID (i.e. el7) # DISTRO_BASE (i.e. EL_7) # from the CHROOT_NAME -ifeq ($(CHROOT_NAME),epel-7-x86_64) +ifeq ($(patsubst %epel-7-x86_64,,$(lastword $(subst +, ,$(CHROOT_NAME)))),) DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) VERSION_ID := 7 DISTRO_ID := el7 @@ -35,14 +35,17 @@ DISTRO_VERSION ?= $(VERSION_ID) ORIG_TARGET_VER := 7 SED_EXPR := 1s/$(DIST)//p endif -ifeq ($(CHROOT_NAME),epel-8-x86_64) -DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) -VERSION_ID := 8 -DISTRO_ID := el8 -DISTRO_BASE := EL_8 -DISTRO_VERSION ?= $(VERSION_ID) -ORIG_TARGET_VER := 8 -SED_EXPR := 1s/$(DIST)//p +ifeq ($(patsubst %epel-8-x86_64,,$(lastword $(subst +, ,$(CHROOT_NAME)))),) +DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) +VERSION_ID := 8.5 +DISTRO_ID := el8 +DISTRO_BASE := EL_8 +ifneq ($(DISTRO_VERSION_EL8),) +override DISTRO_VERSION := $(DISTRO_VERSION_EL8) +endif +DISTRO_VERSION ?= $(VERSION_ID) +ORIG_TARGET_VER := 8.5 +SED_EXPR := 1s/$(DIST)//p endif ifeq ($(CHROOT_NAME),opensuse-leap-15.2-x86_64) VERSION_ID := 15.2 @@ -62,6 +65,18 @@ SED_EXPR := 1p endif endif ifeq ($(ID),centos) +ID = el +endif +ifeq ($(ID),rocky) +ID = el +endif +ifeq ($(ID),almalinux) +ID = el +endif +ifeq ($(ID),rhel) +ID = el +endif +ifeq ($(ID),el) DISTRO_ID := el$(VERSION_ID) DISTRO_BASE := $(basename EL_$(VERSION_ID)) DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) diff --git a/packaging/Makefile_packaging.mk b/packaging/Makefile_packaging.mk index b59782e..8bd618e 100644 --- a/packaging/Makefile_packaging.mk +++ b/packaging/Makefile_packaging.mk @@ -10,7 +10,7 @@ SHELL=/bin/bash -include Makefile.local # default to Leap 15 distro for chrootbuild -CHROOT_NAME ?= opensuse-leap-15.2-x86_64 +CHROOT_NAME ?= opensuse-leap-15.3-x86_64 include packaging/Makefile_distro_vars.mk ifeq ($(DEB_NAME),) @@ -19,7 +19,9 @@ endif CALLING_MAKEFILE := $(word 1, $(MAKEFILE_LIST)) -TOPDIR ?= $(CURDIR) +# this Makefile should always be executed from it's own dir +TOPDIR ?= $(abspath $(dir $(firstword $(MAKEFILE_LIST)))) + BUILD_PREFIX ?= . DOT := . @@ -28,11 +30,6 @@ RPM_BUILD_OPTIONS += $(EXTERNAL_RPM_BUILD_OPTIONS) # some defaults the caller can override PACKAGING_CHECK_DIR ?= ../packaging LOCAL_REPOS ?= true -ifeq ($(ID_LIKE),debian) -DAOS_REPO_TYPE ?= LOCAL -else -DAOS_REPO_TYPE ?= STABLE -endif TEST_PACKAGES ?= ${NAME} # unfortunately we cannot always name the repo the same as the project @@ -43,6 +40,7 @@ LEAP_15_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-rep EL_7_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el7: *\(.*\)/\1/p') EL_8_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el8: *\(.*\)/\1/p') UBUNTU_20_04_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-ubuntu20: *\(.*\)/\1/p') +REPO_FILES_PR ?= $(shell git show -s --format=%B | sed -ne 's/^Repo-files-PR: *\(.*\)/\1/p') ifneq ($(PKG_GIT_COMMIT),) ifeq ($(GITHUB_PROJECT),) @@ -86,7 +84,7 @@ define distro_map case $(DISTRO_ID) in \ el7) distro="centos7" \ ;; \ - el8) distro="centos8" \ + el8) distro="el8" \ ;; \ sle12.3) distro="sles12.3" \ ;; \ @@ -100,27 +98,29 @@ define distro_map endef define install_repos - IFS='|' read -ra BASES <<< "$($(DISTRO_BASE)_LOCAL_REPOS)"; \ - for baseurl in "$${BASES[@]}"; do \ - baseurl="$${baseurl# *}"; \ - $(call install_repo,$$baseurl); \ - done - for repo in $($(DISTRO_BASE)_PR_REPOS) \ - $(PR_REPOS) $(1); do \ - branch="master"; \ - build_number="lastSuccessfulBuild"; \ - if [[ $$repo = *@* ]]; then \ - branch="$${repo#*@}"; \ - repo="$${repo%@*}"; \ - if [[ $$branch = *:* ]]; then \ - build_number="$${branch#*:}"; \ - branch="$${branch%:*}"; \ - fi; \ - fi; \ - $(call distro_map) \ + if [ "$(ID_LIKE)" = "debian" ]; then \ + IFS='|' read -ra BASES <<< "$($(DISTRO_BASE)_LOCAL_REPOS)"; \ + for baseurl in "$${BASES[@]}"; do \ + baseurl="$${baseurl# *}"; \ + $(call install_repo,$$baseurl) \ + done; \ + fi + for repo in $($(DISTRO_BASE)_PR_REPOS) \ + $(PR_REPOS) $(1); do \ + branch="master"; \ + build_number="lastSuccessfulBuild"; \ + if [[ $$repo = *@* ]]; then \ + branch="$${repo#*@}"; \ + repo="$${repo%@*}"; \ + if [[ $$branch = *:* ]]; then \ + build_number="$${branch#*:}"; \ + branch="$${branch%:*}"; \ + fi; \ + fi; \ + $(call distro_map) \ baseurl=$${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$$repo/job/$$branch/; \ - baseurl+=$$build_number/artifact/artifacts/$$distro/; \ - $(call install_repo,$$baseurl); \ + baseurl+=$$build_number/artifact/artifacts/$$distro/; \ + $(call install_repo,$$baseurl) \ done endef @@ -154,6 +154,11 @@ ifeq ($(DL_NAME),) DL_NAME = $(NAME) endif +# this actually should replace all of the downloaders below +$(notdir $(SOURCE)): $(SPEC) $(CALLING_MAKEFILE) + # TODO: need to clean up old ones + $(SPECTOOL) -g $(SPEC) + $(DL_NAME)$(DL_VERSION).linux-amd64.tar.$(SRC_EXT): $(SPEC) $(CALLING_MAKEFILE) rm -f ./$(DL_NAME)*.tar{gz,bz*,xz} $(SPECTOOL) -g $(SPEC) @@ -334,60 +339,29 @@ patch: endif # *_LOCAL_* repos are locally built packages. -# *_GROUP_* repos are a local mirror of a group of upstream repos. -# *_GROUP_* repos may not supply a repomd.xml.key. ifeq ($(LOCAL_REPOS),true) - ifneq ($(REPOSITORY_URL),) - # group repos are not working in Nexus so we hack in the group members directly below - #ifneq ($(DAOS_STACK_$(DISTRO_BASE)_DOCKER_$(DAOS_REPO_TYPE)_REPO),) - #DISTRO_REPOS = $(DAOS_STACK_$(DISTRO_BASE)_DOCKER_$(DAOS_REPO_TYPE)_REPO) - #$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_DOCKER_$(DAOS_REPO_TYPE)_REPO)/ - #endif - ifneq ($(DAOS_STACK_$(DISTRO_BASE)_$(DAOS_REPO_TYPE)_REPO),) - ifeq ($(ID_LIKE),debian) - # $(DISTRO_BASE)_LOCAL_REPOS is a list separated by | because you cannot pass lists - # of values with spaces as environment variables - $(DISTRO_BASE)_LOCAL_REPOS := [trusted=yes] - else - $(DISTRO_BASE)_LOCAL_REPOS := $(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO) - DISTRO_REPOS = disabled # any non-empty value here works and is not used beyond testing if the value is empty or not - endif # ifeq ($(ID_LIKE),debian) - ifeq ($(DISTRO_BASE), EL_8) - # hack to use 8.3 non-group repos on EL_8 - $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(subst $(ORIG_TARGET_VER),$(DISTRO_VERSION),$(REPOSITORY_URL)repository/centos-8.3-base-x86_64-proxy|$(REPOSITORY_URL)repository/centos-8.3-extras-x86_64-proxy|$(REPOSITORY_URL)repository/epel-el-8-x86_64-proxy) - else ifeq ($(DISTRO_BASE), EL_7) - # hack to use 7.9 non-group repos on EL_7 - $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(subst $(ORIG_TARGET_VER),$(DISTRO_VERSION),$(REPOSITORY_URL)repository/centos-7.9-base-x86_64-proxy|$(REPOSITORY_URL)repository/centos-7.9-extras-x86_64-proxy|$(REPOSITORY_URL)repository/centos-7.9-updates-x86_64-proxy|$(REPOSITORY_URL)repository/epel-el-7-x86_64-proxy) - else ifeq ($(DISTRO_BASE), LEAP_15) - # hack to use 15 non-group repos on LEAP_15 - $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(subst $(ORIG_TARGET_VER),$(DISTRO_VERSION),$(REPOSITORY_URL)repository/opensuse-15.2-oss-x86_64-proxy|$(REPOSITORY_URL)repository/opensuse-15.2-update-oss-x86_64-provo-mirror-proxy|$(REPOSITORY_URL)repository/opensuse-15.2-update-non-oss-x86_64-proxy|$(REPOSITORY_URL)repository/opensuse-15.2-non-oss-x86_64-proxy|$(REPOSITORY_URL)repository/opensuse-15.2-repo-sle-update-proxy|$(REPOSITORY_URL)repository/opensuse-15.2-repo-backports-update-proxy) - else - # debian - $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS) $(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_$(DAOS_REPO_TYPE)_REPO) - endif # ifeq ($(DISTRO_BASE), *) - endif #ifneq ($(DAOS_STACK_$(DISTRO_BASE)_$(DAOS_REPO_TYPE)_REPO),) - ifneq ($(DAOS_STACK_$(DISTRO_BASE)_APPSTREAM_REPO),) - $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_APPSTREAM_REPO) - endif - # group repos are not working in Nexus so we hack in the group members directly above - ifneq ($(DAOS_STACK_$(DISTRO_BASE)_POWERTOOLS_REPO),) - $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_POWERTOOLS_REPO) - endif - ifneq ($(ID_LIKE),debian) - ifneq ($(DAOS_STACK_INTEL_ONEAPI_REPO),) - $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(REPOSITORY_URL)$(DAOS_STACK_INTEL_ONEAPI_REPO) - endif # ifneq ($(DAOS_STACK_INTEL_ONEAPI_REPO),) - endif # ifneq ($(ID_LIKE),debian) - endif # ifneq ($(REPOSITORY_URL),) + ifneq ($(ARTIFACTORY_URL),) + ifneq ($(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO),) + DISTRO_REPOS = disabled # any non-empty value here works and is not used beyond testing if the value is empty or not + # convert to artifactory url + DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO := $(subst reposi,artifac,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)) + # $(DISTRO_BASE)_LOCAL_REPOS is a list separated by | because you cannot pass lists + # of values with spaces as environment variables + $(DISTRO_BASE)_LOCAL_REPOS := [trusted=yes] $(ARTIFACTORY_URL)$(subst stack,stack-daos,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)) + $(DISTRO_BASE)_LOCAL_REPOS += |[trusted=yes] $(ARTIFACTORY_URL)$(subst stack,stack-deps,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)) + endif #ifneq ($(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO),) + endif # ifneq ($(ARTIFACTORY_URL),) endif # ifeq ($(LOCAL_REPOS),true) ifeq ($(ID_LIKE),debian) chrootbuild: $(DEB_TOP)/$(DEB_DSC) $(call distro_map) \ DISTRO="$$distro" \ PR_REPOS="$(PR_REPOS)" \ + REPO_FILES_PR="$(REPO_FILES_PR)" \ DISTRO_BASE_PR_REPOS="$($(DISTRO_BASE)_PR_REPOS)" \ JENKINS_URL="$${JENKINS_URL}" \ JOB_REPOS="$(JOB_REPOS)" \ + REPO_FILE_URL="$(REPO_FILE_URL)" \ DISTRO_BASE_LOCAL_REPOS="$($(DISTRO_BASE)_LOCAL_REPOS)" \ VERSION_CODENAME="$(VERSION_CODENAME)" \ DEB_TOP="$(DEB_TOP)" \ @@ -400,26 +374,61 @@ chrootbuild: $(SRPM) $(CALLING_MAKEFILE) DISTRO="$$distro" \ CHROOT_NAME="$(CHROOT_NAME)" \ PR_REPOS="$(PR_REPOS)" \ + REPO_FILES_PR="$(REPO_FILES_PR)" \ DISTRO_BASE_PR_REPOS="$($(DISTRO_BASE)_PR_REPOS)" \ JENKINS_URL="$${JENKINS_URL}" \ JOB_REPOS="$(JOB_REPOS)" \ - DISTRO_BASE_LOCAL_REPOS="$($(DISTRO_BASE)_LOCAL_REPOS)" \ + REPO_FILE_URL="$(REPO_FILE_URL)" \ MOCK_OPTIONS="$(MOCK_OPTIONS)" \ RPM_BUILD_OPTIONS='$(RPM_BUILD_OPTIONS)' \ DISTRO_REPOS='$(DISTRO_REPOS)' \ + ARTIFACTORY_URL="$(ARTIFACTORY_URL)" \ + REPOSITORY_URL="$(REPOSITORY_URL)" \ + DISTRO_VERSION="$(DISTRO_VERSION)" \ TARGET="$<" \ packaging/rpm_chrootbuild endif +podman_chrootbuild: + if ! podman build --build-arg REPO_FILE_URL=$(REPO_FILE_URL) \ + -t $(subst +,-,$(CHROOT_NAME))-chrootbuild \ + -f packaging/Dockerfile.mockbuild .; then \ + echo "Container build failed"; \ + exit 1; \ + fi + rm -f /var/lib/mock/$(CHROOT_NAME)/result/{root,build}.log + podman run --rm --privileged -w $(TOPDIR) -v=$(TOPDIR)/..:$(TOPDIR)/.. \ + -it $(subst +,-,$(CHROOT_NAME))-chrootbuild \ + bash -c 'if ! DISTRO_REPOS=false \ + REPO_FILE_URL=$(REPO_FILE_URL) \ + REPOSITORY_URL=$(REPOSITORY_URL) \ + make REPO_FILES_PR=$(REPO_FILES_PR) \ + MOCK_OPTIONS=$(MOCK_OPTIONS) \ + CHROOT_NAME=$(CHROOT_NAME) -C $(CURDIR) chrootbuild; then \ + cat /var/lib/mock/$(CHROOT_NAME)/{result/{root,build},root/builddir/build/BUILD/*/config}.log; \ + exit 1; \ + fi; \ + rpmlint $$(ls /var/lib/mock/$(CHROOT_NAME)/result/*.rpm | \ + grep -v -e debuginfo -e debugsource -e src.rpm)' + docker_chrootbuild: - $(DOCKER) build --build-arg UID=$$(id -u) -t chrootbuild \ - -f packaging/Dockerfile.mockbuild . - $(DOCKER) run --privileged=true -w $(TOPDIR) -v=$(TOPDIR):$(TOPDIR) \ - -it chrootbuild bash -c "make -C $(CURDIR) \ - CHROOT_NAME=$(CHROOT_NAME) chrootbuild" + if ! $(DOCKER) build --build-arg UID=$$(id -u) -t chrootbuild \ + --build-arg REPO_FILE_URL=$(REPO_FILE_URL) \ + -f packaging/Dockerfile.mockbuild .; then \ + echo "Container build failed"; \ + exit 1; \ + fi + rm -f /var/lib/mock/$(CHROOT_NAME)/result/{root,build}.log + if ! $(DOCKER) run --user=$$(id -u) --privileged=true -w $(TOPDIR) \ + -v=$(TOPDIR):$(TOPDIR) \ + -it chrootbuild bash -c "make -C $(CURDIR) \ + CHROOT_NAME=$(CHROOT_NAME) chrootbuild"; then \ + cat /var/lib/mock/$(CHROOT_NAME)/result/{root,build}.log; \ + exit 1; \ + fi rpmlint: $(SPEC) - rpmlint $< + rpmlint --ignore-unused-rpmlintrc $< packaging_check: if grep -e --repo $(CALLING_MAKEFILE); then \ @@ -456,6 +465,15 @@ test: $(call install_repos,$(REPO_NAME)@$(BRANCH_NAME):$(BUILD_NUMBER)) dnf -y install $(TEST_PACKAGES) +show_NAME: + @echo '$(NAME)' + +show_DISTRO_ID: + @echo '$(DISTRO_ID)' + +show_distro_map: + @$(call distro_map) echo "$$distro" + show_spec: @echo '$(SPEC)' diff --git a/packaging/rpm_chrootbuild b/packaging/rpm_chrootbuild index 8c1008e..03ab4c5 100755 --- a/packaging/rpm_chrootbuild +++ b/packaging/rpm_chrootbuild @@ -2,17 +2,7 @@ set -uex -# shellcheck disable=SC2153 -IFS=\| read -r -a distro_base_local_repos <<< "$DISTRO_BASE_LOCAL_REPOS" -repo_adds=() -repo_dels=() - -if [ -n "${REPOSITORY_URL:-}" ] && [ -n "$DISTRO_REPOS" ]; then - repo_dels+=("--disablerepo=\*") -fi - -: "${WORKSPACE:=$PWD}" -mock_config_dir="$WORKSPACE/mock" +mock_config_dir="${WORKSPACE:-$PWD}/mock" original_cfg_file="/etc/mock/${CHROOT_NAME}.cfg" cfg_file="$mock_config_dir/${CHROOT_NAME}.cfg" mkdir -p "$mock_config_dir" @@ -21,12 +11,49 @@ ln -sf /etc/mock/logging.ini "$mock_config_dir/" cp "$original_cfg_file" "$cfg_file" -if [ "${CHROOT_NAME}" == "epel-8-x86_64" ]; then - echo -e "config_opts['module_enable'] = ['javapackages-tools:201801']" \ - >> "$cfg_file" +if [[ $CHROOT_NAME == *epel-8-x86_64 ]]; then + cat <> "$cfg_file" +config_opts['module_setup_commands'] = [ + ('enable', 'javapackages-tools:201801'), + ('disable', 'go-toolset') +] +EOF + fi +# shellcheck disable=SC2153 +repo_adds=() +repo_dels=() + echo -e "config_opts['yum.conf'] += \"\"\"\n" >> "$cfg_file" + +if { [ -n "${REPOSITORY_URL:-}" ] || [ -n "${ARTIFACTORY_URL:-}" ]; } \ + && [ -n "$DISTRO_REPOS" ]; then + repo_dels+=("--disablerepo=\*") + + if [ -n "${REPO_FILE_URL:-}" ]; then + if [ -n "$REPO_FILES_PR" ]; then + if [[ $REPO_FILES_PR = PR-* ]]; then + build_number="lastSuccessfulBuild" + branch="$REPO_FILES_PR" + if [[ $branch = *:* ]]; then + build_number="${branch#*:}" + branch="${branch%:*}" + fi + REPO_FILE_URL="${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-do/job/repo-files/job/$branch/$build_number/artifact/" + else + if [ ! -d "$REPO_FILES_PR" ]; then + echo "Could not find your specified directory \"$REPO_FILES_PR\" to read repo-files from" + exit 1 + fi + REPO_FILE_URL="file://$(readlink -e "$REPO_FILES_PR")/" + fi + fi + curl -sSf "$REPO_FILE_URL"daos_ci-"$DISTRO"-mock-artifactory.repo >> "$cfg_file" + repo_adds+=("--enablerepo *-artifactory") + fi +fi + for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do branch="master" build_number="lastSuccessfulBuild" @@ -45,7 +72,7 @@ baseurl=${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/jo enabled=1\n\ gpgcheck=False\n" >> "$cfg_file" done -for repo in $JOB_REPOS "${distro_base_local_repos[@]}"; do +for repo in $JOB_REPOS; do repo_name=${repo##*://} repo_name=${repo_name//\//_} repo_adds+=("--enablerepo $repo_name") @@ -56,7 +83,10 @@ enabled=1\n" >> "$cfg_file" done echo "\"\"\"" >> "$cfg_file" +if [ -n "$DISTRO_VERSION" ]; then + releasever_opt=("--config-opts=releasever=$DISTRO_VERSION") +fi # shellcheck disable=SC2086 -eval mock --configdir "$mock_config_dir" -r "${CHROOT_NAME}" \ - ${repo_dels[*]} ${repo_adds[*]} \ - $MOCK_OPTIONS $RPM_BUILD_OPTIONS "$TARGET" +eval mock --configdir "$mock_config_dir" -r "${CHROOT_NAME}" \ + ${repo_dels[*]} ${repo_adds[*]} --disablerepo=\*-debug* \ + "${releasever_opt[@]}" $MOCK_OPTIONS $RPM_BUILD_OPTIONS "$TARGET" diff --git a/packaging/rpmlint--ignore-unused-rpmlintrc.patch b/packaging/rpmlint--ignore-unused-rpmlintrc.patch new file mode 100644 index 0000000..f24075f --- /dev/null +++ b/packaging/rpmlint--ignore-unused-rpmlintrc.patch @@ -0,0 +1,41 @@ +From 060fbdd03d03b0b9835b861a72f9aeb25347277d Mon Sep 17 00:00:00 2001 +From: Martin Liska +Date: Tue, 25 Jan 2022 14:29:05 +0100 +Subject: [PATCH] Add new option --ignore-unused-rpmlintrc. + +The option skips reporting of unused-rpmlintrc-filter errors. + +Fixes: #794. +--- + rpmlint/cli.py | 2 ++ + rpmlint/lint.py | 3 ++- + test/test_lint.py | 3 ++- + 3 files changed, 6 insertions(+), 2 deletions(-) + +diff --git a/rpmlint/cli.py b/rpmlint/cli.py +index 81d5b92b..8a7bc09e 100644 +--- a/rpmlint/cli.py ++++ b/rpmlint/cli.py +@@ -82,6 +82,8 @@ def process_lint_args(argv): + parser.add_argument('-i', '--installed', nargs='+', default='', help='installed packages to be validated by rpmlint') + parser.add_argument('-t', '--time-report', action='store_true', help='print time report for run checks') + parser.add_argument('-T', '--profile', action='store_true', help='print cProfile report') ++ parser.add_argument('--ignore-unused-rpmlintrc', action='store_true', ++ help='Do not report "unused-rpmlintrc-filter" errors') + lint_modes_parser = parser.add_mutually_exclusive_group() + lint_modes_parser.add_argument('-s', '--strict', action='store_true', help='treat all messages as errors') + lint_modes_parser.add_argument('-P', '--permissive', action='store_true', help='treat individual errors as non-fatal') +diff --git a/rpmlint/lint.py b/rpmlint/lint.py +index a98d2c04..1d5e0050 100644 +--- a/rpmlint/lint.py ++++ b/rpmlint/lint.py +@@ -277,7 +277,8 @@ + + # validate used filters in rpmlintrc + if is_last: +- self.output.validate_filters(pkg) ++ if not self.options['ignore_unused_rpmlintrc']: ++ self.output.validate_filters(pkg) + + if spec_checks: + self.specfiles_checked += 1