Support Wi4MPI verification, improved library mapping, and config command

wspear · wspear · commit a523a1014cfa · 2026-01-28T15:56:23.000-08:00
- Wi4MPI: Added installation verification (ELF arch check, binary check), support for ppc64le, and better handling of missing or invalid install directories.
- Launch: Added automatic binding of driver/config directories for graphics/interconnect libraries (ibverbs, OpenCL, Vulkan, etc.) to prevent driver loader errors.
- CLI: Added new `config` command to list, get, and set configuration values.
- Demo: Added arguments for launcher/OSU control, enabled container baseline, and forced local Wi4MPI usage.
diff --git a/e4s_cl/__init__.py b/e4s_cl/__init__.py
@@ -103,11 +103,10 @@ def _get_e4s_cl_script():
                        os.path.join(E4S_CL_HOME, 'system'))))
 """str: System-level E4S Container Launcher files."""
 
-USER_PREFIX = os.path.realpath(
-    os.path.abspath(
-        os.environ.get(
-            '__E4S_CL_USER_PREFIX__',
-            os.path.join(os.path.expanduser('~'), '.local', 'e4s_cl'))))
+USER_PREFIX = os.path.abspath(
+    os.environ.get(
+        '__E4S_CL_USER_PREFIX__',
+        os.path.join(os.path.expanduser('~'), '.local', 'e4s_cl')))
 """str: User-level E4S Container Launcher files."""
 
 _CONTAINER_DIR_ENV = os.environ.get("E4S_CL_CONTAINER_DIR") or os.environ.get("E4S_CL_CONTAINER_DIRECTORY")
diff --git a/e4s_cl/cf/wi4mpi/install.py b/e4s_cl/cf/wi4mpi/install.py
@@ -26,6 +26,7 @@
     VENDOR_BINARIES,
     available_compilers,
 )
+from elftools.elf.elffile import ELFFile
 
 LOGGER = get_logger(__name__)
 
@@ -157,27 +158,77 @@ def _double_tap(cmd):
     return not success
 
 
+def _check_wi4mpi_install(install_dir: Path) -> bool:
+    """
+    Sanity check for the Wi4MPI installation
+    """
+    # Check binary existence
+    binary = install_dir / 'bin' / 'wi4mpi'
+    if not binary.exists():
+        LOGGER.debug("Wi4MPI binary not found at %s", binary)
+        return False
+
+    # Check architecture of shares objects
+    # Expected machine from os.uname().machine
+    machine = os.uname().machine
+    expected = {
+        'x86_64': 'EM_X86_64',
+        'amd64': 'EM_X86_64',
+        'aarch64': 'EM_AARCH64',
+        'ppc64le': 'EM_PPC64',
+        'ppc64': 'EM_PPC64',
+    }.get(machine)
+
+    if expected:
+        # Check libexec shared objects
+        lib_dir = install_dir / 'libexec' / 'wi4mpi'
+        libraries = list(lib_dir.glob('libwi4mpi_*.so'))
+
+        if libraries:
+            for lib in libraries:
+                if not lib.is_file():
+                    continue
+                try:
+                    with open(lib, 'rb') as f:
+                        elf = ELFFile(f)
+                        if elf.header.e_machine != expected:
+                            LOGGER.error(
+                                "Wi4MPI library %s architecture mismatch: expected %s, got %s",
+                                lib.name, expected, elf.header.e_machine)
+                            return False
+                except Exception as err:
+                    LOGGER.debug("Failed to check architecture of %s: %s", lib,
+                                 err)
+
+    # Check if executable runs
+    if run_subprocess([str(binary), '-h'], discard_output=True):
+        LOGGER.error("Wi4MPI binary at %s failed to run", binary)
+        return False
+
+    return True
+
+
 def install_wi4mpi(install_dir: Path) -> Optional[Path]:
     """Clones and installs wi4mpi from github releases"""
 
-    if os.uname().machine not in {'x86_64', 'amd64', 'aarch64'}:
+    if os.uname().machine not in {'x86_64', 'amd64', 'aarch64', 'ppc64le'}:
         LOGGER.warning(
             "Wi4MPI not available for the following architecture: %s",
             os.uname().machine)
         return None
 
-    binary = install_dir / 'bin' / 'wi4mpi'
-    if install_dir.exists() and binary.exists():
-        LOGGER.debug(
-            "Skipping installation for already installed Wi4MPI in %s",
-            install_dir)
-        return install_dir
-
-    if install_dir.exists() and list(install_dir.glob('*')):
-        LOGGER.error(
-            "Attempting Wi4MPI installation in a non-empty directory: %s",
-            str(install_dir))
-        return None
+    if install_dir.exists():
+        if _check_wi4mpi_install(install_dir):
+            LOGGER.debug(
+                "Skipping installation for already installed Wi4MPI in %s",
+                install_dir)
+            return install_dir
+
+        if list(install_dir.glob('*')):
+            LOGGER.error(
+                "Target directory %s is not empty and contains an invalid or incomplete Wi4MPI installation.",
+                install_dir)
+            return None
 
     # Assert CMake is available
     cmake_executable = which("cmake")
diff --git a/e4s_cl/cli/commands/launch.py b/e4s_cl/cli/commands/launch.py
@@ -144,17 +144,19 @@ def _setup_wi4mpi(
         translation = [vendor.replace('mvapich', 'mpich') for vendor in translation]
 
     # Locate the Wi4MPI installation and store it in parameters
-    if parameters.wi4mpi is None:
+    target_dir = parameters.wi4mpi
+    if target_dir is None:
         target_dir = Path(config.CONFIGURATION.wi4mpi_install_directory)
-        LOGGER.debug("Target: %s", target_dir)
-        wi4mpi_install = install_wi4mpi(target_dir)
-        if wi4mpi_install:
-            parameters.wi4mpi = wi4mpi_install
-        else:
-            LOGGER.error(
-                "Wi4MPI is required for this configuration, but installation failed"
-            )
-            return []
+
+    LOGGER.debug("Target: %s", target_dir)
+    wi4mpi_install = install_wi4mpi(target_dir)
+    if wi4mpi_install:
+        parameters.wi4mpi = wi4mpi_install
+    else:
+        LOGGER.error(
+            "Wi4MPI is required for this configuration, but installation failed"
+        )
+        return []
 
     run_c_lib, run_f_lib = wi4mpi_find_libraries(family_metadata,
                                                  mpi_libraries)
@@ -290,7 +292,7 @@ def _construct_parser(self):
         parser.add_argument(
             '--wi4mpi',
             type=arguments.posix_path,
-            help="Path towards a Wi4MPI installation to use",
+            help="Path to a Wi4MPI installation (will be installed there if missing)",
             metavar='installation',
         )
 
@@ -390,6 +392,54 @@ def main(self, argv):
                         if varname in os.environ.keys():
                             launcher.extend(shlex.split(f"-x {varname}"))
 
+        # Helper: bind system configuration for libraries that use plugins/drivers
+        # This prevents the host library from loading incompatible container drivers
+        # by masking the container's configuration directories with the host's.
+        
+        # Define known configuration paths for libraries that require masking
+        system_config_dirs = ['/etc', '/usr/etc', '/usr/local/etc']
+        
+        # Library -> Subdirectories to mask
+        config_masks = {
+            'libibverbs.so': ['libibverbs.d'],
+            'libOpenCL.so': ['OpenCL'],
+            'libvulkan.so': ['vulkan'],
+            'libGLX.so': ['glvnd'],
+            'libEGL.so': ['glvnd']
+        }
+
+        # Check environment variables for driver paths that need directory binding
+        # (Binding the full directory handles scanning/plugins better than individual files)
+        driver_env_vars = {
+            'libibverbs.so': ['IBVERBS_DRIVER_PATH']
+        }
+
+        for lib in getattr(parameters, 'libraries', []):
+            # Check for driver directories relative to lib (Standard convention)
+            if lib.name.startswith('libibverbs.so'):
+                verbs_dir = lib.parent / 'libibverbs'
+                if verbs_dir.exists():
+                    parameters.files.add(verbs_dir)
+            
+            # Check for environment-defined driver paths
+            for prefix, vars in driver_env_vars.items():
+                if lib.name.startswith(prefix):
+                    for var in vars:
+                        if var in os.environ:
+                            path = Path(os.environ[var])
+                            if path.exists():
+                                parameters.files.add(path)
+
+            # Check for configuration directories to mask
+            for prefix, subdirs in config_masks.items():
+                if lib.name.startswith(prefix):
+                    for conf_root in system_config_dirs:
+                        for subdir in subdirs:
+                            conf_path = Path(conf_root) / subdir
+                            if conf_path.exists():
+                                parameters.files.add(conf_path)
+
+
         execute_command = _format_execute(parameters)
 
         def _supports_end_of_options(tokens: List[str]) -> bool:
diff --git a/scripts/demo.sh b/scripts/demo.sh
@@ -41,7 +41,7 @@ E4S_CL_IMAGE_DEF=""
 E4S_CL_REBUILD_IMAGE="0"
 E4S_CL_APPTAINER_BUILD_ARGS=""
 E4S_CL_RUN_HOST_BASELINE="1"
-E4S_CL_RUN_CONTAINER_BASELINE="0"
+E4S_CL_RUN_CONTAINER_BASELINE="1"
 E4S_CL_PRUNE_INTEL_OPENCL="0"
 E4S_CL_SKIP_PROFILE_DETECT="0"
 E4S_CL_WI4MPI_CFLAGS="-Wno-error=implicit-function-declaration -Wno-error=incompatible-pointer-types -Wno-error=format -Wno-error=int-conversion -Wno-error=return-type -include string.h -include sys/time.h"
@@ -52,6 +52,8 @@ E4S_CL_HOST_MPIRUN=""
 E4S_CL_LAUNCHER=""
 E4S_CL_SCHEDULER=""
 E4S_CL_E4SCL_LAUNCH_ARGS=""
+E4S_CL_LAUNCHER_ARGS=""
+E4S_CL_OSU_ARGS=""
 E4S_CL_TIMEOUT_DURATION="60s"
 
 log() { printf "[e4s-cl-test] %s\n" "$*"; }
@@ -129,7 +131,7 @@ Options:
   --cache-dir <path>           Cache directory (default: _e4scl_cache_<tag>)
   --clean-workdir              Delete workdir on exit (default: off)
   --host-baseline <on|off>     Run host-only baseline (default: on)
-  --container-baseline <on|off> Run container-only baseline (default: off)
+  --container-baseline <on|off> Run container-only baseline (default: on)
   --no-prune-intel-opencl      Disable workaround that prunes Intel/OpenCL libs from profile (default: pruning enabled)
   --skip-profile-detect        Skip profile detect if profile already has bindings (default: off)
   --wi4mpi-cflags "..."         Extra C/C++ flags for Wi4MPI build (default: relax GCC 14 errors)
@@ -139,7 +141,9 @@ Options:
   --host-mpirun <path>         Override host mpirun/mpiexec (default: auto-detect)
   --launcher <cmd>             Force launcher (mpirun or srun) (default: auto-detect)
   --scheduler <name>           If "slurm", use srun when available (default: none)
-  --e4scl-launch-args "..."     Extra args for e4s-cl launch (default: none)
+  --launcher-args "..."        Extra arguments for the MPI launcher (e.g. -p partition -N 2)
+  --osu-args "..."             Override OSU benchmark arguments (e.g. "-i 1000 -m 1024:1048576")
+                               Safe for latency, bw, and allreduce.
   --timeout <duration>         Timeout for MPI runs (default: 60s)
   --check                      Check environment prerequisites and exit
   -h, --help                   Show help
@@ -223,6 +227,8 @@ while [[ $# -gt 0 ]]; do
     --launcher) E4S_CL_LAUNCHER="$2"; shift 2 ;;
     --scheduler) E4S_CL_SCHEDULER="$2"; shift 2 ;;
     --e4scl-launch-args) E4S_CL_E4SCL_LAUNCH_ARGS="$2"; shift 2 ;;
+    --launcher-args) E4S_CL_LAUNCHER_ARGS="$2"; shift 2 ;;
+    --osu-args) E4S_CL_OSU_ARGS="$2"; shift 2 ;;
     --timeout) E4S_CL_TIMEOUT_DURATION="$2"; shift 2 ;;
     --check) E4S_CL_ONLY_CHECK="1"; shift ;;
     -h|--help) usage; exit 0 ;;
@@ -334,6 +340,7 @@ run_timed() {
   local out_file="${E4S_CL_WORKDIR}/timing.dat"
 
   log "Running [${label}]..."
+  log "Command: ${cmd[*]}"
   # Use python for distinct wall-clock measurement
   local start
   start=$(python3 -c 'import time; print(time.time())')
@@ -472,6 +479,11 @@ else
   LAUNCHER_ARGS=("-np" "${E4S_CL_MPI_PROCS}")
 fi
 
+if [[ -n "${E4S_CL_LAUNCHER_ARGS}" ]]; then
+  read -r -a EXTRA_LAUNCHER_ARGS <<< "${E4S_CL_LAUNCHER_ARGS}"
+  LAUNCHER_ARGS+=("${EXTRA_LAUNCHER_ARGS[@]}")
+fi
+
 HOST_MPI_VERSION="$(${HOST_MPIRUN} --version 2>/dev/null | head -n 2 || true)"
 HOST_MPI_FAMILY="$(detect_mpi_family "${HOST_MPI_VERSION}")"
 CONTAINER_MPI_VERSION="$(${CONTAINER_CMD} exec "${E4S_CL_IMAGE}" mpirun --version 2>/dev/null | head -n 2 || true)"
@@ -548,6 +560,12 @@ E4S_CL_BIN="${REPO_ROOT}/.venv/bin/e4s-cl"
 
 if [[ "${NEEDS_TRANSLATION}" == "1" ]]; then
   log "Wi4MPI translation required; e4s-cl will install Wi4MPI during launch if missing"
+
+  # Force wi4mpi to use a local directory inside the workdir
+  WI4MPI_LOCAL_PREFIX="${E4S_CL_WORKDIR}/wi4mpi"
+  E4SCL_LAUNCH_ARGS+=("--wi4mpi" "${WI4MPI_LOCAL_PREFIX}")
+  log "CONFIG: using local Wi4MPI prefix: ${WI4MPI_LOCAL_PREFIX}"
+
   if [[ -n "${E4S_CL_WI4MPI_CFLAGS}" ]]; then
     log "CONFIG: Wi4MPI build flags set (for e4s-cl internal use): ${E4S_CL_WI4MPI_CFLAGS}"
     log "CONFIG: These flags will be used by e4s-cl when building Wi4MPI, not for OSU benchmarks"
@@ -625,12 +643,21 @@ printf "osu_url=%s\nosu_sha256=%s\n" "${E4S_CL_OSU_URL}" "${E4S_CL_OSU_SHA256}"
 
 if [[ "${E4S_CL_MODE}" == "light" ]]; then
   OSU_BENCHES=("pt2pt/osu_latency" "pt2pt/osu_bw")
-  OSU_ARGS=("-x" "10" "-i" "100" "-m" "8:1024")
+  OSU_ARGS=("-x" "100" "-i" "1000" "-m" "8:65536")
 else
   OSU_BENCHES=("pt2pt/osu_latency" "pt2pt/osu_bw" "collective/osu_allreduce")
   OSU_ARGS=()
 fi
 
+if [[ -n "${E4S_CL_OSU_ARGS}" ]]; then
+  read -r -a OSU_ARGS <<< "${E4S_CL_OSU_ARGS}"
+else
+  # Default Arguments:
+  # - Latency: 1000 iterations to dampen startup noise
+  # - Bandwidth: up to 1MB (1048576) is usually sufficient to see peak BW without timing out
+  OSU_ARGS=("-x" "100" "-i" "1000" "-m" "8:1048576")
+fi
+
 if [[ "${E4S_CL_MPI_PROCS}" != "2" ]]; then
   if printf '%s\n' "${OSU_BENCHES[@]}" | grep -q "pt2pt/osu_"; then
     log "NOTE: pt2pt OSU benchmarks require exactly 2 processes (current: ${E4S_CL_MPI_PROCS})"
@@ -730,6 +757,9 @@ if ! profile_has_bindings; then
   fi
 fi
 
+log "Profile Content (Libraries/Files to be bound):"
+"${E4S_CL_BIN}" profile show
+
 log "Step 5: Building Container Benchmarks. We compile the same benchmarks *inside* the container against the Container's MPI to demonstrate ABI compatibility or translation."
 OSU_CONT_PREFIX="${E4S_CL_WORKDIR}/osu-container"
 OSU_CONT_META="${OSU_CONT_PREFIX}/.build-meta"
@@ -750,6 +780,7 @@ if [[ "${REBUILD_CONT_OSU}" == "1" ]]; then
     set -euo pipefail; \
     unset CFLAGS; \
     unset CXXFLAGS; \
+    unset CXX; \
     cd /work; \
     rm -rf osu-container-build && mkdir osu-container-build; \
     tar -xzf /cache/osu.tar.gz -C osu-container-build --strip-components=1; \
@@ -764,6 +795,9 @@ else
 fi
 
 log "Step 6: MAIN TEST. Running the container-compiled binary using the Host's MPI. e4s-cl handles the library injection and launcher wrapping."
+log "Debug: Verifying library resolution inside container (ldd)"
+"${E4S_CL_BIN}" launch "${E4SCL_LAUNCH_ARGS[@]}" "${LAUNCHER_BIN}" -n 1 ldd "${OSU_CONT_PREFIX}/libexec/osu-micro-benchmarks/mpi/pt2pt/osu_latency" | grep -E "libmpi|libmpich|libpami|libfabric" || echo "  (ldd check finished, no obvious MPI libs found in grep filter)"
+
 for bench in "${OSU_BENCHES[@]}"; do
   bench_name="$(basename "${bench}")"
   out_file="${E4S_CL_WORKDIR}/e4scl_${bench_name}.txt"
