fix: Fix local stack failing the hello world job

The fix adds the env variable to the executor that tells the executor it
works on the local machine, and not in the docker, so it needs to query
the nginx for streaming jobs via the localhost. The variable is passed
to the executor by the DevExecutorManager.

Impacts: miner, executor

Author: kacper-wolkiewicz-reef

compute_horde_sdk/src/compute_horde_core/certificate.py

@@ -72,6 +72,7 @@ async def start_nginx(
     job_network: str,
     container_name: str = "job-nginx",
     timeout: int = 10,
+    on_host: bool = False,
 ) -> None:
     nginx_conf_file = dir_path / "nginx.conf"
     nginx_conf_file.write_text(nginx_conf)
@@ -86,11 +87,17 @@ async def start_nginx(
         "--network",
         "bridge",  # primary external network
         "-p",
-        f"{port}:443",  # expose nginx port
-        "-v",
-        f"{dir_path}:/etc/nginx/",
-        NGINX_IMAGE,
+        f"{port}:443",  # expose nginx port (HTTPS)
     ]
+    if on_host:
+        cmd.extend(["-p", f"{port + 1}:80"])  # expose health check port when running on host
+    cmd.extend(
+        [
+            "-v",
+            f"{dir_path}:/etc/nginx/",
+            NGINX_IMAGE,
+        ]
+    )
     process = await asyncio.create_subprocess_exec(*cmd)
     _stdout, _stderr = await process.communicate()
     await process.wait()
@@ -103,7 +110,10 @@ async def start_nginx(
     await process.wait()
 
     # wait for nginx to start
-    url = f"http://{ip}/ok"
+    if on_host:
+        url = f"http://localhost:{port + 1}/ok"
+    else:
+        url = f"http://{ip}/ok"
     nginx_started = await check_endpoint(url, timeout)
     if not nginx_started:
         stdout = _stdout.decode() if _stdout else ""

executor/app/src/compute_horde_executor/executor/job_runner.py

@@ -538,16 +538,21 @@ async def after_start_job(self):
                 job_network=self.job_network_name,
                 container_name=self.nginx_container_name,
                 timeout=WAIT_FOR_NGINX_TIMEOUT,
+                on_host=settings.EXECUTOR_ON_HOST,
             )
         except Exception as e:
             raise HordeError(f"Failed to start Nginx: {truncate(str(e))}") from e
 
         assert self.executor_certificate is not None
         # check that the job is ready to serve requests
-        ip = await get_docker_container_ip(self.nginx_container_name, bridge_network=True)
-        logger.debug(f"Checking if streaming job is ready at http://{ip}/health")
+        if settings.EXECUTOR_ON_HOST:
+            health_url = f"http://localhost:{settings.NGINX_PORT + 1}/health"
+        else:
+            ip = await get_docker_container_ip(self.nginx_container_name, bridge_network=True)
+            health_url = f"http://{ip}/health"
+        logger.debug(f"Checking if streaming job is ready at {health_url}")
         job_ready = await check_endpoint(
-            f"http://{ip}/health",
+            health_url,
             WAIT_FOR_STREAMING_JOB_TIMEOUT,  # TODO: TIMEOUTS - Remove timeout?
         )
         if not job_ready:

executor/app/src/compute_horde_executor/settings.py

@@ -291,6 +291,7 @@ def wrapped(*args, **kwargs):
 
 EXECUTOR_TOKEN = env.str("EXECUTOR_TOKEN")
 DEBUG_NO_GPU_MODE = env.bool("DEBUG_NO_GPU_MODE", default=False)
+EXECUTOR_ON_HOST = env.bool("EXECUTOR_ON_HOST", default=False)
 VOLUME_MAX_SIZE_BYTES = env.int("VOLUME_MAX_SIZE_BYTES", default=2147483648)  # 2GB
 OUTPUT_ZIP_UPLOAD_MAX_SIZE_BYTES = env.int(
     "OUTPUT_ZIP_UPLOAD_MAX_SIZE_BYTES", default=2147483648

local_stack/uv.lock

@@ -24,6 +24,19 @@ async def start_new_executor(self, token, executor_class, timeout):
         logger.debug(f"Starting new executor process: {token=}")
         nginx_port = executor_port_dispenser.get_port()
 
+        env = {
+            "MINER_ADDRESS": f"ws://{settings.ADDRESS_FOR_EXECUTORS}:{settings.PORT_FOR_EXECUTORS}",
+            "EXECUTOR_TOKEN": token,
+            "PATH": os.environ["PATH"],
+            "NGINX_PORT": str(nginx_port),
+            "EXECUTOR_ON_HOST": "1",
+            # Enable hf_transfer download acceleration package
+            # https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables#hfhubenablehftransfer
+            "HF_HUB_ENABLE_HF_TRANSFER": "1",
+        }
+        if docker_host := os.environ.get("DOCKER_HOST"):
+            env["DOCKER_HOST"] = docker_host
+
         return subprocess.Popen(
             [
                 "uv",
@@ -33,15 +46,7 @@ async def start_new_executor(self, token, executor_class, timeout):
                 "run_executor",
                 *(await self.get_executor_cmdline_args()),
             ],
-            env={
-                "MINER_ADDRESS": f"ws://{settings.ADDRESS_FOR_EXECUTORS}:{settings.PORT_FOR_EXECUTORS}",
-                "EXECUTOR_TOKEN": token,
-                "PATH": os.environ["PATH"],
-                "NGINX_PORT": str(nginx_port),
-                # Enable hf_transfer download acceleration package
-                # https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables#hfhubenablehftransfer
-                "HF_HUB_ENABLE_HF_TRANSFER": "1",
-            },
+            env=env,
             cwd=executor_dir,
         )
 

miner/app/src/compute_horde_miner/miner/executor_manager/_internal/dev.py

@@ -13,7 +13,7 @@ DATABASE_POOL_URL=
 DATABASE_URL=postgres://postgres:12345@localhost:8432/compute_horde_miner
 
 EXECUTOR_MANAGER_CLASS_PATH=compute_horde_miner.miner.executor_manager.dev:DevExecutorManager
-EXECUTOR_IMAGE=backenddevelopersltd/compute-horde-executor:v0-latest
+EXECUTOR_IMAGE=backenddevelopersltd/compute-horde-executor:v1-latest
 ADDRESS_FOR_EXECUTORS=localhost
 PORT_FOR_EXECUTORS=8000
 BITTENSOR_MINER_ADDRESS=127.0.0.1