feat: Use Pylon client in validator to fetch neurons and commitments

Validator now uses Pylon client to fetch neurons and commitments.

Impacts: validator
Issue: COM-803

Author: kacper-wolkiewicz-reef

validator/app/src/compute_horde_validator/settings.py

@@ -653,7 +653,7 @@ def wrapped(*args, **kwargs):
     },
     "root": {
         "handlers": ["console"],
-        "level": "DEBUG",
+        "level": env.str("ROOT_LOGGER_LEVEL", "DEBUG"),
     },
     "loggers": {
         "django": {

validator/app/src/compute_horde_validator/validator/allowance/tests/mockchain.py

@@ -1,5 +1,6 @@
 import datetime
 from contextlib import contextmanager
+from ipaddress import IPv4Address
 from unittest import mock
 from unittest.mock import patch
 
@@ -9,6 +10,7 @@
 from compute_horde.test_wallet import get_test_validator_wallet
 from compute_horde_core.executor_class import ExecutorClass
 from pydantic import BaseModel
+from pylon.v1 import AxonInfo, AxonProtocol, Neuron, Stakes
 from turbobt.subtensor.runtime.subnet_info import SubnetHyperparams
 
 from compute_horde_validator.validator.allowance.types import MetagraphData, ValidatorModel
@@ -180,7 +182,37 @@ def manifest_responses(block_number) -> list[tuple[str, str | BaseModel, float]]
     ]
 
 
-def _make_neuron(uid, key, stake, is_miner: bool, is_shielded) -> turbobt.Neuron:
+def _make_neuron(uid, key, stake, is_miner: bool) -> Neuron:
+    if not is_miner:
+        ip = IPv4Address("0.0.0.0")
+    else:
+        ip = IPv4Address(f"192.168.1.{uid % 256}")
+    return Neuron(
+        uid=uid,
+        coldkey=key,
+        hotkey=key,
+        active=True,
+        axon_info=AxonInfo(
+            ip=ip,
+            port=8000 + uid if is_miner else 0,
+            protocol=AxonProtocol.HTTP,
+        ),
+        stake=stake,
+        rank=0.0,
+        emission=0.0,
+        incentive=0.0,
+        consensus=0.0,
+        trust=0.0,
+        validator_trust=0.0,
+        dividends=0.0,
+        last_update=0,
+        validator_permit=False,
+        pruning_score=0,
+        stakes=Stakes(alpha=0.0, tao=stake, total=stake),
+    )
+
+
+def _make_turbobt_neuron(uid, key, stake, is_miner: bool, is_shielded) -> turbobt.Neuron:
     return turbobt.Neuron(
         subnet=mock.MagicMock(),
         uid=uid,
@@ -230,18 +262,14 @@ def stake(block_number: int, ind: int, hotkey: str) -> float:
     return float(stake)
 
 
-def list_validators(block_number: int, filter_=True) -> list[turbobt.Neuron]:
+def list_validators(block_number: int, filter_=True) -> list[Neuron]:
     assert block_number >= START_BLOCK
     return list(
         filter(
             (lambda n: n.stake >= 1000) if filter_ else lambda n: True,
             [
                 _make_neuron(
-                    ind + NUM_MINERS,
-                    hotkey,
-                    stake=stake(block_number, ind, hotkey),
-                    is_miner=False,
-                    is_shielded=False,
+                    ind + NUM_MINERS, hotkey, stake=stake(block_number, ind, hotkey), is_miner=False
                 )
                 for ind, hotkey in VALIDATOR_HOTKEYS.items()
                 if (
@@ -254,7 +282,7 @@ def list_validators(block_number: int, filter_=True) -> list[turbobt.Neuron]:
     )
 
 
-def list_neurons(block_number: int, with_shield: bool) -> list[turbobt.Neuron]:
+def list_neurons(block_number: int, with_shield: bool) -> list[Neuron]:
     assert block_number >= START_BLOCK
     return [
         *[
@@ -263,7 +291,6 @@ def list_neurons(block_number: int, with_shield: bool) -> list[turbobt.Neuron]:
                 hotkey,
                 stake=float(ind),
                 is_miner=True,
-                is_shielded=bool(ind % 2) if with_shield else False,
             )
             for ind, hotkey in MINER_HOTKEYS.items()
             if (
@@ -276,6 +303,38 @@ def list_neurons(block_number: int, with_shield: bool) -> list[turbobt.Neuron]:
     ]
 
 
+def list_turbobt_neurons(block_number: int, with_shield: bool) -> list[turbobt.Neuron]:
+    assert block_number >= START_BLOCK
+    validators = list_validators(block_number, filter_=False)
+    return [
+        *[
+            _make_turbobt_neuron(
+                ind,
+                hotkey,
+                stake=float(ind),
+                is_miner=True,
+                is_shielded=bool(ind % 2) if with_shield else False,
+            )
+            for ind, hotkey in MINER_HOTKEYS.items()
+            if (
+                block_number < START_CHANGING_MANIFESTS_BLOCK
+                or not hotkey.startswith("deregging_miner_")
+                or block_number % 5
+            )
+        ],
+        *[
+            _make_turbobt_neuron(
+                v.uid,
+                v.hotkey,
+                stake=v.stake,
+                is_miner=False,
+                is_shielded=False,
+            )
+            for v in validators
+        ],
+    ]
+
+
 def get_block_timestamp(block_number):
     base_time = datetime.datetime(2024, 1, 1, 12, 0, 0, tzinfo=datetime.UTC)
     return base_time + datetime.timedelta(
@@ -300,7 +359,7 @@ def get_current_block(self):
             return self.block_number
 
         def get_shielded_neurons(self):
-            return list_neurons(block_number_, with_shield=False)
+            return list_turbobt_neurons(block_number_, with_shield=False)
 
         def list_neurons(self, block_number):
             return list_neurons(block_number, with_shield=False)

validator/app/src/compute_horde_validator/validator/allowance/tests/test_precaching_supertensor.py

@@ -2,11 +2,11 @@
 from collections.abc import Callable
 from typing import Any, cast
 
-import turbobt
 from celery.utils.log import get_task_logger
 from compute_horde_core.executor_class import ExecutorClass
 from django.db import IntegrityError, transaction
 from django.db.models import Case, Exists, Max, Min, OuterRef, Q, Sum, When
+from pylon.v1 import Neuron
 
 from compute_horde_validator.validator.dynamic_config import get_config
 from compute_horde_validator.validator.locks import Lock, LockType
@@ -109,7 +109,7 @@ def get_stake_share(
     return validator.effective_stake / total_stake_sum
 
 
-def save_neurons(neurons: list[turbobt.Neuron], block: int):
+def save_neurons(neurons: list[Neuron], block: int):
     NeuronModel.objects.bulk_create(
         [
             NeuronModel(