Merge pull request #231 from opentensor/fix/thewhaleking/bug-fixes-1.5.10

bug fixes 1.5.10

Author: thewhaleking

.github/workflows/check-btcli-tests.yml

@@ -61,10 +61,10 @@ jobs:
           echo "Current labels: $LABELS"
           if echo "$LABELS" | grep -q "run-bittensor-cli-tests"; then
             echo "run-cli-tests=true" >> $GITHUB_ENV
-            echo "::set-output name=run-cli-tests::true"
+            echo "run-cli-tests=true" >> $GITHUB_OUTPUT
           else
             echo "run-cli-tests=false" >> $GITHUB_ENV
-            echo "::set-output name=run-cli-tests::false"
+            echo "run-cli-tests=false" >> $GITHUB_OUTPUT
           fi
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -91,7 +91,7 @@ jobs:
         id: get-tests
         run: |
           test_files=$(find ${{ github.workspace }}/btcli/tests/e2e_tests -name "test*.py" | jq -R -s -c 'split("\n") | map(select(. != ""))')
-          echo "::set-output name=test-files::$test_files"
+          echo "test-files=$test_files" >> $GITHUB_OUTPUT
         shell: bash
 
   pull-docker-image:
@@ -147,45 +147,47 @@ jobs:
       - name: Check-out repository
         uses: actions/checkout@v4
 
-      - name: Install dependencies
+      - name: Install system dependencies
         run: |
           sudo apt-get update &&
           sudo apt-get install -y clang curl libssl-dev llvm libudev-dev protobuf-compiler
 
-      - name: Create Python virtual environment
-        working-directory: ${{ github.workspace }}
-        run: python3 -m venv ${{ github.workspace }}/venv
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
 
       - name: Clone Bittensor CLI repo
-        working-directory: ${{ github.workspace }}
         run: git clone https://github.com/opentensor/btcli.git
 
-      - name: Setup Bittensor-cli from cloned repo
+      - name: Checkout btcli staging branch
         working-directory: ${{ github.workspace }}/btcli
         run: |
-          source ${{ github.workspace }}/venv/bin/activate
           git checkout staging
           git fetch origin staging
-          python3 -m pip install --upgrade pip uv
-          uv pip install '.[dev]'
-          uv pip install pytest
+
+      - name: Install btcli dependencies
+        working-directory: ${{ github.workspace }}/btcli
+        run: uv pip install --system '.[dev]'
 
       - name: Clone async-substrate-interface repo
         run: git clone https://github.com/opentensor/async-substrate-interface.git
 
-      - name: Checkout PR async-substrate-interface repo
+      - name: Checkout PR branch in async-substrate-interface
         working-directory: ${{ github.workspace }}/async-substrate-interface
         run: |
           git fetch origin ${{ github.event.pull_request.head.ref }}
           git checkout ${{ github.event.pull_request.head.ref }}
           echo "Current branch: $(git rev-parse --abbrev-ref HEAD)"
 
-      - name: Install async-substrate-interface package
+      - name: Install async-substrate-interface with dev dependencies
         working-directory: ${{ github.workspace }}/async-substrate-interface
         run: |
-          source ${{ github.workspace }}/venv/bin/activate
-          python3 -m pip uninstall async-substrate-interface -y
-          uv pip install .
+          uv pip uninstall --system async-substrate-interface || true
+          uv pip install --system '.[dev]'
 
       - name: Download Cached Docker Image
         uses: actions/download-artifact@v4
@@ -195,10 +197,8 @@ jobs:
       - name: Load Docker Image
         run: docker load -i subtensor-localnet.tar
 
-      - name: Run tests
-        run: |
-          source ${{ github.workspace }}/venv/bin/activate
-          pytest ${{ matrix.test-file }} -s
+      - name: Run e2e tests
+        run: pytest ${{ matrix.test-file }} -s
 
 
   run-unit-test:
@@ -210,46 +210,47 @@ jobs:
       - name: Check-out repository
         uses: actions/checkout@v4
 
-      - name: Install dependencies
+      - name: Install system dependencies
         run: |
           sudo apt-get update &&
           sudo apt-get install -y clang curl libssl-dev llvm libudev-dev protobuf-compiler
 
-      - name: Create Python virtual environment
-        working-directory: ${{ github.workspace }}
-        run: python3 -m venv venv
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
 
       - name: Clone Bittensor CLI repo
-        working-directory: ${{ github.workspace }}
         run: git clone https://github.com/opentensor/btcli.git
 
-      - name: Setup Bittensor SDK from cloned repo
+      - name: Checkout btcli staging branch
         working-directory: ${{ github.workspace }}/btcli
         run: |
-          source ${{ github.workspace }}/venv/bin/activate
           git checkout staging
           git fetch origin staging
-          python3 -m pip install --upgrade pip uv
-          uv pip install '.[dev]'
+
+      - name: Install btcli dependencies
+        working-directory: ${{ github.workspace }}/btcli
+        run: uv pip install --system '.[dev]'
 
       - name: Clone async-substrate-interface repo
         run: git clone https://github.com/opentensor/async-substrate-interface.git
 
-      - name: Checkout PR branch in async-substrate-interface repo
+      - name: Checkout PR branch in async-substrate-interface
         working-directory: ${{ github.workspace }}/async-substrate-interface
         run: |
           git fetch origin ${{ github.event.pull_request.head.ref }}
           git checkout ${{ github.event.pull_request.head.ref }}
           echo "Current branch: $(git rev-parse --abbrev-ref HEAD)"
 
-      - name: Install /async-substrate-interface package
+      - name: Install async-substrate-interface with dev dependencies
         working-directory: ${{ github.workspace }}/async-substrate-interface
         run: |
-          source ${{ github.workspace }}/venv/bin/activate
-          pip uninstall async-substrate-interface -y
-          uv pip install .
+          uv pip uninstall --system async-substrate-interface || true
+          uv pip install --system '.[dev]'
 
-      - name: Run SDK unit tests
-        run: |
-          source ${{ github.workspace }}/venv/bin/activate
-          pytest ${{ github.workspace }}/btcli/tests/unit_tests
+      - name: Run BTCLI unit tests
+        run: pytest ${{ github.workspace }}/btcli/tests/unit_tests

async_substrate_interface/async_substrate.py

@@ -793,7 +793,7 @@ async def _handler(self, ws: ClientConnection) -> Union[None, Exception]:
 
     async def __aexit__(self, exc_type, exc_val, exc_tb):
         if self.shutdown_timer is not None:
-            if not self.state != State.CONNECTING:
+            if self.state != State.CONNECTING:
                 if self._exit_task is not None:
                     self._exit_task.cancel()
                     try:
@@ -802,6 +802,7 @@ async def __aexit__(self, exc_type, exc_val, exc_tb):
                         pass
                 if self.ws is not None:
                     self._exit_task = asyncio.create_task(self._exit_with_timer())
+                self._attempts = 0
 
     async def _exit_with_timer(self):
         """
@@ -952,7 +953,7 @@ async def unsubscribe(
             original_id = get_next_id()
             while original_id in self._in_use_ids:
                 original_id = get_next_id()
-            del self._received_subscriptions[subscription_id]
+            self._received_subscriptions.pop(subscription_id, None)
 
         to_send = {
             "jsonrpc": "2.0",
@@ -1000,11 +1001,12 @@ async def retrieve(self, item_id: str) -> Optional[dict]:
                 elif isinstance((e := self._send_recv_task.result()), Exception):
                     logger.exception(f"Websocket sending exception: {e}")
                     raise e
-        await asyncio.sleep(0.01)
         return None
 
 
 class AsyncSubstrateInterface(SubstrateMixin):
+    ws: "Websocket"
+
     def __init__(
         self,
         url: str,

tests/e2e_tests/test_substrate_addons.py

@@ -1,5 +1,6 @@
 import subprocess
 import time
+import sys
 
 import pytest
 
@@ -13,6 +14,63 @@
 from tests.helpers.settings import ARCHIVE_ENTRYPOINT, LATENT_LITE_ENTRYPOINT
 
 
+def wait_for_output(process, target_string, timeout=60):
+    """
+    Wait for a specific string to appear in the subprocess stdout.
+
+    Args:
+        process: subprocess.Popen object
+        target_string: String to wait for in stdout
+        timeout: Maximum time to wait in seconds
+
+    Returns:
+        bool: True if string was found, False if timeout occurred
+    """
+    import time
+
+    start_time = time.time()
+
+    # Make stdout non-blocking on Unix systems
+    if sys.platform != "win32":
+        import fcntl
+        import os
+
+        flags = fcntl.fcntl(process.stdout, fcntl.F_GETFL)
+        fcntl.fcntl(process.stdout, fcntl.F_SETFL, flags | os.O_NONBLOCK)
+
+    buffer = ""
+    while time.time() - start_time < timeout:
+        try:
+            # Read available data
+            chunk = process.stdout.read(1024)
+            if chunk:
+                chunk_str = chunk.decode("utf-8", errors="ignore")
+                buffer += chunk_str
+                print(chunk_str, end="", flush=True)  # Echo output for visibility
+
+                if target_string in buffer:
+                    return True
+            else:
+                # No data available, sleep briefly
+                time.sleep(0.1)
+        except (BlockingIOError, TypeError):
+            # No data available yet
+            time.sleep(0.1)
+
+        # Check if process has terminated
+        if process.poll() is not None:
+            # Process ended, read remaining output
+            remaining = process.stdout.read()
+            if remaining:
+                remaining_str = remaining.decode("utf-8", errors="ignore")
+                print(remaining_str, end="", flush=True)
+                if target_string in remaining_str:
+                    return True
+            return False
+
+    return False
+
+
 @pytest.fixture(scope="function")
 def docker_containers():
     processes = (
@@ -39,7 +97,12 @@ def single_local_chain():
 
 
 def test_retry_sync_substrate(single_local_chain):
-    time.sleep(10)
+    # Wait for the Docker container to be ready
+    if not wait_for_output(single_local_chain.process, "Imported #1", timeout=60):
+        raise TimeoutError(
+            "Docker container did not start properly - 'Imported #1' not found in output"
+        )
+
     with RetrySyncSubstrate(
         single_local_chain.uri, fallback_chains=[LATENT_LITE_ENTRYPOINT]
     ) as substrate:
@@ -52,13 +115,14 @@ def test_retry_sync_substrate(single_local_chain):
             time.sleep(2)
 
 
-@pytest.mark.skip(
-    "There's an issue with this running in the GitHub runner, "
-    "where it seemingly cannot connect to the docker container. "
-    "It does run locally, however."
-)
 def test_retry_sync_substrate_max_retries(docker_containers):
-    time.sleep(10)
+    # Wait for both Docker containers to be ready
+    for i, container in enumerate(docker_containers):
+        if not wait_for_output(container.process, "Imported #1", timeout=60):
+            raise TimeoutError(
+                f"Docker container {i} did not start properly - 'Imported #1' not found in output"
+            )
+
     with RetrySyncSubstrate(
         docker_containers[0].uri, fallback_chains=[docker_containers[1].uri]
     ) as substrate: