functional tests: wait for sync and wait for height helpers, expose timeouts to environment variables, timeouts for stallment to avoid misleading breaks #1038
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -12,31 +12,58 @@ | |
| `add_node_settings`. | ||
| """ | ||
|
|
||
| import contextlib | ||
| import copy | ||
| import inspect | ||
| import os | ||
| import random | ||
| import re | ||
| import shutil | ||
| import signal | ||
| import socket | ||
| import subprocess | ||
| import sys | ||
| import time | ||
| from datetime import datetime, timezone | ||
| from enum import Enum | ||
| from typing import Any, Dict, List, Optional, Pattern, Tuple | ||
|
|
||
| from requests.exceptions import RequestException | ||
| from test_framework.crypto.pkcs8 import ( | ||
| create_pkcs8_private_key, | ||
| create_pkcs8_self_signed_certificate, | ||
| ) | ||
| from test_framework.daemon import ConfigP2P | ||
| from test_framework.electrum import ConfigElectrum, ConfigTls | ||
| from test_framework.node import Node, NodeType | ||
| from test_framework.rpc import ConfigRPC | ||
| from test_framework.util import Utility | ||
|
|
||
| SYNC_TIMEOUT = float(os.environ.get("FLORESTA_SYNC_TIMEOUT", "120")) | ||
| PEER_CONNECTION_TIMEOUT = float( | ||
| os.environ.get("FLORESTA_PEER_CONNECTION_TIMEOUT", "30") | ||
| ) | ||
|
|
||
|
|
||
| def wait_until(predicate, *, timeout=SYNC_TIMEOUT, interval=0.5): | ||
|
Collaborator
There was a problem hiding this comment. Move this to |
||
| """Wait until ``predicate()`` returns True, or raise after *timeout* seconds. | ||
|
|
||
| This is a general-purpose primitive inspired by Bitcoin Core's | ||
| ``wait_until_helper_internal``. Domain-specific helpers such as | ||
| ``FlorestaTestFramework.wait_for_sync`` build on top of this for | ||
| cases that need stale-state detection or RPC error tolerance. | ||
| """ | ||
| deadline = time.time() + timeout | ||
| while time.time() < deadline: | ||
| if predicate(): | ||
| return | ||
| time.sleep(interval) | ||
|
|
||
| source = inspect.getsource(predicate) | ||
| raise AssertionError( | ||
| f"wait_until() timed out after {timeout}s. Predicate:\n{source}" | ||
| ) | ||
|
|
||
|
|
||
| # pylint: disable=too-many-public-methods | ||
| class FlorestaTestFramework: | ||
|
|
@@ -293,8 +320,8 @@ def wait_for_peers_connections( | |
| Wait for two peers to connect/disconnect to each other. | ||
| """ | ||
| attempts = 0 | ||
| deadline = time.time() + PEER_CONNECTION_TIMEOUT | ||
| while time.time() < deadline: | ||
| if self.check_connection(peer_one, peer_two, is_connected): | ||
| self.log.debug( | ||
| f"Peers {peer_one.variant} and {peer_two.variant} are in the expected " | ||
|
|
@@ -329,6 +356,103 @@ def wait_for_peers_connections( | |
| f"connection state within the timeout. Expected connected: {is_connected}." | ||
| ) | ||
|
|
||
| def wait_for_sync( | ||
|
jaoleal marked this conversation as resolved.
|
||
| self, node: Node, target_height: int, stale_timeout: float = SYNC_TIMEOUT | ||
| ): | ||
| """ | ||
| Wait until a node is synced to the target height and out of IBD with stale detection. | ||
|
|
||
| If the node stops making progress for ``stale_timeout`` seconds, the test fails. | ||
| """ | ||
| last_progress_time = time.time() | ||
| prev_height = None | ||
| prev_validated = None | ||
|
|
||
| while True: | ||
|
Collaborator
There was a problem hiding this comment. Using while true is not a good idea. The ideal approach is to handle the timeout case here, because inside the while, if that happens, it can exit by returning immediately. If it keeps going after the timeout, it ends up causing an error. In this PR here: #897 I added a wait_until helper that already does this automatically, so it makes things easier. |
||
| try: | ||
| info = node.rpc.get_blockchain_info() | ||
| except RequestException as exc: | ||
| self.log.debug(f"Node '{node.variant}' RPC error (will retry): {exc}") | ||
| stale_elapsed = time.time() - last_progress_time | ||
| if stale_elapsed >= stale_timeout: | ||
| raise AssertionError( | ||
| f"Node '{node.variant}' RPC unreachable for {stale_timeout}s " | ||
| f"trying to sync to height {target_height}: {exc}" | ||
| ) from exc | ||
| time.sleep(1) | ||
| continue | ||
|
|
||
| if info["height"] == target_height and not info["ibd"]: | ||
| self.log.debug( | ||
| f"Node '{node.variant}' synced to height {target_height}" | ||
| ) | ||
| return | ||
|
|
||
| cur_height = info["height"] | ||
| cur_validated = info.get("validated") | ||
|
|
||
| if cur_height != prev_height or cur_validated != prev_validated: | ||
| last_progress_time = time.time() | ||
| prev_height = cur_height | ||
| prev_validated = cur_validated | ||
|
|
||
| stale_elapsed = time.time() - last_progress_time | ||
| if stale_elapsed >= stale_timeout: | ||
| raise AssertionError( | ||
| f"Node '{node.variant}' stalled for {stale_timeout}s trying to " | ||
| f"sync to height {target_height}: height={info['height']}, " | ||
| f"validated={cur_validated}, ibd={info['ibd']}" | ||
| ) | ||
|
|
||
| time.sleep(1) | ||
|
|
||
| def wait_for_height( | ||
| self, node: Node, target_height: int, stale_timeout: float = SYNC_TIMEOUT | ||
| ): | ||
| """ | ||
| Wait until a node reaches the target height, regardless of IBD state. | ||
|
|
||
| Use this instead of ``wait_for_sync`` when the node only needs headers | ||
| sync(e.g. syncing from bitcoind which doesn't offer utreexo proofs). | ||
|
|
||
| Uses the same stale-state detection as ``wait_for_sync``. | ||
| """ | ||
| last_progress_time = time.time() | ||
| prev_height = None | ||
|
|
||
| while True: | ||
| try: | ||
| info = node.rpc.get_blockchain_info() | ||
| except RequestException as exc: | ||
| self.log.debug(f"Node '{node.variant}' RPC error (will retry): {exc}") | ||
| stale_elapsed = time.time() - last_progress_time | ||
| if stale_elapsed >= stale_timeout: | ||
| raise AssertionError( | ||
| f"Node '{node.variant}' RPC unreachable for {stale_timeout}s " | ||
| f"trying to reach height {target_height}: {exc}" | ||
| ) from exc | ||
| time.sleep(1) | ||
| continue | ||
|
|
||
| cur_height = info["height"] | ||
| if cur_height == target_height: | ||
| self.log.debug(f"Node '{node.variant}' reached height {target_height}") | ||
| return | ||
|
|
||
| if cur_height != prev_height: | ||
| last_progress_time = time.time() | ||
| prev_height = cur_height | ||
|
|
||
| stale_elapsed = time.time() - last_progress_time | ||
| if stale_elapsed >= stale_timeout: | ||
| raise AssertionError( | ||
| f"Node '{node.variant}' stalled for {stale_timeout}s trying to " | ||
| f"reach height {target_height}: height={cur_height}, " | ||
| f"ibd={info['ibd']}" | ||
| ) | ||
|
|
||
| time.sleep(1) | ||
|
|
||
| def connect_nodes( | ||
| self, | ||
| peer_one: Node, | ||
|
|
||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.