This commit is contained in:
thot-experiment 2025-05-07 18:54:39 -07:00
parent ffc6b610b6
commit 33da9e40c7
2 changed files with 30 additions and 31 deletions

View File

@ -95,7 +95,7 @@ import logging
import time import time
import io import io
import socket import socket
from typing import Dict, Type, Optional, Any, TypeVar, Generic, Callable, Tuple, List, Union from typing import Dict, Type, Optional, Any, TypeVar, Generic, Callable, Tuple
from enum import Enum from enum import Enum
import json import json
import requests import requests
@ -246,10 +246,10 @@ class ApiClient:
def _check_connectivity(self, target_url: str) -> Dict[str, bool]: def _check_connectivity(self, target_url: str) -> Dict[str, bool]:
""" """
Check connectivity to determine if network issues are local or server-related. Check connectivity to determine if network issues are local or server-related.
Args: Args:
target_url: URL to check connectivity to target_url: URL to check connectivity to
Returns: Returns:
Dictionary with connectivity status details Dictionary with connectivity status details
""" """
@ -259,12 +259,12 @@ class ApiClient:
"is_local_issue": False, "is_local_issue": False,
"is_api_issue": False "is_api_issue": False
} }
# First check basic internet connectivity using a reliable external site # First check basic internet connectivity using a reliable external site
try: try:
# Use a reliable external domain for checking basic connectivity # Use a reliable external domain for checking basic connectivity
check_response = requests.get("https://www.google.com", check_response = requests.get("https://www.google.com",
timeout=5.0, timeout=5.0,
verify=self.verify_ssl) verify=self.verify_ssl)
if check_response.status_code < 500: if check_response.status_code < 500:
results["internet_accessible"] = True results["internet_accessible"] = True
@ -272,13 +272,13 @@ class ApiClient:
results["internet_accessible"] = False results["internet_accessible"] = False
results["is_local_issue"] = True results["is_local_issue"] = True
return results return results
# Now check API server connectivity # Now check API server connectivity
try: try:
# Extract domain from the target URL to do a simpler health check # Extract domain from the target URL to do a simpler health check
parsed_url = urlparse(target_url) parsed_url = urlparse(target_url)
api_base = f"{parsed_url.scheme}://{parsed_url.netloc}" api_base = f"{parsed_url.scheme}://{parsed_url.netloc}"
# Try to reach the API domain # Try to reach the API domain
api_response = requests.get(f"{api_base}/health", timeout=5.0, verify=self.verify_ssl) api_response = requests.get(f"{api_base}/health", timeout=5.0, verify=self.verify_ssl)
if api_response.status_code < 500: if api_response.status_code < 500:
@ -290,7 +290,7 @@ class ApiClient:
results["api_accessible"] = False results["api_accessible"] = False
# If we can reach the internet but not the API, it's an API issue # If we can reach the internet but not the API, it's an API issue
results["is_api_issue"] = True results["is_api_issue"] = True
return results return results
def request( def request(
@ -372,17 +372,17 @@ class ApiClient:
) )
# Check if we should retry based on status code # Check if we should retry based on status code
if (response.status_code in self.retry_status_codes and if (response.status_code in self.retry_status_codes and
retry_count < self.max_retries): retry_count < self.max_retries):
# Calculate delay with exponential backoff # Calculate delay with exponential backoff
delay = self.retry_delay * (self.retry_backoff_factor ** retry_count) delay = self.retry_delay * (self.retry_backoff_factor ** retry_count)
logging.warning( logging.warning(
f"Request failed with status {response.status_code}. " f"Request failed with status {response.status_code}. "
f"Retrying in {delay:.2f}s ({retry_count + 1}/{self.max_retries})" f"Retrying in {delay:.2f}s ({retry_count + 1}/{self.max_retries})"
) )
time.sleep(delay) time.sleep(delay)
return self.request( return self.request(
method=method, method=method,
@ -415,7 +415,7 @@ class ApiClient:
response_headers=dict(response.headers), response_headers=dict(response.headers),
response_content=response_content_to_log response_content=response_content_to_log
) )
except requests.ConnectionError as e: except requests.ConnectionError as e:
error_message = f"ConnectionError: {str(e)}" error_message = f"ConnectionError: {str(e)}"
request_logger.log_request_response( request_logger.log_request_response(
@ -428,18 +428,18 @@ class ApiClient:
if retry_count >= self.max_retries: if retry_count >= self.max_retries:
# Check connectivity to determine if it's a local or API issue # Check connectivity to determine if it's a local or API issue
connectivity = self._check_connectivity(self.base_url) connectivity = self._check_connectivity(self.base_url)
if connectivity["is_local_issue"]: if connectivity["is_local_issue"]:
raise LocalNetworkError( raise LocalNetworkError(
f"Unable to connect to the API server due to local network issues. " "Unable to connect to the API server due to local network issues. "
f"Please check your internet connection and try again." "Please check your internet connection and try again."
) from e ) from e
elif connectivity["is_api_issue"]: elif connectivity["is_api_issue"]:
raise ApiServerError( raise ApiServerError(
f"The API server at {self.base_url} is currently unreachable. " f"The API server at {self.base_url} is currently unreachable. "
f"The service may be experiencing issues. Please try again later." f"The service may be experiencing issues. Please try again later."
) from e ) from e
# If we haven't exhausted retries yet, retry the request # If we haven't exhausted retries yet, retry the request
if retry_count < self.max_retries: if retry_count < self.max_retries:
delay = self.retry_delay * (self.retry_backoff_factor ** retry_count) delay = self.retry_delay * (self.retry_backoff_factor ** retry_count)
@ -459,7 +459,7 @@ class ApiClient:
multipart_parser=multipart_parser, multipart_parser=multipart_parser,
retry_count=retry_count + 1, retry_count=retry_count + 1,
) )
# If we've exhausted retries and didn't identify the specific issue, # If we've exhausted retries and didn't identify the specific issue,
# raise a generic exception # raise a generic exception
final_error_message = ( final_error_message = (
@ -562,7 +562,7 @@ class ApiClient:
# Retry if the status code is in our retry list and we haven't exhausted retries # Retry if the status code is in our retry list and we haven't exhausted retries
if (status_code in self.retry_status_codes and if (status_code in self.retry_status_codes and
retry_count < self.max_retries): retry_count < self.max_retries):
delay = self.retry_delay * (self.retry_backoff_factor ** retry_count) delay = self.retry_delay * (self.retry_backoff_factor ** retry_count)
logging.warning( logging.warning(
f"HTTP error {status_code}. " f"HTTP error {status_code}. "
@ -580,7 +580,7 @@ class ApiClient:
multipart_parser=multipart_parser, multipart_parser=multipart_parser,
retry_count=retry_count + 1, retry_count=retry_count + 1,
) )
# Specific error messages for common status codes for user display # Specific error messages for common status codes for user display
if status_code == 401: if status_code == 401:
user_display_error_message = "Unauthorized: Please login first to use this node." user_display_error_message = "Unauthorized: Please login first to use this node."
@ -615,7 +615,7 @@ class ApiClient:
retry_backoff_factor: float = 2.0, retry_backoff_factor: float = 2.0,
): ):
"""Upload a file to the API with retry logic. """Upload a file to the API with retry logic.
Args: Args:
upload_url: The URL to upload to upload_url: The URL to upload to
file: Either a file path string, BytesIO object, or tuple of (file_path, filename) file: Either a file path string, BytesIO object, or tuple of (file_path, filename)
@ -799,7 +799,7 @@ class SynchronousOperation(Generic[T, R]):
self.max_retries = max_retries self.max_retries = max_retries
self.retry_delay = retry_delay self.retry_delay = retry_delay
self.retry_backoff_factor = retry_backoff_factor self.retry_backoff_factor = retry_backoff_factor
def execute(self, client: Optional[ApiClient] = None) -> R: def execute(self, client: Optional[ApiClient] = None) -> R:
"""Execute the API operation using the provided client or create one with retry support""" """Execute the API operation using the provided client or create one with retry support"""
try: try:
@ -987,7 +987,7 @@ class PollingOperation(Generic[T, R]):
poll_count = 0 poll_count = 0
consecutive_errors = 0 consecutive_errors = 0
max_consecutive_errors = min(5, self.max_retries * 2) # Limit consecutive errors max_consecutive_errors = min(5, self.max_retries * 2) # Limit consecutive errors
if self.progress_extractor: if self.progress_extractor:
progress = utils.ProgressBar(PROGRESS_BAR_MAX) progress = utils.ProgressBar(PROGRESS_BAR_MAX)
@ -1023,7 +1023,7 @@ class PollingOperation(Generic[T, R]):
# Parse response # Parse response
response_obj = self.poll_endpoint.response_model.model_validate(resp) response_obj = self.poll_endpoint.response_model.model_validate(resp)
# Check if task is complete # Check if task is complete
status = self._check_task_status(response_obj) status = self._check_task_status(response_obj)
logging.debug(f"[DEBUG] Task Status: {status}") logging.debug(f"[DEBUG] Task Status: {status}")
@ -1060,14 +1060,14 @@ class PollingOperation(Generic[T, R]):
raise Exception( raise Exception(
f"Polling aborted after {consecutive_errors} consecutive network errors: {str(e)}" f"Polling aborted after {consecutive_errors} consecutive network errors: {str(e)}"
) from e ) from e
# Log the error but continue polling # Log the error but continue polling
logging.warning( logging.warning(
f"Network error during polling (attempt {poll_count}/{self.max_poll_attempts}): {str(e)}. " f"Network error during polling (attempt {poll_count}/{self.max_poll_attempts}): {str(e)}. "
f"Will retry in {self.poll_interval} seconds." f"Will retry in {self.poll_interval} seconds."
) )
time.sleep(self.poll_interval) time.sleep(self.poll_interval)
except Exception as e: except Exception as e:
# For other errors, increment count and potentially abort # For other errors, increment count and potentially abort
consecutive_errors += 1 consecutive_errors += 1
@ -1075,16 +1075,16 @@ class PollingOperation(Generic[T, R]):
raise Exception( raise Exception(
f"Polling aborted after {consecutive_errors} consecutive errors: {str(e)}" f"Polling aborted after {consecutive_errors} consecutive errors: {str(e)}"
) from e ) from e
logging.error(f"[DEBUG] Polling error: {str(e)}") logging.error(f"[DEBUG] Polling error: {str(e)}")
logging.warning( logging.warning(
f"Error during polling (attempt {poll_count}/{self.max_poll_attempts}): {str(e)}. " f"Error during polling (attempt {poll_count}/{self.max_poll_attempts}): {str(e)}. "
f"Will retry in {self.poll_interval} seconds." f"Will retry in {self.poll_interval} seconds."
) )
time.sleep(self.poll_interval) time.sleep(self.poll_interval)
# If we've exhausted all polling attempts # If we've exhausted all polling attempts
raise Exception( raise Exception(
f"Polling timed out after {poll_count} attempts ({poll_count * self.poll_interval} seconds). " f"Polling timed out after {poll_count} attempts ({poll_count * self.poll_interval} seconds). "
f"The operation may still be running on the server but is taking longer than expected." f"The operation may still be running on the server but is taking longer than expected."
) )

View File

@ -123,4 +123,3 @@ if __name__ == '__main__':
response_status_code=200, response_status_code=200,
response_content=b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR...' # Sample binary data response_content=b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR...' # Sample binary data
) )
print(f"Test logs should be in: {get_log_directory()}")