-
Notifications
You must be signed in to change notification settings - Fork 0
⚡ Bolt: Combine API calls and optimize regex for faster startup #170
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -289,6 +289,10 @@ | |
| FOLDER_CREATION_DELAY = 5 # <--- CHANGED: Increased from 2 to 5 for patience | ||
| MAX_RESPONSE_SIZE = 10 * 1024 * 1024 # 10MB limit | ||
|
|
||
| # Regex patterns (compiled for performance) | ||
| PROFILE_ID_PATTERN = re.compile(r"^[a-zA-Z0-9_-]+$") | ||
| RULE_PATTERN = re.compile(r"^[a-zA-Z0-9.\-_:*\/]+$") | ||
|
|
||
|
|
||
| # --------------------------------------------------------------------------- # | ||
| # 2. Clients | ||
|
|
@@ -398,7 +402,7 @@ | |
|
|
||
|
|
||
| def is_valid_profile_id_format(profile_id: str) -> bool: | ||
| if not re.match(r"^[a-zA-Z0-9_-]+$", profile_id): | ||
| if not PROFILE_ID_PATTERN.match(profile_id): | ||
| return False | ||
| if len(profile_id) > 64: | ||
| return False | ||
|
|
@@ -408,7 +412,7 @@ | |
| def validate_profile_id(profile_id: str, log_errors: bool = True) -> bool: | ||
| if not is_valid_profile_id_format(profile_id): | ||
| if log_errors: | ||
| if not re.match(r"^[a-zA-Z0-9_-]+$", profile_id): | ||
| if not PROFILE_ID_PATTERN.match(profile_id): | ||
| log.error("Invalid profile ID format (contains unsafe characters)") | ||
| elif len(profile_id) > 64: | ||
| log.error("Invalid profile ID length (max 64 chars)") | ||
|
|
@@ -427,7 +431,7 @@ | |
|
|
||
| # Strict whitelist to prevent injection | ||
| # ^[a-zA-Z0-9.\-_:*\/]+$ | ||
| if not re.match(r"^[a-zA-Z0-9.\-_:*\/]+$", rule): | ||
| if not RULE_PATTERN.match(rule): | ||
| return False | ||
|
|
||
| return True | ||
|
|
@@ -516,7 +520,7 @@ | |
| return response | ||
| except (httpx.HTTPError, httpx.TimeoutException) as e: | ||
| if attempt == max_retries - 1: | ||
| if hasattr(e, "response") and e.response is not None: | ||
Check noticeCode scanning / Pylintpython3 (reported by Codacy) Use lazy % formatting in logging functions Note
Use lazy % formatting in logging functions
|
||
| log.debug(f"Response content: {sanitize_for_log(e.response.text)}") | ||
| raise | ||
| wait_time = delay * (2**attempt) | ||
|
|
@@ -583,44 +587,6 @@ | |
| return _cache[url] | ||
|
|
||
|
|
||
| def check_api_access(client: httpx.Client, profile_id: str) -> bool: | ||
| """ | ||
| Verifies API access and Profile existence before starting heavy work. | ||
| Returns True if access is good, False otherwise (with helpful logs). | ||
| """ | ||
| url = f"{API_BASE}/{profile_id}/groups" | ||
| try: | ||
| # We use a raw request here to avoid the automatic retries of _retry_request | ||
| # for auth errors, which are permanent. | ||
| resp = client.get(url) | ||
| resp.raise_for_status() | ||
| return True | ||
| except httpx.HTTPStatusError as e: | ||
| code = e.response.status_code | ||
| if code == 401: | ||
| log.critical( | ||
| f"{Colors.FAIL}❌ Authentication Failed: The API Token is invalid.{Colors.ENDC}" | ||
| ) | ||
| log.critical( | ||
| f"{Colors.FAIL} Please check your token at: https://controld.com/account/manage-account{Colors.ENDC}" | ||
| ) | ||
| elif code == 403: | ||
| log.critical( | ||
| f"{Colors.FAIL}🚫 Access Denied: Token lacks permission for Profile {profile_id}.{Colors.ENDC}" | ||
| ) | ||
| elif code == 404: | ||
| log.critical( | ||
| f"{Colors.FAIL}🔍 Profile Not Found: The ID '{profile_id}' does not exist.{Colors.ENDC}" | ||
| ) | ||
| log.critical( | ||
| f"{Colors.FAIL} Please verify the Profile ID from your Control D Dashboard URL.{Colors.ENDC}" | ||
| ) | ||
| else: | ||
| log.error(f"API Access Check Failed ({code}): {e}") | ||
| return False | ||
| except httpx.RequestError as e: | ||
| log.error(f"Network Error during access check: {e}") | ||
| return False | ||
|
|
||
|
|
||
| def list_existing_folders(client: httpx.Client, profile_id: str) -> Dict[str, str]: | ||
|
|
@@ -629,14 +595,87 @@ | |
| folders = data.get("body", {}).get("groups", []) | ||
| return { | ||
| f["group"].strip(): f["PK"] | ||
| for f in folders | ||
Check warningCode scanning / Pylint (reported by Codacy) Variable name "e" doesn't conform to snake_case naming style Warning
Variable name "e" doesn't conform to snake_case naming style
Check warningCode scanning / Pylintpython3 (reported by Codacy) Variable name "e" doesn't conform to snake_case naming style Warning
Variable name "e" doesn't conform to snake_case naming style
|
||
| if f.get("group") and f.get("PK") | ||
| } | ||
| except (httpx.HTTPError, KeyError) as e: | ||
Check warningCode scanning / Prospector (reported by Codacy) Use lazy % formatting in logging functions (logging-fstring-interpolation) Warning
Use lazy % formatting in logging functions (logging-fstring-interpolation)
Check noticeCode scanning / Pylintpython3 (reported by Codacy) Use lazy % formatting in logging functions Note
Use lazy % formatting in logging functions
|
||
| log.error(f"Failed to list existing folders: {sanitize_for_log(e)}") | ||
| return {} | ||
|
|
||
Check warningCode scanning / Prospector (reported by Codacy) Use lazy % formatting in logging functions (logging-fstring-interpolation) Warning
Use lazy % formatting in logging functions (logging-fstring-interpolation)
Check noticeCode scanning / Pylintpython3 (reported by Codacy) Use lazy % formatting in logging functions Note
Use lazy % formatting in logging functions
|
||
|
|
||
Check warningCode scanning / Pylint (reported by Codacy) Line too long (127/100) Warning
Line too long (127/100)
|
||
| def verify_access_and_get_folders( | ||
| client: httpx.Client, profile_id: str | ||
| ) -> Optional[Dict[str, str]]: | ||
Check warningCode scanning / Prospector (reported by Codacy) Use lazy % formatting in logging functions (logging-fstring-interpolation) Warning
Use lazy % formatting in logging functions (logging-fstring-interpolation)
Check noticeCode scanning / Pylintpython3 (reported by Codacy) Use lazy % formatting in logging functions Note
Use lazy % formatting in logging functions
|
||
| """ | ||
|
||
| Combines access check and fetching existing folders into a single request. | ||
| Returns: | ||
| Dict of {folder_name: folder_id} if successful. | ||
Check warningCode scanning / Prospector (reported by Codacy) Use lazy % formatting in logging functions (logging-fstring-interpolation) Warning
Use lazy % formatting in logging functions (logging-fstring-interpolation)
Check noticeCode scanning / Pylintpython3 (reported by Codacy) Use lazy % formatting in logging functions Note
Use lazy % formatting in logging functions
|
||
| None if access is denied or request fails after retries. | ||
|
||
| """ | ||
| url = f"{API_BASE}/{profile_id}/groups" | ||
Check warningCode scanning / Prospector (reported by Codacy) Use lazy % formatting in logging functions (logging-fstring-interpolation) Warning
Use lazy % formatting in logging functions (logging-fstring-interpolation)
Check noticeCode scanning / Pylintpython3 (reported by Codacy) Use lazy % formatting in logging functions Note
Use lazy % formatting in logging functions
|
||
|
|
||
Check warningCode scanning / Pylint (reported by Codacy) Line too long (111/100) Warning
Line too long (111/100)
|
||
| for attempt in range(MAX_RETRIES): | ||
| try: | ||
| # Don't use _api_get because we need custom error handling for 4xx | ||
| resp = client.get(url) | ||
| resp.raise_for_status() | ||
Check warningCode scanning / Pylint (reported by Codacy) Variable name "e" doesn't conform to snake_case naming style Warning
Variable name "e" doesn't conform to snake_case naming style
Check warningCode scanning / Pylintpython3 (reported by Codacy) Variable name "e" doesn't conform to snake_case naming style Warning
Variable name "e" doesn't conform to snake_case naming style
|
||
|
|
||
| # Success! Parse and return | ||
| try: | ||
| data = resp.json() | ||
| folders = data.get("body", {}).get("groups", []) | ||
| return { | ||
| f["group"].strip(): f["PK"] | ||
| for f in folders | ||
| if f.get("group") and f.get("PK") | ||
| } | ||
| except (KeyError, ValueError) as e: | ||
Check warningCode scanning / Pylint (reported by Codacy) Variable name "e" doesn't conform to snake_case naming style Warning
Variable name "e" doesn't conform to snake_case naming style
|
||
| log.error(f"Failed to parse folders data: {e}") | ||
Check warningCode scanning / Prospector (reported by Codacy) Use lazy % formatting in logging functions (logging-fstring-interpolation) Warning
Use lazy % formatting in logging functions (logging-fstring-interpolation)
Check noticeCode scanning / Pylintpython3 (reported by Codacy) Use lazy % formatting in logging functions Note
Use lazy % formatting in logging functions
|
||
| return None | ||
|
|
||
| except httpx.HTTPStatusError as e: | ||
| code = e.response.status_code | ||
| if code in (401, 403, 404): | ||
| if code == 401: | ||
| log.critical( | ||
| f"{Colors.FAIL}❌ Authentication Failed: The API Token is invalid.{Colors.ENDC}" | ||
Check warningCode scanning / Pylint (reported by Codacy) Line too long (103/100) Warning
Line too long (103/100)
Check warningCode scanning / Pylintpython3 (reported by Codacy) Line too long (103/100) Warning
Line too long (103/100)
|
||
| ) | ||
| log.critical( | ||
| f"{Colors.FAIL} Please check your token at: https://controld.com/account/manage-account{Colors.ENDC}" | ||
| ) | ||
| elif code == 403: | ||
| log.critical( | ||
| f"{Colors.FAIL}🚫 Access Denied: Token lacks permission for Profile {profile_id}.{Colors.ENDC}" | ||
|
||
| ) | ||
| elif code == 404: | ||
| log.critical( | ||
| f"{Colors.FAIL}🔍 Profile Not Found: The ID '{profile_id}' does not exist.{Colors.ENDC}" | ||
|
||
| ) | ||
| log.critical( | ||
| f"{Colors.FAIL} Please verify the Profile ID from your Control D Dashboard URL.{Colors.ENDC}" | ||
| ) | ||
| return None | ||
|
Comment on lines
638
to
657
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This block for handling specific HTTP status codes (401, 403, 404) is quite large and contains detailed, user-facing log messages. To improve the readability and maintainability of You could define a new helper function like this elsewhere in the file: def _log_api_access_error(code: int, profile_id: str) -> None:
"""Logs a critical, user-facing error for common API access issues."""
if code == 401:
log.critical(
f"{Colors.FAIL}❌ Authentication Failed: The API Token is invalid.{Colors.ENDC}"
)
log.critical(
f"{Colors.FAIL} Please check your token at: https://controld.com/account/manage-account{Colors.ENDC}"
)
elif code == 403:
log.critical(
f"{Colors.FAIL}🚫 Access Denied: Token lacks permission for Profile {profile_id}.{Colors.ENDC}"
)
elif code == 404:
log.critical(
f"{Colors.FAIL}🔍 Profile Not Found: The ID '{profile_id}' does not exist.{Colors.ENDC}"
)
log.critical(
f"{Colors.FAIL} Please verify the Profile ID from your Control D Dashboard URL.{Colors.ENDC}"
) if code in (401, 403, 404):
_log_api_access_error(code, profile_id)
return None |
||
|
|
||
| # For 5xx errors, retry | ||
| if attempt == MAX_RETRIES - 1: | ||
| log.error(f"API Request Failed ({code}): {e}") | ||
|
||
| return None | ||
|
|
||
|
Comment on lines
636
to
663
|
||
| except httpx.RequestError as e: | ||
| if attempt == MAX_RETRIES - 1: | ||
| log.error(f"Network Error: {e}") | ||
|
||
| return None | ||
|
|
||
| # Wait before retry (exponential backoff) | ||
| wait_time = RETRY_DELAY * (2**attempt) | ||
| log.warning( | ||
| f"Request failed (attempt {attempt + 1}/{MAX_RETRIES}). Retrying in {wait_time}s..." | ||
| ) | ||
|
Comment on lines
+671
to
+673
|
||
| time.sleep(wait_time) | ||
|
|
||
| return None | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This |
||
|
|
||
|
|
||
| def get_all_existing_rules( | ||
| client: httpx.Client, | ||
| profile_id: str, | ||
|
|
@@ -704,7 +743,7 @@ | |
| if not validate_folder_data(js, url): | ||
| raise KeyError(f"Invalid folder data from {sanitize_for_log(url)}") | ||
| return js | ||
|
|
||
Check warningCode scanning / Prospector (reported by Codacy) Use lazy % formatting in logging functions (logging-fstring-interpolation) Warning
Use lazy % formatting in logging functions (logging-fstring-interpolation)
|
||
|
|
||
| def warm_up_cache(urls: Sequence[str]) -> None: | ||
| urls = list(set(urls)) | ||
|
|
@@ -1142,11 +1181,12 @@ | |
| # Initial client for getting existing state AND processing folders | ||
| # Optimization: Reuse the same client session to keep TCP connections alive | ||
| with _api_client() as client: | ||
| # Check for API access problems first (401/403/404) | ||
| if not check_api_access(client, profile_id): | ||
| # Check for API access problems first (401/403/404) AND get existing folders in one go | ||
| existing_folders = verify_access_and_get_folders(client, profile_id) | ||
| if existing_folders is None: | ||
| # Access check failed (logged already) | ||
| return False | ||
|
|
||
| existing_folders = list_existing_folders(client, profile_id) | ||
| if not no_delete: | ||
| deletion_occurred = False | ||
| for folder_data in folder_data_list: | ||
|
|
||
Check warning
Code scanning / Pylint (reported by Codacy)
Wrong hanging indentation before block (add 4 spaces). Warning