-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathconfig.py
More file actions
296 lines (236 loc) · 13 KB
/
Copy pathconfig.py
File metadata and controls
296 lines (236 loc) · 13 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
"""
Configuration Module - Environment Variables and Application Settings
This module handles all application configuration including environment variable loading,
type casting, validation, and setting default values. It provides centralized configuration
management for the VLR.GG API scraping application.
Features:
- Environment File Loading: Automatically loads .env files from various locations
- Type Casting: Safely converts environment variables to appropriate Python types
- Validation: Validates configuration values and provides warnings for issues
- Default Values: Provides sensible defaults for all configuration options
- Rate Limiting: Configures request delays and concurrent request limits
- HTTP Settings: Manages headers, timeouts, and client configuration
Notes:
- Environment variables override default values
- Boolean values support multiple formats (true/1/yes/y)
- Invalid type casting falls back to default values
- Configuration validation occurs at module load time
Author: Metehan Şenyer
Version: 1.0
"""
import os
from pathlib import Path
from typing import Any
from dotenv import load_dotenv
from utils.logger import logger
# Default configuration constants for validation and fallback
DEFAULT_TIMEOUT = 10 # HTTP request timeout (seconds) - reasonable balance between performance and reliability
DEFAULT_MIN_DELAY = 0.5 # Minimum request delay (seconds) - to reduce VLR.GG server load
DEFAULT_MAX_DELAY = 2.0 # Maximum request delay (seconds) - to avoid bot detection mechanisms
DEFAULT_COUNT = 10 # Default item count for API responses - optimal for most use cases
DEFAULT_MAX_COUNT = 50 # Maximum allowed item count - prevents excessive scraping loads
DEFAULT_MAX_CONCURRENCY = 2 # Maximum concurrent requests - balances speed with server courtesy
DEFAULT_API_RPM = 60 # API rate limit (requests per minute) - prevents client abuse
def load_env_file() -> None:
"""
Loads environment variables from .env file using multiple search locations.
This function searches for .env files in common locations and loads the first one found.
It implements a fallback strategy to ensure configuration works even without .env files.
Search Locations:
1. Current directory (./.env)
2. Parent directory (../.env)
3. Current working directory (.env)
Returns:
None: Function performs side effects by loading environment variables
Notes:
- Stops at first .env file found - no merging occurs
- Logs warning if no .env file is found
- Application continues with default values if no .env file exists
- Uses python-dotenv for environment variable loading
Side Effects:
- Loads environment variables into os.environ
- May log warning messages through logger
"""
# Search strategy: check common .env file locations in priority order
env_locations = [
Path(".") / ".env", # Current directory - highest priority
Path("..") / ".env", # Parent directory - for nested project structures
Path(os.getcwd()) / ".env", # Working directory - for different execution contexts
]
# Load first .env file found - no merging strategy implemented
for env_path in env_locations:
if env_path.exists():
load_dotenv(dotenv_path=env_path)
return
# Graceful degradation: continue with default values if no .env file found
logger.warning("No .env file found. Using default values.")
def get_env(key: str, default: Any | None = None, cast_type: Any = None) -> Any:
"""
Retrieves environment variable with safe type casting and fallback handling.
This utility function provides type-safe environment variable access with automatic
type conversion and graceful error handling. It supports common Python types and
implements special handling for boolean values.
Args:
key (str): Environment variable name to retrieve
default (Any | None, optional): Fallback value if variable not found or casting fails.
Defaults to None.
cast_type (Any, optional): Python type to cast the value to (int, float, bool, str).
Defaults to None (returns string).
Returns:
Any: Environment variable value cast to specified type, or default value
Raises:
No exceptions raised - all errors are handled gracefully with fallback to default
Type Casting Rules:
- bool: Accepts "true", "1", "yes", "y" (case-insensitive) as True
- int/float: Uses standard Python type conversion
- str: Returns value as-is (default behavior)
- Invalid casting: Falls back to default value with warning
Algorithm:
1. Retrieve raw environment variable value
2. Return default if value is None
3. Apply special boolean conversion logic
4. Attempt type casting with error handling
5. Log warnings and return defaults on casting failures
Notes:
- Boolean conversion is case-insensitive for user convenience
- Type casting errors are logged but don't interrupt application flow
- Designed for configuration loading where reliability is critical
"""
# Retrieve raw environment variable value
value = os.getenv(key, default)
# Early return for None values - avoid casting None
if value is None:
return default
# Special handling for boolean values - supports multiple truthy representations
if cast_type is bool:
return str(value).lower() in ("true", "1", "yes", "y")
elif cast_type is not None:
# Attempt type casting with comprehensive error handling
try:
return cast_type(value)
except (ValueError, TypeError):
# Log casting failure with detailed context for debugging
logger.warning(
"Could not cast env; using default",
extra={
"key": key,
"value": value,
"cast_type": getattr(cast_type, "__name__", str(cast_type)),
"default": default,
},
)
# Graceful degradation: return default instead of crashing
return default
# Default case: return string value as-is
return value
# Initialize environment variables from .env file
load_env_file()
# ================================================
# API Application Settings
# ================================================
# FastAPI application metadata - used in OpenAPI documentation and UI
API_TITLE = get_env("API_TITLE", "VLR.GG API") # Application title shown in docs
API_DESCRIPTION = get_env(
"API_DESCRIPTION", "Valorant esports data API scraped from vlr.gg"
) # Detailed description for API consumers
API_VERSION = get_env("API_VERSION", "1.0.0") # Semantic version for API changes
# Development mode settings - controls logging behavior
API_DEVELOPMENT = get_env("API_DEVELOPMENT", False, bool) # Enable development mode for detailed logging
# ================================================
# Server Configuration
# ================================================
# Web server binding configuration - for uvicorn deployment
HOST = get_env("HOST", "0.0.0.0") # Listen on all interfaces for containerization
PORT = get_env("PORT", 8000, int) # Standard port for FastAPI applications
DEBUG = get_env("DEBUG", False, bool) # Enable debug mode for development
# ================================================
# VLR.GG Scraping Configuration
# ================================================
# Target website base URL - source of all scraped data
VLR_BASE_URL = get_env("VLR_BASE_URL", "https://www.vlr.gg") # Official VLR.GG domain
# HTTP request settings for web scraping - balances performance with server courtesy
HTTP_TIMEOUT = get_env("HTTP_TIMEOUT", DEFAULT_TIMEOUT, int) # Prevent hanging requests
REQUEST_MIN_DELAY = get_env("REQUEST_MIN_DELAY", DEFAULT_MIN_DELAY, float) # Rate limiting floor
REQUEST_MAX_DELAY = get_env("REQUEST_MAX_DELAY", DEFAULT_MAX_DELAY, float) # Rate limiting ceiling
MAX_CONCURRENT_REQUESTS = get_env(
"MAX_CONCURRENT_REQUESTS", DEFAULT_MAX_CONCURRENCY, int
) # Concurrent request limit
# ================================================
# HTTP Headers Configuration
# ================================================
# Default HTTP headers for web scraping - mimics legitimate browser requests
DEFAULT_HEADERS: dict[str, str] = {
# User-Agent: Modern Chrome browser on Windows 10 - commonly accepted by websites
"User-Agent": get_env(
"USER_AGENT",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
),
# Accept: Standard content type preferences for HTML scraping
"Accept": get_env(
"ACCEPT",
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
),
# Accept-Language: English preference - matches VLR.GG's primary language
"Accept-Language": get_env("ACCEPT_LANGUAGE", "en-US,en;q=0.5"),
}
# ================================================
# API Pagination and Response Limits
# ================================================
# News API pagination settings - controls response sizes for news endpoints
DEFAULT_NEWS_COUNT = get_env("DEFAULT_NEWS_COUNT", DEFAULT_COUNT, int) # Default news items per response
MAX_NEWS_COUNT = get_env("MAX_NEWS_COUNT", DEFAULT_MAX_COUNT, int) # Maximum news items allowed
SEARCH_NEWS_PAGE_LIMIT = get_env("SEARCH_NEWS_PAGE_LIMIT", 10, int) # News search result limit
# Matches API pagination settings - controls response sizes for matches endpoints
DEFAULT_MATCHES_COUNT = get_env("DEFAULT_MATCHES_COUNT", DEFAULT_COUNT, int) # Default matches per response
MAX_MATCHES_COUNT = get_env("MAX_MATCHES_COUNT", DEFAULT_MAX_COUNT, int) # Maximum matches allowed
# Player API pagination settings - controls response sizes for player endpoints
DEFAULT_PLAYER_MATCHES_COUNT = get_env(
"DEFAULT_PLAYER_MATCHES_COUNT", DEFAULT_COUNT, int
) # Default player matches per response
MAX_PLAYER_MATCHES_COUNT = get_env("MAX_PLAYER_MATCHES_COUNT", DEFAULT_MAX_COUNT, int) # Maximum player matches
# Events API pagination settings - controls response sizes for events endpoints
DEFAULT_EVENTS_COUNT = get_env("DEFAULT_EVENTS_COUNT", DEFAULT_COUNT, int) # Default events per response
MAX_EVENTS_COUNT = get_env("MAX_EVENTS_COUNT", DEFAULT_MAX_COUNT, int) # Maximum events allowed
# ================================================
# Configuration Validation and Auto-Correction
# ================================================
# Rate limiting configuration validation - ensures logical delay values
if REQUEST_MIN_DELAY > REQUEST_MAX_DELAY:
logger.warning(
"REQUEST_MIN_DELAY is greater than REQUEST_MAX_DELAY. Setting them equal."
)
REQUEST_MIN_DELAY = REQUEST_MAX_DELAY # Auto-correct invalid configuration
# ================================================
# API Rate Limiting and Client Protection
# ================================================
# Inbound API rate limiting - protects server from client abuse
API_RATE_LIMIT_PER_MINUTE = get_env("API_RATE_LIMIT_PER_MINUTE", DEFAULT_API_RPM, int) # Requests per minute per IP
# ================================================
# CORS (Cross-Origin Resource Sharing) Configuration
# ================================================
# CORS settings - controls browser access from different domains
CORS_ALLOW_ORIGINS = get_env("CORS_ALLOW_ORIGINS", "*") # Allowed origins (* = all)
CORS_ALLOW_METHODS = get_env("CORS_ALLOW_METHODS", "GET") # Allowed HTTP methods (read-only API)
CORS_ALLOW_HEADERS = get_env("CORS_ALLOW_HEADERS", "*") # Allowed request headers
# ================================================
# Network and Proxy Configuration
# ================================================
# Proxy configuration - for deployment behind load balancers/reverse proxies
TRUST_PROXY = get_env("TRUST_PROXY", False, bool) # Trust X-Forwarded-For header for client IP
# ================================================
# Web Scraping Ethics and Compliance
# ================================================
# Robots.txt compliance settings - respects website scraping policies
RESPECT_ROBOTS = get_env("RESPECT_ROBOTS", True, bool) # Honor robots.txt restrictions
ROBOTS_TTL_SECONDS = get_env("ROBOTS_TTL_SECONDS", 86400, int) # Cache robots.txt for 24 hours
# ================================================
# Caching and Performance Configuration
# ================================================
# HTTP response caching settings - improves client performance
STATIC_CACHE_SECONDS = get_env("STATIC_CACHE_SECONDS", 86400, int) # Static file cache (24 hours)
DOCS_CACHE_SECONDS = get_env("DOCS_CACHE_SECONDS", 300, int) # Documentation cache (5 minutes)
# ================================================
# HTTP Session Management
# ================================================
# Shared HTTP session configuration - experimental performance optimization
USE_SHARED_HTTP_SESSION = get_env("USE_SHARED_HTTP_SESSION", False, bool) # Enable shared AsyncClient