mischbeck/TornActivityTracker
1
0
Fork 0
Code Issues 5 Pull Requests Packages Projects Releases Wiki Activity

refactors logging and config

Browse Source
This commit is contained in:
Michael Beck
2025-02-10 16:34:11 +01:00
parent d1f562ce94
commit 33621bdec4
14 changed files with 114 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
app/models.py
View File

@@ -6,14 +6,7 @@ import time
from datetime import datetime, timedelta
from requests.exceptions import ConnectionError, Timeout, RequestException
from app.logging_config import get_logger
from app.config import load_config
config = load_config()
API_KEY = config['DEFAULT']['API_KEY']
logger = get_logger()
from flask import current_app
class Scraper:
def __init__(self, faction_id, fetch_interval, run_interval, app):
@@ -23,19 +16,21 @@ class Scraper:
self.end_time = datetime.now() + timedelta(days=run_interval)
self.data_file_name = os.path.join(app.config['DATA']['DATA_DIR'], f"{self.faction_id}-{datetime.now().strftime('%Y-%m-%d-%H-%M')}.csv")
self.scraping_active = False
self.API_KEY = app.config['DEFAULT']['API_KEY']
self.logger = app.logger
print(self.data_file_name)
def fetch_faction_data(self):
url = f"https://api.torn.com/faction/{self.faction_id}?selections=&key={API_KEY}"
url = f"https://api.torn.com/faction/{self.faction_id}?selections=&key={self.API_KEY}"
response = requests.get(url)
if response.status_code == 200:
return response.json()
logger.warning(f"Failed to fetch faction data for faction ID {self.faction_id}. Response: {response.text}")
current_app.logger.warning(f"Failed to fetch faction data for faction ID {self.faction_id}. Response: {response.text}")
return None
def fetch_user_activity(self, user_id):
url = f"https://api.torn.com/user/{user_id}?selections=basic,profile&key={API_KEY}"
url = f"https://api.torn.com/user/{user_id}?selections=basic,profile&key={self.API_KEY}"
retries = 3
for attempt in range(retries):
try:
@@ -43,45 +38,51 @@ class Scraper:
response.raise_for_status()
return response.json()
except ConnectionError as e:
logger.error(f"Connection error while fetching user activity for user ID {user_id}: {e}")
current_app.logger.error(f"Connection error while fetching user activity for user ID {user_id}: {e}")
except Timeout as e:
logger.error(f"Timeout error while fetching user activity for user ID {user_id}: {e}")
current_app.logger.error(f"Timeout error while fetching user activity for user ID {user_id}: {e}")
except RequestException as e:
logger.error(f"Error while fetching user activity for user ID {user_id}: {e}")
current_app.logger.error(f"Error while fetching user activity for user ID {user_id}: {e}")
if attempt < retries - 1:
current_app.logger.debug(f"Retrying {attempt + 1}/{retries} for user {user_id}")
time.sleep(2 ** attempt) # Exponential backoff
return None
def start_scraping(self) -> None:
def start_scraping(self, app) -> None:
"""Starts the scraping process until the end time is reached or stopped manually."""
self.scraping_active = True
logger.info(f"Starting scraping for faction ID {self.faction_id}")
logger.debug(f"Fetch interval: {self.fetch_interval}s, Run interval: {self.run_interval} days, End time: {self.end_time}")
MAX_FAILURES = 5 # Stop after 5 consecutive failures
failure_count = 0
# Anwendungskontext explizit setzen
with app.app_context():
current_app.logger.info(f"Starting scraping for faction ID {self.faction_id}")
current_app.logger.debug(f"Fetch interval: {self.fetch_interval}s, Run interval: {self.run_interval} days, End time: {self.end_time}")
while datetime.now() < self.end_time and self.scraping_active:
logger.info(f"Fetching data at {datetime.now()}")
faction_data = self.fetch_faction_data()
MAX_FAILURES = 5 # Stop after 5 consecutive failures
failure_count = 0
if not faction_data or "members" not in faction_data:
logger.warning(f"No faction data found for ID {self.faction_id} (Failure {failure_count + 1}/{MAX_FAILURES})")
failure_count += 1
if failure_count >= MAX_FAILURES:
logger.error(f"Max failures reached ({MAX_FAILURES}). Stopping scraping.")
break
while datetime.now() < self.end_time and self.scraping_active:
current_app.logger.info(f"Fetching data at {datetime.now()}")
faction_data = self.fetch_faction_data()
if not faction_data or "members" not in faction_data:
current_app.logger.warning(f"No faction data found for ID {self.faction_id} (Failure {failure_count + 1}/{MAX_FAILURES})")
failure_count += 1
if failure_count >= MAX_FAILURES:
current_app.logger.error(f"Max failures reached ({MAX_FAILURES}). Stopping scraping.")
break
time.sleep(self.fetch_interval)
continue
current_app.logger.info(f"Fetched {len(faction_data['members'])} members for faction {self.faction_id}")
failure_count = 0 # Reset failure count on success
user_activity_data = self.process_faction_members(faction_data["members"])
self.save_data(user_activity_data)
current_app.logger.info(f"Data appended to {self.data_file_name}")
time.sleep(self.fetch_interval)
continue
self.handle_scraping_end()
failure_count = 0 # Reset failure count on success
user_activity_data = self.process_faction_members(faction_data["members"])
self.save_data(user_activity_data)
logger.info(f"Data appended to {self.data_file_name}")
time.sleep(self.fetch_interval)
self.handle_scraping_end()
def process_faction_members(self, members: Dict[str, Dict]) -> List[Dict]:
"""Processes and retrieves user activity for all faction members."""
@@ -96,16 +97,16 @@ class Scraper:
"status": user_activity.get("status", {}).get("state", ""),
"timestamp": datetime.now().timestamp(),
})
logger.info(f"Fetched data for user {user_id} ({user_activity.get('name', '')})")
current_app.logger.info(f"Fetched data for user {user_id} ({user_activity.get('name', '')})")
else:
logger.warning(f"Failed to fetch data for user {user_id}")
current_app.logger.warning(f"Failed to fetch data for user {user_id}")
return user_activity_data
def save_data(self, user_activity_data: List[Dict]) -> None:
"""Saves user activity data to a CSV file."""
if not user_activity_data:
logger.warning("No data to save.")
current_app.logger.warning("No data to save.")
return
df = pd.DataFrame(user_activity_data)
@@ -117,22 +118,22 @@ class Scraper:
try:
with open(self.data_file_name, "a" if file_exists else "w") as f:
df.to_csv(f, mode="a" if file_exists else "w", header=not file_exists, index=False)
logger.info(f"Data successfully saved to {self.data_file_name}")
current_app.logger.info(f"Data successfully saved to {self.data_file_name}")
except Exception as e:
logger.error(f"Error saving data to {self.data_file_name}: {e}")
current_app.logger.error(f"Error saving data to {self.data_file_name}: {e}")
def handle_scraping_end(self) -> None:
"""Handles cleanup and logging when scraping ends."""
if not self.scraping_active:
logger.warning(f"Scraping stopped manually at {datetime.now()}")
current_app.logger.warning(f"Scraping stopped manually at {datetime.now()}")
elif datetime.now() >= self.end_time:
logger.warning(f"Scraping stopped due to timeout at {datetime.now()} (Run interval: {self.run_interval} days)")
current_app.logger.warning(f"Scraping stopped due to timeout at {datetime.now()} (Run interval: {self.run_interval} days)")
else:
logger.error(f"Unexpected stop at {datetime.now()}")
current_app.logger.error(f"Unexpected stop at {datetime.now()}")
logger.info("Scraping completed.")
current_app.logger.info("Scraping completed.")
self.scraping_active = False
def stop_scraping(self):
self.scraping_active = False
logger.debug("Scraping stopped by user")
current_app.logger.debug("Scraping stopped by user")