From 33621bdec49479c235e9f2219a94142370c00b25 Mon Sep 17 00:00:00 2001 From: Michael Beck Date: Mon, 10 Feb 2025 16:34:11 +0100 Subject: [PATCH] refactors logging and config --- app/analysis/basePlotAnalysis.py | 1 - app/analysis/basePlotlyAnalysis.py | 1 - app/analysis/plot_bar_activity-user.py | 4 - app/analysis/plot_bar_peak_hours.py | 1 - app/analysis/plot_line_activity-user.py | 4 - app/analysis/plotly_heat_user-activity.py | 4 - app/analysis/plotly_line_activity-user.py | 4 - app/api.py | 44 +++++------ app/app.py | 27 +++---- app/config.py | 9 ++- app/logging_config.py | 47 +++++------ app/models.py | 95 ++++++++++++----------- app/util.py | 9 +-- app/views.py | 11 +-- 14 files changed, 114 insertions(+), 147 deletions(-) diff --git a/app/analysis/basePlotAnalysis.py b/app/analysis/basePlotAnalysis.py index 2b6a15d..262bc5f 100644 --- a/app/analysis/basePlotAnalysis.py +++ b/app/analysis/basePlotAnalysis.py @@ -2,7 +2,6 @@ import os import pandas as pd import matplotlib.pyplot as plt import seaborn as sns -from flask import url_for from abc import ABC, abstractmethod from .base import BaseAnalysis diff --git a/app/analysis/basePlotlyAnalysis.py b/app/analysis/basePlotlyAnalysis.py index 291aff3..bc04660 100644 --- a/app/analysis/basePlotlyAnalysis.py +++ b/app/analysis/basePlotlyAnalysis.py @@ -1,7 +1,6 @@ import os import pandas as pd import plotly.graph_objects as go -from flask import url_for from abc import ABC, abstractmethod from .base import BaseAnalysis diff --git a/app/analysis/plot_bar_activity-user.py b/app/analysis/plot_bar_activity-user.py index 6f8eaf1..2f78ca7 100644 --- a/app/analysis/plot_bar_activity-user.py +++ b/app/analysis/plot_bar_activity-user.py @@ -4,13 +4,9 @@ import seaborn as sns from .basePlotAnalysis import BasePlotAnalysis from flask import current_app, url_for -from app.logging_config import get_logger - import matplotlib matplotlib.use('Agg') -logger = get_logger() - class PlotTopActiveUsers(BasePlotAnalysis): """ Class for analyzing the most active users and generating a bar chart. diff --git a/app/analysis/plot_bar_peak_hours.py b/app/analysis/plot_bar_peak_hours.py index 95dff34..a090b56 100644 --- a/app/analysis/plot_bar_peak_hours.py +++ b/app/analysis/plot_bar_peak_hours.py @@ -3,7 +3,6 @@ import pandas as pd import matplotlib.pyplot as plt import seaborn as sns from .basePlotAnalysis import BasePlotAnalysis -from flask import current_app, url_for import matplotlib matplotlib.use('Agg') diff --git a/app/analysis/plot_line_activity-user.py b/app/analysis/plot_line_activity-user.py index d5cf072..0396d46 100644 --- a/app/analysis/plot_line_activity-user.py +++ b/app/analysis/plot_line_activity-user.py @@ -4,13 +4,9 @@ import seaborn as sns from .basePlotAnalysis import BasePlotAnalysis from flask import current_app, url_for -from app.logging_config import get_logger - import matplotlib matplotlib.use('Agg') -logger = get_logger() - class PlotLineActivityAllUsers(BasePlotAnalysis): """ Class for analyzing user activity trends over multiple days and generating a line graph. diff --git a/app/analysis/plotly_heat_user-activity.py b/app/analysis/plotly_heat_user-activity.py index a73bc3c..51462aa 100644 --- a/app/analysis/plotly_heat_user-activity.py +++ b/app/analysis/plotly_heat_user-activity.py @@ -5,10 +5,6 @@ import plotly.graph_objects as go from .basePlotlyAnalysis import BasePlotlyAnalysis from flask import current_app, url_for -from app.logging_config import get_logger - -logger = get_logger() - class PlotlyActivityHeatmap(BasePlotlyAnalysis): """ Class for analyzing user activity trends over multiple days and generating an interactive heatmap. diff --git a/app/analysis/plotly_line_activity-user.py b/app/analysis/plotly_line_activity-user.py index cd56b80..0e49e5d 100644 --- a/app/analysis/plotly_line_activity-user.py +++ b/app/analysis/plotly_line_activity-user.py @@ -4,10 +4,6 @@ from plotly.subplots import make_subplots from .basePlotlyAnalysis import BasePlotlyAnalysis from flask import current_app, url_for -from app.logging_config import get_logger - -logger = get_logger() - class PlotlyLineActivityAllUsers(BasePlotlyAnalysis): """ Class for analyzing user activity trends over multiple days and generating an interactive line graph. diff --git a/app/api.py b/app/api.py index 3efbe3a..dcb9ae8 100644 --- a/app/api.py +++ b/app/api.py @@ -7,15 +7,10 @@ from datetime import datetime import pandas as pd from app.models import Scraper -from app.util import create_zip, delete_old_zips, tail, get_size +from app.util import create_zip, delete_old_zips, tail from app.config import load_config -from app.logging_config import get_logger from app.forms import ScrapingForm -config = load_config() -logger = get_logger() -log_file_name = logger.handlers[0].baseFilename - scraping_thread = None scraper = None scrape_lock = threading.Lock() @@ -23,10 +18,11 @@ scrape_lock = threading.Lock() def register_api(app): @app.route('/start_scraping', methods=['POST']) def start_scraping(): + global scraping_thread, scraper with scrape_lock: scraper = current_app.config.get('SCRAPER') if scraper is not None and scraper.scraping_active: - logger.warning("Can't start scraping process: scraping already in progress") + current_app.logger.warning("Can't start scraping process: scraping already in progress") return jsonify({"status": "Scraping already in progress"}) form = ScrapingForm() @@ -35,10 +31,10 @@ def register_api(app): fetch_interval = form.fetch_interval.data run_interval = form.run_interval.data - scraper = Scraper(faction_id, fetch_interval, run_interval, current_app) + scraper = Scraper(faction_id, fetch_interval, run_interval, app) scraper.scraping_active = True - scraping_thread = threading.Thread(target=scraper.start_scraping) + scraping_thread = threading.Thread(target=scraper.start_scraping, args=(app,)) scraping_thread.daemon = True scraping_thread.start() @@ -56,19 +52,21 @@ def register_api(app): scraper.stop_scraping() current_app.config['SCRAPING_ACTIVE'] = False - logger.debug("Scraping stopped by user") + current_app.logger.debug("Scraping stopped by user") return jsonify({"status": "Scraping stopped"}) @app.route('/logfile', methods=['GET']) def logfile(): + log_file_name = current_app.logger.handlers[0].baseFilename + page = int(request.args.get('page', 0)) # Page number - lines_per_page = int(request.args.get('lines_per_page', config['LOGGING']['VIEW_PAGE_LINES'])) # Lines per page + lines_per_page = int(request.args.get('lines_per_page', current_app.config['LOGGING']['VIEW_PAGE_LINES'])) # Lines per page log_file_path = log_file_name # Path to the current log file if not os.path.isfile(log_file_path): - logger.error("Log file not found") + current_app.logger.error("Log file not found") return jsonify({"error": "Log file not found"}), 404 - log_lines = list(tail(log_file_path, config['LOGGING']['VIEW_MAX_LINES'])) + log_lines = list(tail(log_file_path, current_app.config['LOGGING']['VIEW_MAX_LINES'])) log_lines = log_lines[::-1] # Reverse the list @@ -123,14 +121,15 @@ def register_api(app): @app.route('/delete_files', methods=['POST']) def delete_files(): + log_file_name = current_app.logger.handlers[0].baseFilename file_paths = request.json.get('file_paths', []) if not file_paths: return jsonify({"error": "No files specified"}), 400 errors = [] - data_dir = os.path.abspath(config['DATA']['DATA_DIR']) - log_dir = os.path.abspath(config['LOGGING']['LOG_DIR']) + data_dir = os.path.abspath(current_app.config['DATA']['DATA_DIR']) + log_dir = os.path.abspath(current_app.config['LOGGING']['LOG_DIR']) for file_path in file_paths: if file_path.startswith('/data/'): @@ -171,40 +170,39 @@ def register_api(app): @app.route('/data/') def download_data_file(filename): - data_dir = os.path.abspath(config['DATA']['DATA_DIR']) + data_dir = os.path.abspath(current_app.config['DATA']['DATA_DIR']) file_path = os.path.join(data_dir, filename) return send_from_directory(directory=data_dir, path=filename, as_attachment=True) @app.route('/log/') def download_log_file(filename): - log_dir = os.path.abspath(config['LOGGING']['LOG_DIR']) + log_dir = os.path.abspath(current_app.config['LOGGING']['LOG_DIR']) file_path = os.path.join(log_dir, filename) return send_from_directory(directory=log_dir, path=filename, as_attachment=True) @app.route('/tmp/') def download_tmp_file(filename): - tmp_dir = os.path.abspath(config['TEMP']['TEMP_DIR']) + tmp_dir = os.path.abspath(current_app.config['TEMP']['TEMP_DIR']) file_path = os.path.join(tmp_dir, filename) return send_from_directory(directory=tmp_dir, path=filename, as_attachment=True) - @app.route('/config/lines_per_page') def get_lines_per_page(): - lines_per_page = config['LOGGING']['VIEW_PAGE_LINES'] + lines_per_page = current_app.config['LOGGING']['VIEW_PAGE_LINES'] return jsonify({"lines_per_page": lines_per_page}) @app.route('/scraping_status', methods=['GET']) def scraping_status(): if scraper is None: - logger.debug("Scraper is not initialized.") + current_app.logger.debug("Scraper is not initialized.") return jsonify({"scraping_active": False}) if scraper.scraping_active: - logger.debug("Scraping is active.") + current_app.logger.debug("Scraping is active.") return jsonify({"scraping_active": True}) else: - logger.debug("Scraping is not active.") + current_app.logger.debug("Scraping is not active.") return jsonify({"scraping_active": False}) \ No newline at end of file diff --git a/app/app.py b/app/app.py index b9afd09..1cf41a3 100644 --- a/app/app.py +++ b/app/app.py @@ -7,35 +7,36 @@ from app.api import register_api from app.config import load_config from app.filters import register_filters -def init_app(): - config = load_config() +from app.logging_config import init_logger - # Initialize app +def init_app(): app = Flask(__name__) - # Load configuration - app.config['SECRET_KEY'] = config['DEFAULT']['SECRET_KEY'] - app.config['API_KEY'] = config['DEFAULT']['API_KEY'] + config = load_config() - app.config['DATA'] = config['DATA'] - app.config['TEMP'] = config['TEMP'] - app.config['LOGGING'] = config['LOGGING'] + app.config['SECRET_KEY'] = config['DEFAULT']['SECRET_KEY'] # Move bootstrap settings to root level - for key in config['BOOTSTRAP']: - app.config[key.upper()] = config['BOOTSTRAP'][key] + for key, value in config.get('BOOTSTRAP', {}).items(): + app.config[key.upper()] = value bootstrap = Bootstrap5(app) - # Initialize global variables + # Store the entire config in Flask app + app.config.update(config) + + # Initialize other settings app.config['SCRAPING_ACTIVE'] = False app.config['SCRAPING_THREAD'] = None app.config['DATA_FILE_NAME'] = None app.config['LOG_FILE_NAME'] = "log/" + datetime.now().strftime('%Y-%m-%d-%H-%M') + '.log' + # Initialize logging + app.logger = init_logger(app.config) + # Register routes register_views(app) register_api(app) register_filters(app) - + return app \ No newline at end of file diff --git a/app/config.py b/app/config.py index 96fd0be..71ef4bd 100644 --- a/app/config.py +++ b/app/config.py @@ -1,7 +1,8 @@ -import configparser +from configobj import ConfigObj import os def load_config(): - config = configparser.ConfigParser() - config.read(os.path.join(os.path.dirname(__file__), '..', 'config.ini')) - return config \ No newline at end of file + config_path = os.path.join(os.path.dirname(__file__), '..', 'config.ini') + + # Load config while preserving sections as nested dicts + return ConfigObj(config_path) diff --git a/app/logging_config.py b/app/logging_config.py index f407007..3f9f0b1 100644 --- a/app/logging_config.py +++ b/app/logging_config.py @@ -4,36 +4,31 @@ from queue import Queue import os from datetime import datetime -from app.config import load_config +from flask import current_app -config = load_config() +def init_logger(config): + LOG_DIR = config.get('LOGGING', {}).get('LOG_DIR', 'log') -# Define the log directory and ensure it exists -LOG_DIR = config['LOGGING']['LOG_DIR'] -if not os.path.exists(LOG_DIR): - os.makedirs(LOG_DIR) + if not os.path.exists(LOG_DIR): + os.makedirs(LOG_DIR) -# Generate the log filename dynamically -log_file_name = os.path.join(LOG_DIR, datetime.now().strftime('%Y-%m-%d-%H-%M') + '.log') + log_file_name = os.path.join(LOG_DIR, datetime.now().strftime('%Y-%m-%d-%H-%M') + '.log') -# Initialize the logger -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) + logger = logging.getLogger(__name__) + logger.setLevel(logging.DEBUG) -# File handler -file_handler = logging.FileHandler(log_file_name, mode='w') -file_handler.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(asctime)s - %(levelname)s: %(message)s', - datefmt='%m/%d/%Y %I:%M:%S %p') -file_handler.setFormatter(formatter) -logger.addHandler(file_handler) + file_handler = logging.FileHandler(log_file_name, mode='w') + file_handler.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(asctime)s - %(levelname)s: %(message)s', + datefmt='%m/%d/%Y %I:%M:%S %p') + file_handler.setFormatter(formatter) + logger.addHandler(file_handler) -# Queue handler for real-time logging -log_queue = Queue() -queue_handler = QueueHandler(log_queue) -queue_handler.setLevel(logging.DEBUG) -logger.addHandler(queue_handler) + log_queue = Queue() + queue_handler = QueueHandler(log_queue) + queue_handler.setLevel(logging.DEBUG) + logger.addHandler(queue_handler) -# Function to get logger in other modules -def get_logger(): - return logger + logger.debug("Logger initialized") + + return logger \ No newline at end of file diff --git a/app/models.py b/app/models.py index 187061d..0f6e7d7 100644 --- a/app/models.py +++ b/app/models.py @@ -6,14 +6,7 @@ import time from datetime import datetime, timedelta from requests.exceptions import ConnectionError, Timeout, RequestException -from app.logging_config import get_logger - -from app.config import load_config - -config = load_config() -API_KEY = config['DEFAULT']['API_KEY'] - -logger = get_logger() +from flask import current_app class Scraper: def __init__(self, faction_id, fetch_interval, run_interval, app): @@ -23,19 +16,21 @@ class Scraper: self.end_time = datetime.now() + timedelta(days=run_interval) self.data_file_name = os.path.join(app.config['DATA']['DATA_DIR'], f"{self.faction_id}-{datetime.now().strftime('%Y-%m-%d-%H-%M')}.csv") self.scraping_active = False + self.API_KEY = app.config['DEFAULT']['API_KEY'] + self.logger = app.logger print(self.data_file_name) def fetch_faction_data(self): - url = f"https://api.torn.com/faction/{self.faction_id}?selections=&key={API_KEY}" + url = f"https://api.torn.com/faction/{self.faction_id}?selections=&key={self.API_KEY}" response = requests.get(url) if response.status_code == 200: return response.json() - logger.warning(f"Failed to fetch faction data for faction ID {self.faction_id}. Response: {response.text}") + current_app.logger.warning(f"Failed to fetch faction data for faction ID {self.faction_id}. Response: {response.text}") return None def fetch_user_activity(self, user_id): - url = f"https://api.torn.com/user/{user_id}?selections=basic,profile&key={API_KEY}" + url = f"https://api.torn.com/user/{user_id}?selections=basic,profile&key={self.API_KEY}" retries = 3 for attempt in range(retries): try: @@ -43,45 +38,51 @@ class Scraper: response.raise_for_status() return response.json() except ConnectionError as e: - logger.error(f"Connection error while fetching user activity for user ID {user_id}: {e}") + current_app.logger.error(f"Connection error while fetching user activity for user ID {user_id}: {e}") except Timeout as e: - logger.error(f"Timeout error while fetching user activity for user ID {user_id}: {e}") + current_app.logger.error(f"Timeout error while fetching user activity for user ID {user_id}: {e}") except RequestException as e: - logger.error(f"Error while fetching user activity for user ID {user_id}: {e}") + current_app.logger.error(f"Error while fetching user activity for user ID {user_id}: {e}") if attempt < retries - 1: + current_app.logger.debug(f"Retrying {attempt + 1}/{retries} for user {user_id}") time.sleep(2 ** attempt) # Exponential backoff return None - def start_scraping(self) -> None: + def start_scraping(self, app) -> None: """Starts the scraping process until the end time is reached or stopped manually.""" self.scraping_active = True - logger.info(f"Starting scraping for faction ID {self.faction_id}") - logger.debug(f"Fetch interval: {self.fetch_interval}s, Run interval: {self.run_interval} days, End time: {self.end_time}") - MAX_FAILURES = 5 # Stop after 5 consecutive failures - failure_count = 0 + # Anwendungskontext explizit setzen + with app.app_context(): + current_app.logger.info(f"Starting scraping for faction ID {self.faction_id}") + current_app.logger.debug(f"Fetch interval: {self.fetch_interval}s, Run interval: {self.run_interval} days, End time: {self.end_time}") - while datetime.now() < self.end_time and self.scraping_active: - logger.info(f"Fetching data at {datetime.now()}") - faction_data = self.fetch_faction_data() + MAX_FAILURES = 5 # Stop after 5 consecutive failures + failure_count = 0 - if not faction_data or "members" not in faction_data: - logger.warning(f"No faction data found for ID {self.faction_id} (Failure {failure_count + 1}/{MAX_FAILURES})") - failure_count += 1 - if failure_count >= MAX_FAILURES: - logger.error(f"Max failures reached ({MAX_FAILURES}). Stopping scraping.") - break + while datetime.now() < self.end_time and self.scraping_active: + current_app.logger.info(f"Fetching data at {datetime.now()}") + faction_data = self.fetch_faction_data() + + if not faction_data or "members" not in faction_data: + current_app.logger.warning(f"No faction data found for ID {self.faction_id} (Failure {failure_count + 1}/{MAX_FAILURES})") + failure_count += 1 + if failure_count >= MAX_FAILURES: + current_app.logger.error(f"Max failures reached ({MAX_FAILURES}). Stopping scraping.") + break + time.sleep(self.fetch_interval) + continue + + current_app.logger.info(f"Fetched {len(faction_data['members'])} members for faction {self.faction_id}") + failure_count = 0 # Reset failure count on success + user_activity_data = self.process_faction_members(faction_data["members"]) + self.save_data(user_activity_data) + + current_app.logger.info(f"Data appended to {self.data_file_name}") time.sleep(self.fetch_interval) - continue + + self.handle_scraping_end() - failure_count = 0 # Reset failure count on success - user_activity_data = self.process_faction_members(faction_data["members"]) - self.save_data(user_activity_data) - - logger.info(f"Data appended to {self.data_file_name}") - time.sleep(self.fetch_interval) - - self.handle_scraping_end() def process_faction_members(self, members: Dict[str, Dict]) -> List[Dict]: """Processes and retrieves user activity for all faction members.""" @@ -96,16 +97,16 @@ class Scraper: "status": user_activity.get("status", {}).get("state", ""), "timestamp": datetime.now().timestamp(), }) - logger.info(f"Fetched data for user {user_id} ({user_activity.get('name', '')})") + current_app.logger.info(f"Fetched data for user {user_id} ({user_activity.get('name', '')})") else: - logger.warning(f"Failed to fetch data for user {user_id}") + current_app.logger.warning(f"Failed to fetch data for user {user_id}") return user_activity_data def save_data(self, user_activity_data: List[Dict]) -> None: """Saves user activity data to a CSV file.""" if not user_activity_data: - logger.warning("No data to save.") + current_app.logger.warning("No data to save.") return df = pd.DataFrame(user_activity_data) @@ -117,22 +118,22 @@ class Scraper: try: with open(self.data_file_name, "a" if file_exists else "w") as f: df.to_csv(f, mode="a" if file_exists else "w", header=not file_exists, index=False) - logger.info(f"Data successfully saved to {self.data_file_name}") + current_app.logger.info(f"Data successfully saved to {self.data_file_name}") except Exception as e: - logger.error(f"Error saving data to {self.data_file_name}: {e}") + current_app.logger.error(f"Error saving data to {self.data_file_name}: {e}") def handle_scraping_end(self) -> None: """Handles cleanup and logging when scraping ends.""" if not self.scraping_active: - logger.warning(f"Scraping stopped manually at {datetime.now()}") + current_app.logger.warning(f"Scraping stopped manually at {datetime.now()}") elif datetime.now() >= self.end_time: - logger.warning(f"Scraping stopped due to timeout at {datetime.now()} (Run interval: {self.run_interval} days)") + current_app.logger.warning(f"Scraping stopped due to timeout at {datetime.now()} (Run interval: {self.run_interval} days)") else: - logger.error(f"Unexpected stop at {datetime.now()}") + current_app.logger.error(f"Unexpected stop at {datetime.now()}") - logger.info("Scraping completed.") + current_app.logger.info("Scraping completed.") self.scraping_active = False def stop_scraping(self): self.scraping_active = False - logger.debug("Scraping stopped by user") \ No newline at end of file + current_app.logger.debug("Scraping stopped by user") \ No newline at end of file diff --git a/app/util.py b/app/util.py index 56bff4c..29025bc 100644 --- a/app/util.py +++ b/app/util.py @@ -1,13 +1,10 @@ import os import zipfile from datetime import datetime, timedelta - -from app.state import data_file_name, log_file_name +from flask import current_app from app.config import load_config -config = load_config() - def create_zip(file_paths, zip_name, app): temp_dir = os.path.abspath(app.config['TEMP']['TEMP_DIR']) zip_path = os.path.join(temp_dir, zip_name) @@ -18,7 +15,7 @@ def create_zip(file_paths, zip_name, app): return zip_path def delete_old_zips(): - temp_dir = os.path.abspath(config['TEMP']['TEMP_DIR']) + temp_dir = os.path.abspath(current_app.config['TEMP']['TEMP_DIR']) now = datetime.now() for filename in os.listdir(temp_dir): if filename.endswith('.zip'): @@ -33,7 +30,7 @@ def tail(filename, n): yield '' return - page_size = int(config['LOGGING']['TAIL_PAGE_SIZE']) + page_size = int(current_app.config['LOGGING']['TAIL_PAGE_SIZE']) offsets = [] count = _n = n if n >= 0 else -n diff --git a/app/views.py b/app/views.py index f49aa67..939e0fa 100644 --- a/app/views.py +++ b/app/views.py @@ -6,16 +6,9 @@ from app.forms import ScrapingForm from app.util import get_size from app.config import load_config from app.api import scraper as scraper -from app.logging_config import get_logger from app.analysis import load_data, load_analysis_modules - -print(f"A imported log_file_name: {log_file_name}") - -config = load_config() -logger = get_logger() - views_bp = Blueprint("views", __name__) def register_views(app): @@ -42,8 +35,8 @@ def register_views(app): if not scraper: print("Scraper not initialized") - data_dir = os.path.abspath(config['DATA']['DATA_DIR']) - log_dir = os.path.abspath(config['LOGGING']['LOG_DIR']) + data_dir = os.path.abspath(current_app.config['DATA']['DATA_DIR']) + log_dir = os.path.abspath(current_app.config['LOGGING']['LOG_DIR']) data_files = glob.glob(os.path.join(data_dir, "*.csv")) log_files = glob.glob(os.path.join(log_dir, "*.log"))