refactors logging and config

app/analysis/basePlotAnalysis.py

@@ -2,7 +2,6 @@ import os
 import pandas as pd
 import matplotlib.pyplot as plt
 import seaborn as sns
-from flask import url_for
 from abc import ABC, abstractmethod
 
 from .base import BaseAnalysis

app/analysis/basePlotlyAnalysis.py

@@ -1,7 +1,6 @@
 import os
 import pandas as pd
 import plotly.graph_objects as go
-from flask import url_for
 from abc import ABC, abstractmethod
 
 from .base import BaseAnalysis

app/analysis/plot_bar_activity-user.py

@@ -4,13 +4,9 @@ import seaborn as sns
 from .basePlotAnalysis import BasePlotAnalysis
 from flask import current_app, url_for
 
-from app.logging_config import get_logger
-
 import matplotlib
 matplotlib.use('Agg')
 
-logger = get_logger()
-
 class PlotTopActiveUsers(BasePlotAnalysis):
     """
     Class for analyzing the most active users and generating a bar chart.

app/analysis/plot_bar_peak_hours.py

@@ -3,7 +3,6 @@ import pandas as pd
 import matplotlib.pyplot as plt
 import seaborn as sns
 from .basePlotAnalysis import BasePlotAnalysis
-from flask import current_app, url_for
 
 import matplotlib
 matplotlib.use('Agg')

app/analysis/plot_line_activity-user.py

@@ -4,13 +4,9 @@ import seaborn as sns
 from .basePlotAnalysis import BasePlotAnalysis
 from flask import current_app, url_for
 
-from app.logging_config import get_logger
-
 import matplotlib
 matplotlib.use('Agg')
 
-logger = get_logger()
-
 class PlotLineActivityAllUsers(BasePlotAnalysis):
     """
     Class for analyzing user activity trends over multiple days and generating a line graph.

app/analysis/plotly_heat_user-activity.py

@@ -5,10 +5,6 @@ import plotly.graph_objects as go
 from .basePlotlyAnalysis import BasePlotlyAnalysis
 from flask import current_app, url_for
 
-from app.logging_config import get_logger
-
-logger = get_logger()
-
 class PlotlyActivityHeatmap(BasePlotlyAnalysis):
     """
     Class for analyzing user activity trends over multiple days and generating an interactive heatmap.

app/analysis/plotly_line_activity-user.py

@@ -4,10 +4,6 @@ from plotly.subplots import make_subplots
 from .basePlotlyAnalysis import BasePlotlyAnalysis
 from flask import current_app, url_for
 
-from app.logging_config import get_logger
-
-logger = get_logger()
-
 class PlotlyLineActivityAllUsers(BasePlotlyAnalysis):
     """
     Class for analyzing user activity trends over multiple days and generating an interactive line graph.

app/api.py

@@ -7,15 +7,10 @@ from datetime import datetime
 import pandas as pd
 
 from app.models import Scraper
-from app.util import create_zip, delete_old_zips, tail, get_size
+from app.util import create_zip, delete_old_zips, tail
 from app.config import load_config
-from app.logging_config import get_logger
 from app.forms import ScrapingForm
 
-config = load_config()
-logger = get_logger()
-log_file_name = logger.handlers[0].baseFilename
-
 scraping_thread = None
 scraper = None
 scrape_lock = threading.Lock()
@@ -23,10 +18,11 @@ scrape_lock = threading.Lock()
 def register_api(app):
     @app.route('/start_scraping', methods=['POST'])
     def start_scraping():
+        global scraping_thread, scraper
         with scrape_lock:
             scraper = current_app.config.get('SCRAPER')
             if scraper is not None and scraper.scraping_active:
-                logger.warning("Can't start scraping process: scraping already in progress")
+                current_app.logger.warning("Can't start scraping process: scraping already in progress")
                 return jsonify({"status": "Scraping already in progress"})
 
             form = ScrapingForm()
@@ -35,10 +31,10 @@ def register_api(app):
                 fetch_interval = form.fetch_interval.data
                 run_interval = form.run_interval.data
 
-                scraper = Scraper(faction_id, fetch_interval, run_interval, current_app)
+                scraper = Scraper(faction_id, fetch_interval, run_interval, app)
                 scraper.scraping_active = True
 
-                scraping_thread = threading.Thread(target=scraper.start_scraping)
+                scraping_thread = threading.Thread(target=scraper.start_scraping, args=(app,))
                 scraping_thread.daemon = True
                 scraping_thread.start()
 
@@ -56,19 +52,21 @@ def register_api(app):
 
         scraper.stop_scraping()
         current_app.config['SCRAPING_ACTIVE'] = False
-        logger.debug("Scraping stopped by user")
+        current_app.logger.debug("Scraping stopped by user")
         return jsonify({"status": "Scraping stopped"})    
     @app.route('/logfile', methods=['GET'])
     def logfile():
+        log_file_name = current_app.logger.handlers[0].baseFilename
+
         page = int(request.args.get('page', 0))  # Page number
-        lines_per_page = int(request.args.get('lines_per_page', config['LOGGING']['VIEW_PAGE_LINES']))  # Lines per page
+        lines_per_page = int(request.args.get('lines_per_page', current_app.config['LOGGING']['VIEW_PAGE_LINES']))  # Lines per page
         log_file_path = log_file_name  # Path to the current log file
 
         if not os.path.isfile(log_file_path):
-            logger.error("Log file not found")
+            current_app.logger.error("Log file not found")
             return jsonify({"error": "Log file not found"}), 404
 
-        log_lines = list(tail(log_file_path, config['LOGGING']['VIEW_MAX_LINES']))
+        log_lines = list(tail(log_file_path, current_app.config['LOGGING']['VIEW_MAX_LINES']))
 
         log_lines = log_lines[::-1]  # Reverse the list
 
@@ -123,14 +121,15 @@ def register_api(app):
 
     @app.route('/delete_files', methods=['POST'])
     def delete_files():
+        log_file_name = current_app.logger.handlers[0].baseFilename
         file_paths = request.json.get('file_paths', [])
 
         if not file_paths:
             return jsonify({"error": "No files specified"}), 400
 
         errors = []
-        data_dir = os.path.abspath(config['DATA']['DATA_DIR'])
+        data_dir = os.path.abspath(current_app.config['DATA']['DATA_DIR'])
-        log_dir = os.path.abspath(config['LOGGING']['LOG_DIR'])
+        log_dir = os.path.abspath(current_app.config['LOGGING']['LOG_DIR'])
 
         for file_path in file_paths:
             if file_path.startswith('/data/'):
@@ -171,40 +170,39 @@ def register_api(app):
 
     @app.route('/data/<path:filename>')
     def download_data_file(filename):
-        data_dir = os.path.abspath(config['DATA']['DATA_DIR'])
+        data_dir = os.path.abspath(current_app.config['DATA']['DATA_DIR'])
         file_path = os.path.join(data_dir, filename)
 
         return send_from_directory(directory=data_dir, path=filename, as_attachment=True)
 
     @app.route('/log/<path:filename>')
     def download_log_file(filename):
-        log_dir = os.path.abspath(config['LOGGING']['LOG_DIR'])
+        log_dir = os.path.abspath(current_app.config['LOGGING']['LOG_DIR'])
         file_path = os.path.join(log_dir, filename)
 
         return send_from_directory(directory=log_dir, path=filename, as_attachment=True)
 
     @app.route('/tmp/<path:filename>')
     def download_tmp_file(filename):
-        tmp_dir = os.path.abspath(config['TEMP']['TEMP_DIR'])
+        tmp_dir = os.path.abspath(current_app.config['TEMP']['TEMP_DIR'])
         file_path = os.path.join(tmp_dir, filename)
 
         return send_from_directory(directory=tmp_dir, path=filename, as_attachment=True)
 
-
     @app.route('/config/lines_per_page')
     def get_lines_per_page():
-        lines_per_page = config['LOGGING']['VIEW_PAGE_LINES']
+        lines_per_page = current_app.config['LOGGING']['VIEW_PAGE_LINES']
         return jsonify({"lines_per_page": lines_per_page})
 
     @app.route('/scraping_status', methods=['GET'])
     def scraping_status():
         if scraper is None:
-            logger.debug("Scraper is not initialized.")
+            current_app.logger.debug("Scraper is not initialized.")
             return jsonify({"scraping_active": False})
 
         if scraper.scraping_active:
-            logger.debug("Scraping is active.")
+            current_app.logger.debug("Scraping is active.")
             return jsonify({"scraping_active": True})
         else:
-            logger.debug("Scraping is not active.")
+            current_app.logger.debug("Scraping is not active.")
             return jsonify({"scraping_active": False})

app/app.py

@@ -7,32 +7,33 @@ from app.api import register_api
 from app.config import load_config
 from app.filters import register_filters
 
-def init_app():
+from app.logging_config import init_logger
-    config = load_config()
 
-    # Initialize app
+def init_app():
     app = Flask(__name__)
 
-    # Load configuration
+    config = load_config()
-    app.config['SECRET_KEY'] = config['DEFAULT']['SECRET_KEY']
-    app.config['API_KEY'] = config['DEFAULT']['API_KEY']
 
-    app.config['DATA'] = config['DATA']
+    app.config['SECRET_KEY'] = config['DEFAULT']['SECRET_KEY']
-    app.config['TEMP'] = config['TEMP']
-    app.config['LOGGING'] = config['LOGGING']
     
     # Move bootstrap settings to root level
-    for key in config['BOOTSTRAP']:
+    for key, value in config.get('BOOTSTRAP', {}).items():
-        app.config[key.upper()] = config['BOOTSTRAP'][key]
+        app.config[key.upper()] = value
 
     bootstrap = Bootstrap5(app)
 
-    # Initialize global variables
+    # Store the entire config in Flask app
+    app.config.update(config)
+
+    # Initialize other settings
     app.config['SCRAPING_ACTIVE'] = False
     app.config['SCRAPING_THREAD'] = None
     app.config['DATA_FILE_NAME'] = None
     app.config['LOG_FILE_NAME'] = "log/" + datetime.now().strftime('%Y-%m-%d-%H-%M') + '.log'
 
+    # Initialize logging
+    app.logger = init_logger(app.config)
+
     # Register routes
     register_views(app)
     register_api(app)

app/config.py

@@ -1,7 +1,8 @@
-import configparser
+from configobj import ConfigObj
 import os
 
 def load_config():
-    config = configparser.ConfigParser()
+    config_path = os.path.join(os.path.dirname(__file__), '..', 'config.ini')
-    config.read(os.path.join(os.path.dirname(__file__), '..', 'config.ini'))
+
-    return config
+    # Load config while preserving sections as nested dicts
+    return ConfigObj(config_path)

app/logging_config.py

@@ -4,36 +4,31 @@ from queue import Queue
 import os
 from datetime import datetime
 
-from app.config import load_config
+from flask import current_app
 
-config = load_config()
+def init_logger(config):
+    LOG_DIR = config.get('LOGGING', {}).get('LOG_DIR', 'log')
 
-# Define the log directory and ensure it exists
+    if not os.path.exists(LOG_DIR):
-LOG_DIR = config['LOGGING']['LOG_DIR']
+        os.makedirs(LOG_DIR)
-if not os.path.exists(LOG_DIR):
-    os.makedirs(LOG_DIR)
 
-# Generate the log filename dynamically
+    log_file_name = os.path.join(LOG_DIR, datetime.now().strftime('%Y-%m-%d-%H-%M') + '.log')
-log_file_name = os.path.join(LOG_DIR, datetime.now().strftime('%Y-%m-%d-%H-%M') + '.log')
 
-# Initialize the logger
+    logger = logging.getLogger(__name__)
-logger = logging.getLogger(__name__)
+    logger.setLevel(logging.DEBUG)
-logger.setLevel(logging.DEBUG)
 
-# File handler
+    file_handler = logging.FileHandler(log_file_name, mode='w')
-file_handler = logging.FileHandler(log_file_name, mode='w')
+    file_handler.setLevel(logging.DEBUG)
-file_handler.setLevel(logging.DEBUG)  
+    formatter = logging.Formatter('%(asctime)s - %(levelname)s: %(message)s',
-formatter = logging.Formatter('%(asctime)s - %(levelname)s: %(message)s',
+                                datefmt='%m/%d/%Y %I:%M:%S %p')
-                              datefmt='%m/%d/%Y %I:%M:%S %p')
+    file_handler.setFormatter(formatter)
-file_handler.setFormatter(formatter)
+    logger.addHandler(file_handler)
-logger.addHandler(file_handler)
 
-# Queue handler for real-time logging 
+    log_queue = Queue()
-log_queue = Queue()
+    queue_handler = QueueHandler(log_queue)
-queue_handler = QueueHandler(log_queue)
+    queue_handler.setLevel(logging.DEBUG)
-queue_handler.setLevel(logging.DEBUG)
+    logger.addHandler(queue_handler)
-logger.addHandler(queue_handler)
+
+    logger.debug("Logger initialized")
     
-# Function to get logger in other modules
-def get_logger():
     return logger

app/models.py

@@ -6,14 +6,7 @@ import time
 from datetime import datetime, timedelta
 from requests.exceptions import ConnectionError, Timeout, RequestException
 
-from app.logging_config import get_logger
+from flask import current_app
-
-from app.config import load_config
-
-config = load_config()
-API_KEY = config['DEFAULT']['API_KEY']
-
-logger = get_logger()
 
 class Scraper:
     def __init__(self, faction_id, fetch_interval, run_interval, app):
@@ -23,19 +16,21 @@ class Scraper:
         self.end_time = datetime.now() + timedelta(days=run_interval)
         self.data_file_name = os.path.join(app.config['DATA']['DATA_DIR'], f"{self.faction_id}-{datetime.now().strftime('%Y-%m-%d-%H-%M')}.csv")
         self.scraping_active = False
+        self.API_KEY = app.config['DEFAULT']['API_KEY']
+        self.logger = app.logger
         
         print(self.data_file_name)
         
     def fetch_faction_data(self):
-        url = f"https://api.torn.com/faction/{self.faction_id}?selections=&key={API_KEY}"
+        url = f"https://api.torn.com/faction/{self.faction_id}?selections=&key={self.API_KEY}"
         response = requests.get(url)
         if response.status_code == 200:
             return response.json()
-        logger.warning(f"Failed to fetch faction data for faction ID {self.faction_id}. Response: {response.text}")
+        current_app.logger.warning(f"Failed to fetch faction data for faction ID {self.faction_id}. Response: {response.text}")
         return None
 
     def fetch_user_activity(self, user_id):
-        url = f"https://api.torn.com/user/{user_id}?selections=basic,profile&key={API_KEY}"
+        url = f"https://api.torn.com/user/{user_id}?selections=basic,profile&key={self.API_KEY}"
         retries = 3
         for attempt in range(retries):
             try:
@@ -43,45 +38,51 @@ class Scraper:
                 response.raise_for_status()
                 return response.json()
             except ConnectionError as e:
-                logger.error(f"Connection error while fetching user activity for user ID {user_id}: {e}")
+                current_app.logger.error(f"Connection error while fetching user activity for user ID {user_id}: {e}")
             except Timeout as e:
-                logger.error(f"Timeout error while fetching user activity for user ID {user_id}: {e}")
+                current_app.logger.error(f"Timeout error while fetching user activity for user ID {user_id}: {e}")
             except RequestException as e:
-                logger.error(f"Error while fetching user activity for user ID {user_id}: {e}")
+                current_app.logger.error(f"Error while fetching user activity for user ID {user_id}: {e}")
             if attempt < retries - 1:
+                current_app.logger.debug(f"Retrying {attempt + 1}/{retries} for user {user_id}")
                 time.sleep(2 ** attempt)  # Exponential backoff
         return None
 
-    def start_scraping(self) -> None:
+    def start_scraping(self, app) -> None:
         """Starts the scraping process until the end time is reached or stopped manually."""
         self.scraping_active = True
-        logger.info(f"Starting scraping for faction ID {self.faction_id}")
-        logger.debug(f"Fetch interval: {self.fetch_interval}s, Run interval: {self.run_interval} days, End time: {self.end_time}")
 
-        MAX_FAILURES = 5  # Stop after 5 consecutive failures
+        # Anwendungskontext explizit setzen
-        failure_count = 0
+        with app.app_context():
+            current_app.logger.info(f"Starting scraping for faction ID {self.faction_id}")
+            current_app.logger.debug(f"Fetch interval: {self.fetch_interval}s, Run interval: {self.run_interval} days, End time: {self.end_time}")
 
-        while datetime.now() < self.end_time and self.scraping_active:
+            MAX_FAILURES = 5  # Stop after 5 consecutive failures
-            logger.info(f"Fetching data at {datetime.now()}")
+            failure_count = 0
-            faction_data = self.fetch_faction_data()
 
-            if not faction_data or "members" not in faction_data:
+            while datetime.now() < self.end_time and self.scraping_active:
-                logger.warning(f"No faction data found for ID {self.faction_id} (Failure {failure_count + 1}/{MAX_FAILURES})")
+                current_app.logger.info(f"Fetching data at {datetime.now()}")
-                failure_count += 1
+                faction_data = self.fetch_faction_data()
-                if failure_count >= MAX_FAILURES:
+
-                    logger.error(f"Max failures reached ({MAX_FAILURES}). Stopping scraping.")
+                if not faction_data or "members" not in faction_data:
-                    break
+                    current_app.logger.warning(f"No faction data found for ID {self.faction_id} (Failure {failure_count + 1}/{MAX_FAILURES})")
+                    failure_count += 1
+                    if failure_count >= MAX_FAILURES:
+                        current_app.logger.error(f"Max failures reached ({MAX_FAILURES}). Stopping scraping.")
+                        break
+                    time.sleep(self.fetch_interval)
+                    continue
+                
+                current_app.logger.info(f"Fetched {len(faction_data['members'])} members for faction {self.faction_id}")
+                failure_count = 0  # Reset failure count on success
+                user_activity_data = self.process_faction_members(faction_data["members"])
+                self.save_data(user_activity_data)
+
+                current_app.logger.info(f"Data appended to {self.data_file_name}")
                 time.sleep(self.fetch_interval)
-                continue
             
-            failure_count = 0  # Reset failure count on success
+            self.handle_scraping_end()
-            user_activity_data = self.process_faction_members(faction_data["members"])
-            self.save_data(user_activity_data)
 
-            logger.info(f"Data appended to {self.data_file_name}")
-            time.sleep(self.fetch_interval)
-        
-        self.handle_scraping_end()
     
     def process_faction_members(self, members: Dict[str, Dict]) -> List[Dict]:
         """Processes and retrieves user activity for all faction members."""
@@ -96,16 +97,16 @@ class Scraper:
                     "status": user_activity.get("status", {}).get("state", ""),
                     "timestamp": datetime.now().timestamp(),
                 })
-                logger.info(f"Fetched data for user {user_id} ({user_activity.get('name', '')})")
+                current_app.logger.info(f"Fetched data for user {user_id} ({user_activity.get('name', '')})")
             else:
-                logger.warning(f"Failed to fetch data for user {user_id}")
+                current_app.logger.warning(f"Failed to fetch data for user {user_id}")
 
         return user_activity_data
 
     def save_data(self, user_activity_data: List[Dict]) -> None:
         """Saves user activity data to a CSV file."""
         if not user_activity_data:
-            logger.warning("No data to save.")
+            current_app.logger.warning("No data to save.")
             return
 
         df = pd.DataFrame(user_activity_data)
@@ -117,22 +118,22 @@ class Scraper:
         try:
             with open(self.data_file_name, "a" if file_exists else "w") as f:
                 df.to_csv(f, mode="a" if file_exists else "w", header=not file_exists, index=False)
-            logger.info(f"Data successfully saved to {self.data_file_name}")
+            current_app.logger.info(f"Data successfully saved to {self.data_file_name}")
         except Exception as e:
-            logger.error(f"Error saving data to {self.data_file_name}: {e}")
+            current_app.logger.error(f"Error saving data to {self.data_file_name}: {e}")
 
     def handle_scraping_end(self) -> None:
         """Handles cleanup and logging when scraping ends."""
         if not self.scraping_active:
-            logger.warning(f"Scraping stopped manually at {datetime.now()}")
+            current_app.logger.warning(f"Scraping stopped manually at {datetime.now()}")
         elif datetime.now() >= self.end_time:
-            logger.warning(f"Scraping stopped due to timeout at {datetime.now()} (Run interval: {self.run_interval} days)")
+            current_app.logger.warning(f"Scraping stopped due to timeout at {datetime.now()} (Run interval: {self.run_interval} days)")
         else:
-            logger.error(f"Unexpected stop at {datetime.now()}")
+            current_app.logger.error(f"Unexpected stop at {datetime.now()}")
 
-        logger.info("Scraping completed.")
+        current_app.logger.info("Scraping completed.")
         self.scraping_active = False
     
     def stop_scraping(self):
         self.scraping_active = False
-        logger.debug("Scraping stopped by user")
+        current_app.logger.debug("Scraping stopped by user")

app/util.py

@@ -1,13 +1,10 @@
 import os
 import zipfile
 from datetime import datetime, timedelta
-
+from flask import current_app
-from app.state import data_file_name, log_file_name
 
 from app.config import load_config
 
-config = load_config()
-
 def create_zip(file_paths, zip_name, app):
     temp_dir = os.path.abspath(app.config['TEMP']['TEMP_DIR'])
     zip_path = os.path.join(temp_dir, zip_name)
@@ -18,7 +15,7 @@ def create_zip(file_paths, zip_name, app):
     return zip_path
 
 def delete_old_zips():
-    temp_dir = os.path.abspath(config['TEMP']['TEMP_DIR'])
+    temp_dir = os.path.abspath(current_app.config['TEMP']['TEMP_DIR'])
     now = datetime.now()
     for filename in os.listdir(temp_dir):
         if filename.endswith('.zip'):
@@ -33,7 +30,7 @@ def tail(filename, n):
         yield ''
         return
 
-    page_size = int(config['LOGGING']['TAIL_PAGE_SIZE'])
+    page_size = int(current_app.config['LOGGING']['TAIL_PAGE_SIZE'])
     offsets = []
     count = _n = n if n >= 0 else -n
 

app/views.py

@@ -6,16 +6,9 @@ from app.forms import ScrapingForm
 from app.util import get_size
 from app.config import load_config
 from app.api import scraper as scraper
-from app.logging_config import get_logger
 
 from app.analysis import load_data, load_analysis_modules
 
-
-print(f"A imported log_file_name: {log_file_name}")
-
-config = load_config()
-logger = get_logger()
-
 views_bp = Blueprint("views", __name__)
 
 def register_views(app):
@@ -42,8 +35,8 @@ def register_views(app):
         if not scraper:
             print("Scraper not initialized")
         
-        data_dir = os.path.abspath(config['DATA']['DATA_DIR'])
+        data_dir = os.path.abspath(current_app.config['DATA']['DATA_DIR'])
-        log_dir = os.path.abspath(config['LOGGING']['LOG_DIR'])
+        log_dir = os.path.abspath(current_app.config['LOGGING']['LOG_DIR'])
 
         data_files = glob.glob(os.path.join(data_dir, "*.csv"))
         log_files = glob.glob(os.path.join(log_dir, "*.log"))