mischbeck/TornActivityTracker
1
0
Fork 0
Code Issues 5 Pull Requests Packages Projects Releases Wiki Activity

refactors logging and config

Browse Source
This commit is contained in:
Michael Beck
2025-02-10 16:34:11 +01:00
parent d1f562ce94
commit 33621bdec4
14 changed files with 114 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
app/analysis/basePlotAnalysis.py
View File

@@ -2,7 +2,6 @@ import os
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from flask import url_for
from abc import ABC, abstractmethod
from .base import BaseAnalysis

1
app/analysis/basePlotlyAnalysis.py
View File

@@ -1,7 +1,6 @@
import os
import pandas as pd
import plotly.graph_objects as go
from flask import url_for
from abc import ABC, abstractmethod
from .base import BaseAnalysis

4
app/analysis/plot_bar_activity-user.py
View File

@@ -4,13 +4,9 @@ import seaborn as sns
from .basePlotAnalysis import BasePlotAnalysis
from flask import current_app, url_for
from app.logging_config import get_logger
import matplotlib
matplotlib.use('Agg')
logger = get_logger()
class PlotTopActiveUsers(BasePlotAnalysis):
"""
Class for analyzing the most active users and generating a bar chart.

1
app/analysis/plot_bar_peak_hours.py
View File

@@ -3,7 +3,6 @@ import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from .basePlotAnalysis import BasePlotAnalysis
from flask import current_app, url_for
import matplotlib
matplotlib.use('Agg')

4
app/analysis/plot_line_activity-user.py
View File

@@ -4,13 +4,9 @@ import seaborn as sns
from .basePlotAnalysis import BasePlotAnalysis
from flask import current_app, url_for
from app.logging_config import get_logger
import matplotlib
matplotlib.use('Agg')
logger = get_logger()
class PlotLineActivityAllUsers(BasePlotAnalysis):
"""
Class for analyzing user activity trends over multiple days and generating a line graph.

4
app/analysis/plotly_heat_user-activity.py
View File

@@ -5,10 +5,6 @@ import plotly.graph_objects as go
from .basePlotlyAnalysis import BasePlotlyAnalysis
from flask import current_app, url_for
from app.logging_config import get_logger
logger = get_logger()
class PlotlyActivityHeatmap(BasePlotlyAnalysis):
"""
Class for analyzing user activity trends over multiple days and generating an interactive heatmap.

4
app/analysis/plotly_line_activity-user.py
View File

@@ -4,10 +4,6 @@ from plotly.subplots import make_subplots
from .basePlotlyAnalysis import BasePlotlyAnalysis
from flask import current_app, url_for
from app.logging_config import get_logger
logger = get_logger()
class PlotlyLineActivityAllUsers(BasePlotlyAnalysis):
"""
Class for analyzing user activity trends over multiple days and generating an interactive line graph.

44
app/api.py
View File

@@ -7,15 +7,10 @@ from datetime import datetime
import pandas as pd
from app.models import Scraper
from app.util import create_zip, delete_old_zips, tail, get_size
from app.util import create_zip, delete_old_zips, tail
from app.config import load_config
from app.logging_config import get_logger
from app.forms import ScrapingForm
config = load_config()
logger = get_logger()
log_file_name = logger.handlers[0].baseFilename
scraping_thread = None
scraper = None
scrape_lock = threading.Lock()
@@ -23,10 +18,11 @@ scrape_lock = threading.Lock()
def register_api(app):
@app.route('/start_scraping', methods=['POST'])
def start_scraping():
global scraping_thread, scraper
with scrape_lock:
scraper = current_app.config.get('SCRAPER')
if scraper is not None and scraper.scraping_active:
logger.warning("Can't start scraping process: scraping already in progress")
current_app.logger.warning("Can't start scraping process: scraping already in progress")
return jsonify({"status": "Scraping already in progress"})
form = ScrapingForm()
@@ -35,10 +31,10 @@ def register_api(app):
fetch_interval = form.fetch_interval.data
run_interval = form.run_interval.data
scraper = Scraper(faction_id, fetch_interval, run_interval, current_app)
scraper = Scraper(faction_id, fetch_interval, run_interval, app)
scraper.scraping_active = True
scraping_thread = threading.Thread(target=scraper.start_scraping)
scraping_thread = threading.Thread(target=scraper.start_scraping, args=(app,))
scraping_thread.daemon = True
scraping_thread.start()
@@ -56,19 +52,21 @@ def register_api(app):
scraper.stop_scraping()
current_app.config['SCRAPING_ACTIVE'] = False
logger.debug("Scraping stopped by user")
current_app.logger.debug("Scraping stopped by user")
return jsonify({"status": "Scraping stopped"})
@app.route('/logfile', methods=['GET'])
def logfile():
log_file_name = current_app.logger.handlers[0].baseFilename
page = int(request.args.get('page', 0)) # Page number
lines_per_page = int(request.args.get('lines_per_page', config['LOGGING']['VIEW_PAGE_LINES'])) # Lines per page
lines_per_page = int(request.args.get('lines_per_page', current_app.config['LOGGING']['VIEW_PAGE_LINES'])) # Lines per page
log_file_path = log_file_name # Path to the current log file
if not os.path.isfile(log_file_path):
logger.error("Log file not found")
current_app.logger.error("Log file not found")
return jsonify({"error": "Log file not found"}), 404
log_lines = list(tail(log_file_path, config['LOGGING']['VIEW_MAX_LINES']))
log_lines = list(tail(log_file_path, current_app.config['LOGGING']['VIEW_MAX_LINES']))
log_lines = log_lines[::-1] # Reverse the list
@@ -123,14 +121,15 @@ def register_api(app):
@app.route('/delete_files', methods=['POST'])
def delete_files():
log_file_name = current_app.logger.handlers[0].baseFilename
file_paths = request.json.get('file_paths', [])
if not file_paths:
return jsonify({"error": "No files specified"}), 400
errors = []
data_dir = os.path.abspath(config['DATA']['DATA_DIR'])
log_dir = os.path.abspath(config['LOGGING']['LOG_DIR'])
data_dir = os.path.abspath(current_app.config['DATA']['DATA_DIR'])
log_dir = os.path.abspath(current_app.config['LOGGING']['LOG_DIR'])
for file_path in file_paths:
if file_path.startswith('/data/'):
@@ -171,40 +170,39 @@ def register_api(app):
@app.route('/data/<path:filename>')
def download_data_file(filename):
data_dir = os.path.abspath(config['DATA']['DATA_DIR'])
data_dir = os.path.abspath(current_app.config['DATA']['DATA_DIR'])
file_path = os.path.join(data_dir, filename)
return send_from_directory(directory=data_dir, path=filename, as_attachment=True)
@app.route('/log/<path:filename>')
def download_log_file(filename):
log_dir = os.path.abspath(config['LOGGING']['LOG_DIR'])
log_dir = os.path.abspath(current_app.config['LOGGING']['LOG_DIR'])
file_path = os.path.join(log_dir, filename)
return send_from_directory(directory=log_dir, path=filename, as_attachment=True)
@app.route('/tmp/<path:filename>')
def download_tmp_file(filename):
tmp_dir = os.path.abspath(config['TEMP']['TEMP_DIR'])
tmp_dir = os.path.abspath(current_app.config['TEMP']['TEMP_DIR'])
file_path = os.path.join(tmp_dir, filename)
return send_from_directory(directory=tmp_dir, path=filename, as_attachment=True)
@app.route('/config/lines_per_page')
def get_lines_per_page():
lines_per_page = config['LOGGING']['VIEW_PAGE_LINES']
lines_per_page = current_app.config['LOGGING']['VIEW_PAGE_LINES']
return jsonify({"lines_per_page": lines_per_page})
@app.route('/scraping_status', methods=['GET'])
def scraping_status():
if scraper is None:
logger.debug("Scraper is not initialized.")
current_app.logger.debug("Scraper is not initialized.")
return jsonify({"scraping_active": False})
if scraper.scraping_active:
logger.debug("Scraping is active.")
current_app.logger.debug("Scraping is active.")
return jsonify({"scraping_active": True})
else:
logger.debug("Scraping is not active.")
current_app.logger.debug("Scraping is not active.")
return jsonify({"scraping_active": False})

27
app/app.py
View File

@@ -7,35 +7,36 @@ from app.api import register_api
from app.config import load_config
from app.filters import register_filters
def init_app():
config = load_config()
from app.logging_config import init_logger
# Initialize app
def init_app():
app = Flask(__name__)
# Load configuration
app.config['SECRET_KEY'] = config['DEFAULT']['SECRET_KEY']
app.config['API_KEY'] = config['DEFAULT']['API_KEY']
config = load_config()
app.config['DATA'] = config['DATA']
app.config['TEMP'] = config['TEMP']
app.config['LOGGING'] = config['LOGGING']
app.config['SECRET_KEY'] = config['DEFAULT']['SECRET_KEY']
# Move bootstrap settings to root level
for key in config['BOOTSTRAP']:
app.config[key.upper()] = config['BOOTSTRAP'][key]
for key, value in config.get('BOOTSTRAP', {}).items():
app.config[key.upper()] = value
bootstrap = Bootstrap5(app)
# Initialize global variables
# Store the entire config in Flask app
app.config.update(config)
# Initialize other settings
app.config['SCRAPING_ACTIVE'] = False
app.config['SCRAPING_THREAD'] = None
app.config['DATA_FILE_NAME'] = None
app.config['LOG_FILE_NAME'] = "log/" + datetime.now().strftime('%Y-%m-%d-%H-%M') + '.log'
# Initialize logging
app.logger = init_logger(app.config)
# Register routes
register_views(app)
register_api(app)
register_filters(app)
return app

9
app/config.py
View File

@@ -1,7 +1,8 @@
import configparser
from configobj import ConfigObj
import os
def load_config():
config = configparser.ConfigParser()
config.read(os.path.join(os.path.dirname(__file__), '..', 'config.ini'))
return config
config_path = os.path.join(os.path.dirname(__file__), '..', 'config.ini')
# Load config while preserving sections as nested dicts
return ConfigObj(config_path)

47
app/logging_config.py
View File

@@ -4,36 +4,31 @@ from queue import Queue
import os
from datetime import datetime
from app.config import load_config
from flask import current_app
config = load_config()
def init_logger(config):
LOG_DIR = config.get('LOGGING', {}).get('LOG_DIR', 'log')
# Define the log directory and ensure it exists
LOG_DIR = config['LOGGING']['LOG_DIR']
if not os.path.exists(LOG_DIR):
os.makedirs(LOG_DIR)
if not os.path.exists(LOG_DIR):
os.makedirs(LOG_DIR)
# Generate the log filename dynamically
log_file_name = os.path.join(LOG_DIR, datetime.now().strftime('%Y-%m-%d-%H-%M') + '.log')
log_file_name = os.path.join(LOG_DIR, datetime.now().strftime('%Y-%m-%d-%H-%M') + '.log')
# Initialize the logger
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
# File handler
file_handler = logging.FileHandler(log_file_name, mode='w')
file_handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(levelname)s: %(message)s',
datefmt='%m/%d/%Y %I:%M:%S %p')
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
file_handler = logging.FileHandler(log_file_name, mode='w')
file_handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(levelname)s: %(message)s',
datefmt='%m/%d/%Y %I:%M:%S %p')
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
# Queue handler for real-time logging
log_queue = Queue()
queue_handler = QueueHandler(log_queue)
queue_handler.setLevel(logging.DEBUG)
logger.addHandler(queue_handler)
log_queue = Queue()
queue_handler = QueueHandler(log_queue)
queue_handler.setLevel(logging.DEBUG)
logger.addHandler(queue_handler)
# Function to get logger in other modules
def get_logger():
return logger
logger.debug("Logger initialized")
return logger

95
app/models.py
View File

@@ -6,14 +6,7 @@ import time
from datetime import datetime, timedelta
from requests.exceptions import ConnectionError, Timeout, RequestException
from app.logging_config import get_logger
from app.config import load_config
config = load_config()
API_KEY = config['DEFAULT']['API_KEY']
logger = get_logger()
from flask import current_app
class Scraper:
def __init__(self, faction_id, fetch_interval, run_interval, app):
@@ -23,19 +16,21 @@ class Scraper:
self.end_time = datetime.now() + timedelta(days=run_interval)
self.data_file_name = os.path.join(app.config['DATA']['DATA_DIR'], f"{self.faction_id}-{datetime.now().strftime('%Y-%m-%d-%H-%M')}.csv")
self.scraping_active = False
self.API_KEY = app.config['DEFAULT']['API_KEY']
self.logger = app.logger
print(self.data_file_name)
def fetch_faction_data(self):
url = f"https://api.torn.com/faction/{self.faction_id}?selections=&key={API_KEY}"
url = f"https://api.torn.com/faction/{self.faction_id}?selections=&key={self.API_KEY}"
response = requests.get(url)
if response.status_code == 200:
return response.json()
logger.warning(f"Failed to fetch faction data for faction ID {self.faction_id}. Response: {response.text}")
current_app.logger.warning(f"Failed to fetch faction data for faction ID {self.faction_id}. Response: {response.text}")
return None
def fetch_user_activity(self, user_id):
url = f"https://api.torn.com/user/{user_id}?selections=basic,profile&key={API_KEY}"
url = f"https://api.torn.com/user/{user_id}?selections=basic,profile&key={self.API_KEY}"
retries = 3
for attempt in range(retries):
try:
@@ -43,45 +38,51 @@ class Scraper:
response.raise_for_status()
return response.json()
except ConnectionError as e:
logger.error(f"Connection error while fetching user activity for user ID {user_id}: {e}")
current_app.logger.error(f"Connection error while fetching user activity for user ID {user_id}: {e}")
except Timeout as e:
logger.error(f"Timeout error while fetching user activity for user ID {user_id}: {e}")
current_app.logger.error(f"Timeout error while fetching user activity for user ID {user_id}: {e}")
except RequestException as e:
logger.error(f"Error while fetching user activity for user ID {user_id}: {e}")
current_app.logger.error(f"Error while fetching user activity for user ID {user_id}: {e}")
if attempt < retries - 1:
current_app.logger.debug(f"Retrying {attempt + 1}/{retries} for user {user_id}")
time.sleep(2 ** attempt) # Exponential backoff
return None
def start_scraping(self) -> None:
def start_scraping(self, app) -> None:
"""Starts the scraping process until the end time is reached or stopped manually."""
self.scraping_active = True
logger.info(f"Starting scraping for faction ID {self.faction_id}")
logger.debug(f"Fetch interval: {self.fetch_interval}s, Run interval: {self.run_interval} days, End time: {self.end_time}")
MAX_FAILURES = 5 # Stop after 5 consecutive failures
failure_count = 0
# Anwendungskontext explizit setzen
with app.app_context():
current_app.logger.info(f"Starting scraping for faction ID {self.faction_id}")
current_app.logger.debug(f"Fetch interval: {self.fetch_interval}s, Run interval: {self.run_interval} days, End time: {self.end_time}")
while datetime.now() < self.end_time and self.scraping_active:
logger.info(f"Fetching data at {datetime.now()}")
faction_data = self.fetch_faction_data()
MAX_FAILURES = 5 # Stop after 5 consecutive failures
failure_count = 0
if not faction_data or "members" not in faction_data:
logger.warning(f"No faction data found for ID {self.faction_id} (Failure {failure_count + 1}/{MAX_FAILURES})")
failure_count += 1
if failure_count >= MAX_FAILURES:
logger.error(f"Max failures reached ({MAX_FAILURES}). Stopping scraping.")
break
while datetime.now() < self.end_time and self.scraping_active:
current_app.logger.info(f"Fetching data at {datetime.now()}")
faction_data = self.fetch_faction_data()
if not faction_data or "members" not in faction_data:
current_app.logger.warning(f"No faction data found for ID {self.faction_id} (Failure {failure_count + 1}/{MAX_FAILURES})")
failure_count += 1
if failure_count >= MAX_FAILURES:
current_app.logger.error(f"Max failures reached ({MAX_FAILURES}). Stopping scraping.")
break
time.sleep(self.fetch_interval)
continue
current_app.logger.info(f"Fetched {len(faction_data['members'])} members for faction {self.faction_id}")
failure_count = 0 # Reset failure count on success
user_activity_data = self.process_faction_members(faction_data["members"])
self.save_data(user_activity_data)
current_app.logger.info(f"Data appended to {self.data_file_name}")
time.sleep(self.fetch_interval)
continue
self.handle_scraping_end()
failure_count = 0 # Reset failure count on success
user_activity_data = self.process_faction_members(faction_data["members"])
self.save_data(user_activity_data)
logger.info(f"Data appended to {self.data_file_name}")
time.sleep(self.fetch_interval)
self.handle_scraping_end()
def process_faction_members(self, members: Dict[str, Dict]) -> List[Dict]:
"""Processes and retrieves user activity for all faction members."""
@@ -96,16 +97,16 @@ class Scraper:
"status": user_activity.get("status", {}).get("state", ""),
"timestamp": datetime.now().timestamp(),
})
logger.info(f"Fetched data for user {user_id} ({user_activity.get('name', '')})")
current_app.logger.info(f"Fetched data for user {user_id} ({user_activity.get('name', '')})")
else:
logger.warning(f"Failed to fetch data for user {user_id}")
current_app.logger.warning(f"Failed to fetch data for user {user_id}")
return user_activity_data
def save_data(self, user_activity_data: List[Dict]) -> None:
"""Saves user activity data to a CSV file."""
if not user_activity_data:
logger.warning("No data to save.")
current_app.logger.warning("No data to save.")
return
df = pd.DataFrame(user_activity_data)
@@ -117,22 +118,22 @@ class Scraper:
try:
with open(self.data_file_name, "a" if file_exists else "w") as f:
df.to_csv(f, mode="a" if file_exists else "w", header=not file_exists, index=False)
logger.info(f"Data successfully saved to {self.data_file_name}")
current_app.logger.info(f"Data successfully saved to {self.data_file_name}")
except Exception as e:
logger.error(f"Error saving data to {self.data_file_name}: {e}")
current_app.logger.error(f"Error saving data to {self.data_file_name}: {e}")
def handle_scraping_end(self) -> None:
"""Handles cleanup and logging when scraping ends."""
if not self.scraping_active:
logger.warning(f"Scraping stopped manually at {datetime.now()}")
current_app.logger.warning(f"Scraping stopped manually at {datetime.now()}")
elif datetime.now() >= self.end_time:
logger.warning(f"Scraping stopped due to timeout at {datetime.now()} (Run interval: {self.run_interval} days)")
current_app.logger.warning(f"Scraping stopped due to timeout at {datetime.now()} (Run interval: {self.run_interval} days)")
else:
logger.error(f"Unexpected stop at {datetime.now()}")
current_app.logger.error(f"Unexpected stop at {datetime.now()}")
logger.info("Scraping completed.")
current_app.logger.info("Scraping completed.")
self.scraping_active = False
def stop_scraping(self):
self.scraping_active = False
logger.debug("Scraping stopped by user")
current_app.logger.debug("Scraping stopped by user")

9
app/util.py
View File

@@ -1,13 +1,10 @@
import os
import zipfile
from datetime import datetime, timedelta
from app.state import data_file_name, log_file_name
from flask import current_app
from app.config import load_config
config = load_config()
def create_zip(file_paths, zip_name, app):
temp_dir = os.path.abspath(app.config['TEMP']['TEMP_DIR'])
zip_path = os.path.join(temp_dir, zip_name)
@@ -18,7 +15,7 @@ def create_zip(file_paths, zip_name, app):
return zip_path
def delete_old_zips():
temp_dir = os.path.abspath(config['TEMP']['TEMP_DIR'])
temp_dir = os.path.abspath(current_app.config['TEMP']['TEMP_DIR'])
now = datetime.now()
for filename in os.listdir(temp_dir):
if filename.endswith('.zip'):
@@ -33,7 +30,7 @@ def tail(filename, n):
yield ''
return
page_size = int(config['LOGGING']['TAIL_PAGE_SIZE'])
page_size = int(current_app.config['LOGGING']['TAIL_PAGE_SIZE'])
offsets = []
count = _n = n if n >= 0 else -n

11
app/views.py
View File

@@ -6,16 +6,9 @@ from app.forms import ScrapingForm
from app.util import get_size
from app.config import load_config
from app.api import scraper as scraper
from app.logging_config import get_logger
from app.analysis import load_data, load_analysis_modules
print(f"A imported log_file_name: {log_file_name}")
config = load_config()
logger = get_logger()
views_bp = Blueprint("views", __name__)
def register_views(app):
@@ -42,8 +35,8 @@ def register_views(app):
if not scraper:
print("Scraper not initialized")
data_dir = os.path.abspath(config['DATA']['DATA_DIR'])
log_dir = os.path.abspath(config['LOGGING']['LOG_DIR'])
data_dir = os.path.abspath(current_app.config['DATA']['DATA_DIR'])
log_dir = os.path.abspath(current_app.config['LOGGING']['LOG_DIR'])
data_files = glob.glob(os.path.join(data_dir, "*.csv"))
log_files = glob.glob(os.path.join(log_dir, "*.log"))