1st restructure

app/app.py

@@ -0,0 +1,470 @@
+from flask import Flask, request, render_template, Response, jsonify, url_for
+from flask_bootstrap import Bootstrap5 # from package boostrap_flask
+from app.forms import ScrapingForm
+import requests
+import pandas as pd
+import time
+from datetime import datetime, timedelta
+import threading
+import logging
+from logging.handlers import QueueHandler
+from queue import Queue
+import os
+import glob
+from datetime import datetime
+from flask import send_from_directory
+import configparser
+
+import zipfile
+import os
+from datetime import timedelta
+
+
+app = Flask(__name__)
+
+# Load configuration
+config = configparser.ConfigParser()
+config.read('config.ini')
+
+app.config['SECRET_KEY'] = config['DEFAULT']['SECRET_KEY']
+API_KEY = config['DEFAULT']['API_KEY']
+
+bootstrap = Bootstrap5(app)
+
+# Move every setting from config['BOOTSTRAP'] to the root level of config
+for key in config['BOOTSTRAP']:
+    key = key.upper()
+    app.config[key] = config['BOOTSTRAP'][key]
+    if key == 'SECRET_KEY':
+        continue
+    elif key == 'API_KEY':
+        continue
+    print(f"Loaded config: {key} = {app.config[key]}")
+
+# Global state
+scraping_active = False
+scraping_thread = None
+data_file_name = None
+log_file_name = "log/" + datetime.now().strftime('%Y-%m-%d-%H-%M') + '.log'
+
+# Initialize the logger
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)  # Adjust as needed
+
+# Make any logger.info() call go to both the log file and the queue.
+# 1) FILE HANDLER
+file_handler = logging.FileHandler(log_file_name, mode='w')
+file_handler.setLevel(logging.DEBUG)  # or INFO, WARNING, etc.
+formatter = logging.Formatter('%(asctime)s - %(levelname)s: %(message)s',
+                              datefmt='%m/%d/%Y %I:%M:%S %p')
+file_handler.setFormatter(formatter)
+
+logger.addHandler(file_handler)
+
+# 2) QUEUE HANDLER
+log_queue = Queue()
+queue_handler = QueueHandler(log_queue)
+queue_handler.setLevel(logging.DEBUG)
+logger.addHandler(queue_handler)
+
+
+
+def create_zip(file_paths, zip_name):
+    zip_path = os.path.join(app.root_path, 'temp', zip_name)
+    with zipfile.ZipFile(zip_path, 'w') as zipf:
+        for file_path in file_paths:
+            arcname = os.path.basename(file_path)
+            zipf.write(file_path, arcname)
+    return zip_path
+
+def delete_old_zips():
+    temp_dir = os.path.join(app.root_path, 'temp')
+    now = datetime.now()
+    for filename in os.listdir(temp_dir):
+        if filename.endswith('.zip'):
+            file_path = os.path.join(temp_dir, filename)
+            file_time = datetime.fromtimestamp(os.path.getmtime(file_path))
+            if now - file_time > timedelta(hours=1):
+                os.remove(file_path)
+                logger.info(f"Deleted old zip file: {filename}")
+
+def fetch_faction_data(faction_id):
+    url = f"https://api.torn.com/faction/{faction_id}?selections=&key={API_KEY}"
+    response = requests.get(url)
+    if response.status_code == 200:
+        logger.info(f"Fetched data for faction ID {faction_id}")
+        return response.json()
+    else:
+        logger.warning(f"Failed to fetch faction data for faction ID {faction_id}")
+        return None
+
+def fetch_user_activity(user_id):
+    url = f"https://api.torn.com/user/{user_id}?selections=basic,profile&key={API_KEY}"
+    response = requests.get(url)
+    if response.status_code == 200:
+        return response.json()
+    else:
+        logger.error(f"Failed to fetch user activity for user ID {user_id}")
+        return None
+
+def scrape_data(faction_id, fetch_interval, run_interval):
+    global scraping_active
+    global data_file_name
+    
+    end_time = datetime.now() + timedelta(days=run_interval)
+    data_file_name = f"data/{faction_id}-{datetime.now().strftime('%Y-%m-%d-%H-%M')}.csv"
+
+    while datetime.now() < end_time and scraping_active:
+        logger.info(f"Fetching data at {datetime.now()}")
+        faction_data = fetch_faction_data(faction_id)
+        if faction_data and 'members' in faction_data:
+            user_activity_data = []
+            for user_id, user_info in faction_data['members'].items():
+                user_activity = fetch_user_activity(user_id)
+                if user_activity:
+                    user_activity_data.append({
+                        'user_id': user_id,
+                        'name': user_activity.get('name', ''),
+                        'last_action': user_activity.get('last_action', {}).get('timestamp', 0),
+                        'stadata_file_nametus': user_activity.get('status', {}).get('state', ''),
+                        'timestamp': datetime.now().timestamp()
+                    })
+                    logger.info(f"Fetched data for user {user_id} ({user_activity.get('name', '')})")
+
+            # Append data to the file
+            df = pd.DataFrame(user_activity_data)
+            df['last_action'] = pd.to_datetime(df['last_action'], unit='s')
+            df['timestamp'] = pd.to_datetime(df['timestamp'], unit='s')
+
+            if not os.path.isfile(data_file_name):
+                df.to_csv(data_file_name, index=False)
+            else:
+                df.to_csv(data_file_name, mode='a', header=False, index=False)
+
+            logger.info(f"Data appended to {data_file_name}")
+
+        time.sleep(fetch_interval)
+    else:
+        if datetime.now() < end_time:
+            logger.warning(f"Scraping stopped at {datetime.now()}")
+        elif scraping_active == False:
+            logger.warning(f"Scraping stopped at {datetime.now()} due to user request")
+        else:
+            logger.error(f"Scraping stopped due to timeout at {datetime.now()}")
+    logger.info("Scraping completed.")
+    scraping_active = False
+
+def generate_statistics(df):
+    df['hour'] = df['timestamp'].dt.hour
+    activity_by_hour = df.groupby('hour').size()
+    return activity_by_hour
+
+# Taken from:
+# https://gist.github.com/amitsaha/5990310?permalink_comment_id=3017951#gistcomment-3017951
+def tail(filename, n):
+    stat = os.stat(filename)
+    n = int(n)
+    if stat.st_size == 0 or n == 0:
+        yield ''
+        return
+
+    page_size = int(config['LOGGING']['TAIL_PAGE_SIZE'])
+    offsets = []
+    count = _n = n if n >= 0 else -n
+
+    last_byte_read = last_nl_byte = starting_offset = stat.st_size - 1
+
+    with open(filename, 'r') as f:
+        while count > 0:
+            starting_byte = last_byte_read - page_size
+            if last_byte_read == 0:
+                offsets.append(0)
+                break
+            elif starting_byte < 0:
+                f.seek(0)
+                text = f.read(last_byte_read)
+            else:
+                f.seek(starting_byte)
+                text = f.read(page_size)
+
+            for i in range(-1, -1*len(text)-1, -1):
+                last_byte_read -= 1
+                if text[i] == '\n':
+                    last_nl_byte = last_byte_read
+                    starting_offset = last_nl_byte + 1
+                    offsets.append(starting_offset)
+            count -= 1
+
+    offsets = offsets[len(offsets)-_n:]
+    offsets.reverse()
+
+    with open(filename, 'r') as f:
+        for i, offset in enumerate(offsets):
+            f.seek(offset)
+
+            if i == len(offsets) - 1:
+                yield f.read()
+            else:
+                bytes_to_read = offsets[i+1] - offset
+                yield f.read(bytes_to_read)
+
+def is_data_file_in_use(filename):
+    if(data_file_name == None):
+        return False
+    if os.path.join(app.root_path, filename.lstrip('/')) == os.path.join(app.root_path, data_file_name.lstrip('/')) and scraping_active:
+        return True
+    return False
+
+@app.route('/is_data_file_in_use/<path:filename>')
+def is_data_file_in_use_json(filename):
+    return jsonify(is_data_file_in_use(filename))
+
+def is_log_file_in_use(filename):
+    if(log_file_name == None):
+        return False
+    if os.path.join(app.root_path, filename.lstrip('/')) == os.path.join(app.root_path, log_file_name.lstrip('/')):
+        return True
+    return False
+
+@app.route('/is_log_file_in_use/<path:filename>')
+def is_log_file_in_use_json(filename):
+    print(filename)
+    return jsonify(is_log_file_in_use(filename))
+
+@app.route('/')
+def index():
+    form = ScrapingForm()
+    return render_template('index.html', form=form)
+
+@app.route('/start_scraping', methods=['POST'])
+def start_scraping():
+    global scraping_active, scraping_thread
+    form = ScrapingForm()
+    if form.validate_on_submit():
+        if scraping_active:
+            logger.warning("Can't start scraping process: scraping already in progress")
+            return jsonify({"status": "Scraping already in progress"})
+        
+        scraping_active = True
+        
+        faction_id = form.faction_id.data
+        fetch_interval = form.fetch_interval.data
+        run_interval = form.run_interval.data
+
+        # Start scraping in a separate thread
+        scraping_thread = threading.Thread(target=scrape_data, args=(faction_id, fetch_interval, run_interval))
+        scraping_thread.daemon = True
+        scraping_thread.start()
+
+        return jsonify({"status": "Scraping started"})
+    return jsonify({"status": "Invalid form data"})
+
+@app.route('/stop_scraping', methods=['POST'])
+def stop_scraping():
+    global scraping_active
+    if not scraping_active:
+        return jsonify({"status": "No scraping in progress"})
+    
+    scraping_active = False
+    logger.debug("scraping_active set to False")
+    return jsonify({"status": "Scraping stopped"})
+
+@app.route('/scraping_status', methods=['GET'])
+def scraping_status():
+    global scraping_active
+    logger.debug(f"scraping_status called: scraping_active = {scraping_active}")
+    return jsonify({"scraping_active": scraping_active})
+
+@app.route('/logs')
+def logs():
+    def generate():
+        while True:
+            if not log_queue.empty():
+                log = log_queue.get().getMessage()
+                yield f"data: {log}\n\n"
+            time.sleep(0.1)
+    return Response(generate(), mimetype='text/event-stream')
+
+@app.route('/logfile', methods=['GET'])
+def logfile():
+    page = int(request.args.get('page', 0))  # Page number
+    lines_per_page = int(request.args.get('lines_per_page', config['LOGGING']['VIEW_PAGE_LINES']))  # Lines per page
+    log_file_path = log_file_name  # Path to the current log file
+
+    if not os.path.isfile(log_file_path):
+        logging.error("Log file not found")
+        return jsonify({"error": "Log file not found"}), 404
+
+    log_lines = list(tail(log_file_path, config['LOGGING']['VIEW_MAX_LINES']))
+    
+    log_lines = log_lines[::-1]  # Reverse the list
+    
+    start = page * lines_per_page
+    end = start + lines_per_page
+    paginated_lines = log_lines[start:end] if start < len(log_lines) else []
+
+    return jsonify({
+        "log": paginated_lines,
+        "total_lines": len(log_lines),
+        "pages": (len(log_lines) + lines_per_page - 1) // lines_per_page,
+        "start_line": len(log_lines) - start  # Starting line number for the current page
+    })
+
+@app.route('/results')
+def results():
+    # Assuming the scraping is done and data is saved somewhere
+    faction_id = request.args.get('faction_id')
+    filename = f"data/{faction_id}-{datetime.now().strftime('%Y-%m-%d-%H-%M')}.csv"
+    if os.path.isfile(filename):
+        df = pd.read_csv(filename)
+        stats = generate_statistics(df)
+        return render_template('results.html', stats=stats.to_dict())
+    else:
+        return "No data found."
+
+@app.route('/analyze')
+def analyze():
+    return render_template('analyze.html');
+
+@app.route('/log_viewer')
+def log_viewer():
+    return render_template('log_viewer.html');
+
+@app.route('/download_results')
+def download_results():
+    data_files = glob.glob("data/*.csv")
+    log_files = glob.glob("log/*.log")
+    
+    def get_file_info(file_path):
+        return {
+            "name": file_path,
+            "name_display": os.path.basename(file_path),
+            "last_modified": os.path.getmtime(file_path),
+            "created": os.path.getctime(file_path),
+            "size": get_size(file_path)
+        }
+       
+    data_files_info = [get_file_info(file) for file in data_files]
+    log_files_info = [get_file_info(file) for file in log_files]
+    
+    for data_file in data_files_info:
+        if is_data_file_in_use(data_file['name']):
+            data_file['active'] = True
+        else:
+            data_file['active'] = False
+    
+    for log_file in log_files_info:
+        if is_log_file_in_use(log_file['name']):
+            log_file['active'] = True
+        else:
+            log_file['active'] = False
+    
+    files = {"data": data_files_info, "log": log_files_info}
+    
+    return render_template('download_results.html', files=files)
+
+@app.route('/download_files', methods=['POST'])
+def download_files():
+    delete_old_zips()  # Clean up old zip files
+    
+    file_paths = request.json.get('file_paths', [])
+    
+    if not file_paths:
+        return jsonify({"error": "No files specified"}), 400
+    
+    # Validate and correct file paths
+    valid_file_paths = []
+    for file_path in file_paths:
+        if file_path.startswith('/data/'):
+            corrected_path = file_path.lstrip('/')
+            full_path = os.path.join(app.root_path, corrected_path)
+            if os.path.isfile(full_path):
+                valid_file_paths.append(full_path)
+        elif file_path.startswith('/log/'):
+            corrected_path = file_path.lstrip('/')
+            full_path = os.path.join(app.root_path, corrected_path)
+            if os.path.isfile(full_path):
+                valid_file_paths.append(full_path)
+    
+    if not valid_file_paths:
+        return jsonify({"error": "No valid files specified"}), 400
+    
+    # Create a unique zip file name
+    zip_name = f"files_{datetime.now().strftime('%Y%m%d%H%M%S')}.zip"
+    zip_path = create_zip(valid_file_paths, zip_name)
+    
+    return send_from_directory(directory='temp', path=zip_name, as_attachment=True)
+
+@app.route('/delete_files', methods=['POST'])
+def delete_files():
+    file_paths = request.json.get('file_paths', [])
+    
+    if not file_paths:
+        return jsonify({"error": "No files specified"}), 400
+    
+    errors = []
+    for file_path in file_paths:
+        full_file_path = os.path.join(app.root_path, file_path.lstrip('/'))
+        print(f"Attempting to delete: {file_path}")  # Debugging line
+        print(f"Full path: {full_file_path}")  # Debugging line
+        print(f"file_path: {file_path}")  # Debugging line
+
+        # Check if the file is in either the logs or the data files folder
+        if not (full_file_path.startswith(os.path.join(app.root_path, 'log')) or
+                full_file_path.startswith(os.path.join(app.root_path, 'data'))):
+            errors.append({"file": file_path, "error": "File not in allowed directory"})
+            continue
+
+        # Check if it's the currently active log file
+        if is_log_file_in_use(file_path):
+            errors.append({"file": file_path, "error": "Cannot delete active log file."})
+            continue
+        
+        # Check if it's an active data file
+        if is_data_file_in_use(file_path):
+            errors.append({"file": file_path, "error": "Cannot delete active data file."})
+            continue
+
+        if not os.path.isfile(full_file_path):
+            errors.append({"file": file_path, "error": "File not found"})
+            continue
+
+        try:
+            os.remove(full_file_path)
+        except Exception as e:
+            errors.append({"file": file_path, "error": str(e)})
+
+    if errors:
+        return jsonify({"errors": errors}), 207  # Multi-Status response
+    return jsonify({"success": True}), 200
+
+@app.template_filter('datetimeformat')
+def datetimeformat(value):
+    return datetime.fromtimestamp(value).strftime('%Y-%m-%d %H:%M:%S')
+
+def get_size(path):
+        size = os.path.getsize(path)
+        if size < 1024:
+            return f"{size} bytes"
+        elif size < pow(1024,2):
+            return f"{round(size/1024, 2)} KB"
+        elif size < pow(1024,3):
+            return f"{round(size/(pow(1024,2)), 2)} MB"
+        elif size < pow(1024,4):
+            return f"{round(size/(pow(1024,3)), 2)} GB"
+
+@app.route('/data/<path:filename>')
+def download_data_file(filename):
+    return send_from_directory('data', filename)
+
+@app.route('/log/<path:filename>')
+def download_log_file(filename):
+    return send_from_directory('log', filename)
+
+@app.route('/config/lines_per_page')
+def get_lines_per_page():
+    lines_per_page = config['LOGGING']['VIEW_PAGE_LINES']
+    return jsonify({"lines_per_page": lines_per_page})
+
+if __name__ == '__main__':
+    app.run(debug=True, threaded=True)

app/forms.py

@@ -0,0 +1,9 @@
+from flask_wtf import FlaskForm
+from wtforms import StringField, IntegerField, SubmitField
+from wtforms.validators import DataRequired
+
+class ScrapingForm(FlaskForm):
+    faction_id = StringField('Faction ID', validators=[DataRequired()], default='9686')
+    fetch_interval = IntegerField('Fetch Interval (seconds)', validators=[DataRequired()], default=60)
+    run_interval = IntegerField('Run Interval (days)', validators=[DataRequired()], default=1)
+    submit = SubmitField('Start')

app/static/color_mode.js

@@ -0,0 +1,26 @@
+
+document.addEventListener('DOMContentLoaded', () => {
+  const themeToggle = document.getElementById('bd-theme');
+
+  // Check if a theme preference is saved in localStorage
+  const savedTheme = localStorage.getItem('theme');
+  
+  if (savedTheme === 'dark') {
+      themeToggle.checked = true;
+      document.documentElement.setAttribute('data-bs-theme', 'dark');
+  } else {
+      themeToggle.checked = false;
+      document.documentElement.setAttribute('data-bs-theme', 'light');
+  }
+
+  // Add event listener to toggle theme on checkbox change
+  themeToggle.addEventListener('change', () => {
+      if (themeToggle.checked) {
+          document.documentElement.setAttribute('data-bs-theme', 'dark');
+          localStorage.setItem('theme', 'dark');
+      } else {
+          document.documentElement.setAttribute('data-bs-theme', 'light');
+          localStorage.setItem('theme', 'light');
+      }
+  });
+});

app/static/download_results.js

@@ -0,0 +1,104 @@
+async function deleteFiles(filePaths) {
+    try {
+        const response = await fetch('/delete_files', {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({ file_paths: filePaths })
+        });
+
+        const data = await response.json();
+
+        if (data.success) {
+            alert('Files deleted successfully');
+            location.reload();
+        } else {
+            alert(`Error deleting files: ${JSON.stringify(data.errors)}`);
+        }
+    } catch (error) {
+        console.error('Error:', error);
+        alert('An error occurred while deleting files.');
+    }
+}
+
+function getSelectedFiles() {
+    return Array.from(document.querySelectorAll('input[name="fileCheckbox"]:checked'))
+        .map(checkbox => checkbox.value);
+}
+
+function deleteSelectedFiles() {
+    const selectedFiles = getSelectedFiles();
+    if (selectedFiles.length > 0) {
+        deleteFiles(selectedFiles);
+    } else {
+        alert('No files selected');
+    }
+}
+
+async function downloadSelectedFiles() {
+    const selectedFiles = getSelectedFiles();
+    if (selectedFiles.length === 0) {
+        alert('No files selected');
+        return;
+    }
+
+    try {
+        const response = await fetch('/download_files', {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({ file_paths: selectedFiles })
+        });
+
+        if (!response.ok) {
+            const errorData = await response.json();
+            throw new Error(errorData.error || 'Failed to download files.');
+        }
+
+        const blob = await response.blob();
+        if (blob.type !== 'application/zip') {
+            throw new Error('Received invalid ZIP file.');
+        }
+
+        const url = window.URL.createObjectURL(blob);
+        const a = document.createElement('a');
+        a.href = url;
+        a.download = 'files.zip';
+        document.body.appendChild(a);
+        a.click();
+        a.remove();
+        window.URL.revokeObjectURL(url);
+    } catch (error) {
+        console.error('Download error:', error);
+        alert(`Error: ${error.message}`);
+    }
+}
+
+function sortTable(columnIndex, tableId) {
+    const table = document.getElementById(tableId);
+    const tbody = table.querySelector('tbody');
+    const rows = Array.from(tbody.rows);
+    const isAscending = table.dataset.sortAsc === 'true';
+
+    rows.sort((rowA, rowB) => {
+        const cellA = rowA.cells[columnIndex].innerText.trim().toLowerCase();
+        const cellB = rowB.cells[columnIndex].innerText.trim().toLowerCase();
+        return cellA.localeCompare(cellB) * (isAscending ? 1 : -1);
+    });
+
+    // Toggle sorting order for next click
+    table.dataset.sortAsc = !isAscending;
+
+    // Reinsert sorted rows
+    rows.forEach(row => tbody.appendChild(row));
+}
+
+function checkAllCheckboxes(tableId, checkAllCheckboxId) {
+    const table = document.getElementById(tableId);
+    const checkboxes = table.querySelectorAll('input[name="fileCheckbox"]');
+    const checkAllCheckbox = document.getElementById(checkAllCheckboxId);
+
+    checkboxes.forEach(checkbox => checkbox.checked = checkAllCheckbox.checked);
+}

app/static/index.js

@@ -0,0 +1,144 @@
+class LogScraperApp {
+    constructor() {
+        this.form = document.getElementById('scrapingForm');
+        this.stopButton = document.getElementById('stopButton');
+        this.logsElement = document.getElementById('logs');
+        this.prevPageButton = document.getElementById('prevPage');
+        this.nextPageButton = document.getElementById('nextPage');
+        this.pageInfo = document.getElementById('pageInfo');
+        this.startButton = document.getElementById('startButton');
+        
+        this.currentPage = 0;
+        this.linesPerPage = null;
+        this.autoRefreshInterval = null;
+
+        this.init();
+    }
+
+    async init() {
+        await this.fetchConfig();
+        await this.checkScrapingStatus();
+        this.addEventListeners();
+    }
+
+    async fetchConfig() {
+        try {
+            const response = await fetch('/config/lines_per_page');
+            const data = await response.json();
+            this.linesPerPage = data.lines_per_page;
+            this.fetchLogs(this.currentPage);
+        } catch (error) {
+            console.error('Error fetching config:', error);
+        }
+    }
+
+    async fetchLogs(page) {
+        try {
+            const response = await fetch(`/logfile?page=${page}&lines_per_page=${this.linesPerPage}`);
+            const data = await response.json();
+            
+            if (data.error) {
+                this.logsElement.textContent = data.error;
+            } else {
+                this.logsElement.innerHTML = data.log.map((line, index) => {
+                    const lineNumber = data.start_line - index;
+                    return `<span class="line-number">${lineNumber}</span> ${line}`;
+                }).join('');
+
+                this.updatePagination(data.total_lines);
+            }
+        } catch (error) {
+            console.error('Error fetching logs:', error);
+        }
+    }
+
+    updatePagination(totalLines) {
+        this.prevPageButton.disabled = this.currentPage === 0;
+        this.nextPageButton.disabled = (this.currentPage + 1) * this.linesPerPage >= totalLines;
+        this.pageInfo.textContent = `Page ${this.currentPage + 1} of ${Math.ceil(totalLines / this.linesPerPage)}`;
+    }
+
+    startAutoRefresh() {
+        this.autoRefreshInterval = setInterval(() => this.fetchLogs(this.currentPage), 5000);
+    }
+
+    stopAutoRefresh() {
+        clearInterval(this.autoRefreshInterval);
+    }
+
+    async checkScrapingStatus() {
+        try {
+            const response = await fetch('/scraping_status');
+            const data = await response.json();
+            if (data.scraping_active) {
+                this.startButton.disabled = true;
+                this.stopButton.disabled = false;
+                this.startAutoRefresh();
+            } else {
+                this.startButton.disabled = false;
+                this.stopButton.disabled = true;
+            }
+            this.fetchLogs(this.currentPage);
+        } catch (error) {
+            console.error('Error checking scraping status:', error);
+        }
+    }
+
+    async startScraping(event) {
+        event.preventDefault();
+        const formData = new FormData(this.form);
+        try {
+            const response = await fetch('/start_scraping', {
+                method: 'POST',
+                body: formData
+            });
+            const data = await response.json();
+
+            console.log(data);
+            if (data.status === "Scraping started") {
+                this.startButton.disabled = true;
+                this.stopButton.disabled = false;
+                this.startAutoRefresh();
+            }
+        } catch (error) {
+            console.error('Error starting scraping:', error);
+        }
+    }
+
+    async stopScraping() {
+        try {
+            const response = await fetch('/stop_scraping', { method: 'POST' });
+            const data = await response.json();
+
+            console.log(data);
+            if (data.status === "Scraping stopped") {
+                this.startButton.disabled = false;
+                this.stopButton.disabled = true;
+                this.stopAutoRefresh();
+            }
+        } catch (error) {
+            console.error('Error stopping scraping:', error);
+        }
+    }
+
+    addEventListeners() {
+        this.prevPageButton.addEventListener('click', () => {
+            if (this.currentPage > 0) {
+                this.currentPage--;
+                this.fetchLogs(this.currentPage);
+            }
+        });
+
+        this.nextPageButton.addEventListener('click', () => {
+            this.currentPage++;
+            this.fetchLogs(this.currentPage);
+        });
+
+        this.form.addEventListener('submit', (event) => this.startScraping(event));
+
+        this.stopButton.addEventListener('click', () => this.stopScraping());
+    }
+}
+
+// Initialize the application when DOM is fully loaded
+document.addEventListener('DOMContentLoaded', () => new LogScraperApp());

app/static/style.css

@@ -0,0 +1,217 @@
+/* LIGHT MODE (default) */
+:root {
+    --bs-body-bg: #f8f9fa; /* Light background */
+    --bs-body-color: #212529; /* Dark text */
+
+    --bs-primary: #007bff;
+    --bs-primary-bg-subtle: #cce5ff;
+    --bs-primary-border-subtle: #80bdff;
+    --bs-primary-text-emphasis: #004085;
+
+    --bs-secondary: #6c757d;
+    --bs-secondary-bg-subtle: #e2e3e5;
+    --bs-secondary-border-subtle: #c8cbcf;
+    --bs-secondary-text-emphasis: #383d41;
+
+    --bs-success: #198754;
+    --bs-success-bg-subtle: #d4edda;
+    --bs-success-border-subtle: #a3cfbb;
+    --bs-success-text-emphasis: #155724;
+
+    --bs-danger: #dc3545;
+    --bs-danger-bg-subtle: #f8d7da;
+    --bs-danger-border-subtle: #f1aeb5;
+    --bs-danger-text-emphasis: #721c24;
+
+    --bs-warning: #ffc107;
+    --bs-warning-bg-subtle: #fff3cd;
+    --bs-warning-border-subtle: #ffeeba;
+    --bs-warning-text-emphasis: #856404;
+
+    --bs-info: #17a2b8;
+    --bs-info-bg-subtle: #d1ecf1;
+    --bs-info-border-subtle: #bee5eb;
+    --bs-info-text-emphasis: #0c5460;
+
+    --bs-light: #f8f9fa;
+    --bs-light-bg-subtle: #ffffff;
+    --bs-light-border-subtle: #d6d8db;
+    --bs-light-text-emphasis: #6c757d;
+
+    --bs-dark: #343a40;
+    --bs-dark-bg-subtle: #212529;
+    --bs-dark-border-subtle: #1d2124;
+    --bs-dark-text-emphasis: #ffffff;
+
+    --bs-border-color: #dee2e6; /* Default border color */
+}
+
+/* DARK MODE */
+[data-bs-theme="dark"] {
+    --bs-body-bg: #121212;
+    --bs-body-color: #e9ecef;
+
+    --bs-primary: #1e90ff;
+    --bs-primary-bg-subtle: #1c2b36;
+    --bs-primary-border-subtle: #374b58;
+    --bs-primary-text-emphasis: #a0c4ff;
+
+    --bs-secondary: #adb5bd;
+    --bs-secondary-bg-subtle: #2d3238;
+    --bs-secondary-border-subtle: #3e444a;
+    --bs-secondary-text-emphasis: #ced4da;
+
+    --bs-success: #00c851;
+    --bs-success-bg-subtle: #1b3425;
+    --bs-success-border-subtle: #3b6147;
+    --bs-success-text-emphasis: #b9f6ca;
+
+    --bs-danger: #ff4444;
+    --bs-danger-bg-subtle: #381717;
+    --bs-danger-border-subtle: #633030;
+    --bs-danger-text-emphasis: #ffcccb;
+
+    --bs-warning: #ffbb33;
+    --bs-warning-bg-subtle: #3a2b19;
+    --bs-warning-border-subtle: #67512e;
+    --bs-warning-text-emphasis: #ffd700;
+
+    --bs-info: #33b5e5;
+    --bs-info-bg-subtle: #182e38;
+    --bs-info-border-subtle: #305564;
+    --bs-info-text-emphasis: #66d1ff;
+
+    --bs-light: #343a40;
+    --bs-light-bg-subtle: #2c3137;
+    --bs-light-border-subtle: #464b50;
+    --bs-light-text-emphasis: #e9ecef;
+
+    --bs-dark: #ffffff;
+    --bs-dark-bg-subtle: #f8f9fa;
+    --bs-dark-border-subtle: #e9ecef;
+    --bs-dark-text-emphasis: #121212;
+
+    --bs-border-color: #495057;
+}
+
+
+[data-bs-theme="dark"] .shadow {
+    box-shadow: var(--bs-box-shadow) !important;
+}
+
+[data-bs-theme="dark"] .shadow-sm {
+    box-shadow: var(--bs-box-shadow-sm) !important;
+}
+
+[data-bs-theme="dark"] .shadow-lg {
+    box-shadow: var(--bs-box-shadow-lg) !important;
+}
+
+:root {
+    --bs-primary: var(--primary);
+    --bs-secondary: var(--secondary);
+    --bs-body-bg: var(--background);
+    --bs-body-color: var(--text-color);
+}
+
+[data-bs-theme="dark"] {
+    --bs-primary: var(--primary);
+    --bs-secondary: var(--secondary);
+    --bs-body-bg: var(--background);
+    --bs-body-color: var(--text-color);
+}
+
+
+/* Dark Mode Toggle Button */
+/* Hide the default checkbox */
+#color-mode-toggle input[type=checkbox] {
+    height: 0;
+    width: 0;
+    visibility: hidden;
+}
+
+
+/* Style the switch */
+#color-mode-toggle label {
+    cursor: pointer;
+    width: 70px;
+    height: 30px;
+    background: grey;
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+    border-radius: 30px;
+    position: relative;
+    padding: 5px 15px;
+    box-shadow: inset 0 0 5px rgba(0, 0, 0, 0.3);
+}
+
+/* The moving toggle circle */
+#color-mode-toggle label:after {
+    content: '';
+    position: absolute;
+    top: 5px;
+    left: 5px;
+    width: 20px;
+    height: 20px;
+    background: white;
+    border-radius: 50%;
+    transition: 0.3s;
+}
+
+/* Sun and Moon Icons */
+.icon {
+    font-size: 15px;
+    position: absolute;
+    transition: 0.3s;
+}
+
+/* Position Sun on the left */
+.sun {
+    left: 10px;
+    /* color: var(--bs-dark) */
+    color: var(--sun-color);
+}
+
+/* Position Moon on the right */
+.moon {
+    right: 10px;
+    /* color: var(--bs-light); */
+    color: var(--sun-color);
+}
+
+/* Move the toggle circle when checked */
+#color-mode-toggle input:checked + label {
+    background: var(--bs-light);
+}
+
+#color-mode-toggle input:checked + label:after {
+    left: calc(100% - 25px);
+    background: var(--bs-dark);
+}
+
+/* Hide moon when in dark mode */
+#color-mode-toggle input:checked + label .sun {
+    opacity: 100;
+}
+
+#color-mode-toggle input:checked + label .moon {
+    opacity: 0;
+}
+
+/* Hide sun when in light mode */
+#color-mode-toggle input:not(:checked) + label .moon {
+    opacity: 100;
+}
+
+#color-mode-toggle  input:not(:checked) + label .sun {
+    opacity: 0;
+}
+
+.line-number {
+    display: inline-block;
+    width: 30px;
+    text-align: right;
+    margin-right: 10px;
+    color: #888;
+}

app/templates/analyze.html

@@ -0,0 +1,16 @@
+{% extends 'base.html' %}
+{% block content %}
+        <section class="container-fluid d-flex justify-content-center">
+            <div class="container-md my-5 mx-2 shadow-lg p-4 ">
+                <div class="container-sm">
+                    <div class="row">
+                        <div class="col">
+                            <h2>Analyze</h2>
+                        </div>
+                        <div class="col">
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </section>
+{% endblock content %}

app/templates/base.html

@@ -0,0 +1,29 @@
+<!-- app/templates/layouts/base.html -->
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    {% block head %}
+        <meta charset="UTF-8">
+        <title>TornActivityTracker{% block title %}{% endblock %}</title>
+        <meta charset="utf-8">
+        <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+        {% block styles %}
+        {{ bootstrap.load_css() }}
+        <link rel="stylesheet" href="{{url_for('static', filename='style.css')}}">
+        <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons/font/bootstrap-icons.css">
+        {% endblock %}
+    {% endblock %}
+</head>
+<body>
+    <header>
+        {% include 'includes/navigation.html' %}
+    </header>
+    <main>
+        {% block content %}
+        {% endblock %}
+    </main>
+    {% block scripts %}
+    {% include 'includes/scripts.html' %}
+    {% endblock %}
+</body>
+</html>

app/templates/download_results.html

@@ -0,0 +1,105 @@
+{% extends 'base.html' %}
+{% block content %}
+        <section class="container-fluid d-flex justify-content-center">
+            <div class="container-md my-5 mx-2 shadow-lg p-4 ">
+                <div class="container-sm">
+                    <div class="row">
+                        <div class="col">
+                            <h2>Data Files</h2>
+                        </div>
+                        <div class="col">
+                            <div class="btn-group btn-group-sm">
+                                <button class="btn btn-warning" onclick="deleteSelectedFiles()">Delete Selected Files</button>
+                                <button class="btn btn-success" onclick="downloadSelectedFiles()">Download Selected Files</button>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+                <table id="dataFilesTable" class="table table-striped table-bordered table-hover">
+                    <thead>
+                        <tr>
+                            <th width="2%"><input type="checkbox" id="checkAllData" onclick="checkAllCheckboxes('dataFilesTable', 'checkAllData')"></th>
+                            <th onclick="sortTable(1, 'dataFilesTable')">File Name</th>
+                            <th onclick="sortTable(2, 'dataFilesTable')">Last Modified</th>
+                            <th onclick="sortTable(3, 'dataFilesTable')">Created</th>
+                            <th onclick="sortTable(4, 'dataFilesTable')">Size</th>
+                            <th>Action</th>
+                            <th>Status</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {% for file in files.data %}
+                        <tr>
+                            <td><input type="checkbox" name="fileCheckbox" value="{{ url_for('download_log_file', filename=file.name_display) }}"{{ ' disabled' if file.active }}></td>
+                            <td><a href="{{ url_for('download_data_file', filename=file.name_display) }}" target="_blank">{{ file.name_display }}</a></td>
+                            <td>{{ file.last_modified | datetimeformat }}</td>
+                            <td>{{ file.created | datetimeformat }}</td>
+                            <td>{{ file.size }}</td>
+                            <td>
+                                <button class="btn btn-sm btn-warning" onclick="deleteFiles(['{{ file.name }}'])"{{ ' disabled' if file.active }}>Delete</button>
+                            </td>
+                            <td>
+                                <span id="status-{{ file.name_display }}" class="badge {{ 'bg-danger' if file.active else 'bg-success' }}">
+                                    {{ 'In Use' if file.active else 'Available' }}
+                                </span>
+                            </td>
+                        </tr>
+                        {% endfor %}
+                    </tbody>
+                </table>
+            </div>
+        </section>
+        <section class="container-fluid d-flex justify-content-center">
+            <div class="container-md my-5 mx-2 shadow-lg p-4 ">
+                <div class="container-sm">
+                    <div class="row">
+                        <div class="col">
+                            <h2>Log Files</h2>
+                        </div>
+                        <div class="col">
+                            <div class="btn-group btn-group-sm">
+                                <button class="btn btn-warning" onclick="deleteSelectedFiles()">Delete Selected Files</button>
+                                <button class="btn btn-success" onclick="downloadSelectedFiles()">Download Selected Files</button>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+                <table id="logFilesTable" class="table table-striped table-bordered table-hover">
+                    <thead>
+                        <tr>
+                            <th width="2%"><input type="checkbox" id="checkAllLog" onclick="checkAllCheckboxes('logFilesTable', 'checkAllLog')"></th>
+                            <th onclick="sortTable(1, 'logFilesTable')">File Name</th>
+                            <th onclick="sortTable(2, 'logFilesTable')">Last Modified</th>
+                            <th onclick="sortTable(3, 'logFilesTable')">Created</th>
+                            <th onclick="sortTable(4, 'logFilesTable')">Size</th>
+                            <th>Action</th>
+                            <th>Status</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {% for file in files.log %}
+                        <tr>
+                            <td><input type="checkbox" name="fileCheckbox" value="{{ url_for('download_log_file', filename=file.name_display) }}"{{ ' disabled' if file.active }}></td>
+                            <td><a href="{{ url_for('download_log_file', filename=file.name_display) }}" target="_blank">{{ file.name_display }}</a></td>
+                            <td>{{ file.last_modified | datetimeformat }}</td>
+                            <td>{{ file.created | datetimeformat }}</td>
+                            <td>{{ file.size }}</td>
+                            <td>
+                                <button class="btn btn-sm btn-warning" onclick="deleteFiles(['{{ file.name }}'])"{{ ' disabled' if file.active }}>Delete</button>
+                            </td>
+                            <td>
+                                <span id="status-{{ file.name_display }}" class="badge {{ 'bg-danger' if file.active else 'bg-success' }}">
+                                    {{ 'In Use' if file.active else 'Available' }}
+                                </span>
+                            </td>
+                        </tr>
+                        {% endfor %}
+                    </tbody>
+                </table>
+            </div>
+        </section>
+        {% block scripts %}
+        {{ bootstrap.load_js() }}
+        <script src="{{url_for('.static', filename='download_results.js')}}"></script>
+{% endblock %}
+{% endblock content %}

app/templates/includes/navigation.html

@@ -0,0 +1,17 @@
+<!-- app/templates/includes/navigation.html -->
+<nav class="navbar navbar-nav navbar-expand-md bg-primary">
+    <div class="container-fluid">
+        <a class="navbar-brand" href="/">Torn User Activity Scraper</a>
+        {% from 'bootstrap4/nav.html' import render_nav_item %}
+        {{ render_nav_item('analyze', 'Analyze') }}
+        {{ render_nav_item('download_results', 'Files') }}
+        {{ render_nav_item('log_viewer', 'Logs') }}
+        <div class="d-flex" id="color-mode-toggle">
+            <input type="checkbox" id="bd-theme" />
+            <label for="bd-theme">
+                <span class="icon sun"><i class="bi bi-brightness-high"></i></span>
+                <span class="icon moon"><i class="bi bi-moon-stars"></i></span>
+            </label>    
+        </div>
+    </div>
+</nav>

app/templates/includes/scripts.html

@@ -0,0 +1,2 @@
+{{ bootstrap.load_js() }}
+<script src="{{url_for('static', filename='color_mode.js')}}"></script>

app/templates/index.html

@@ -0,0 +1,46 @@
+{% extends 'base.html' %}
+{% block content %}
+        <section id="scrapingFormContainer" class="container-fluid d-flex justify-content-center">
+            <div class="container-md my-5 mx-2 shadow-lg p-4 ">
+                <h2>Scraper <span id="activity_indicator" class="badge text-bg-danger">Inactive</span></h2>
+                <form id="scrapingForm" method="POST" action="{{ url_for('start_scraping') }}">
+                    {{ form.hidden_tag() }}
+                    <div class="form-group">
+                        {{ form.faction_id.label(class="form-control-label") }}
+                        {{ form.faction_id(class="form-control") }}
+                    </div>
+                    <div class="form-group">
+                        {{ form.fetch_interval.label(class="form-control-label") }}
+                        {{ form.fetch_interval(class="form-control") }}
+                    </div>
+                    <div class="form-group">
+                        {{ form.run_interval.label(class="form-control-label") }}
+                        {{ form.run_interval(class="form-control") }}
+                    </div>
+                </form>
+                <div class="btn-group btn-group m-2" role="group">
+                    {{ form.submit(class="btn btn-success", type="submit", id="startButton", form="scrapingForm") }}
+                    <button class="btn btn-warning" type="button" id="stopButton">Stop</button>
+                </div>
+            </div>
+        </section>
+        <section id="resultsContainer" class="container-fluid d-flex justify-content-center">
+            <div class="container-md my-5 mx-2 shadow-lg p-4" style="height: 500px;">
+                <div class="row">
+                    <div class="col-8">
+                        <h2>Logs</h2>
+                        <pre id="logs" class="pre-scrollable" style="height: 350px; overflow:scroll; "><code></code></pre>
+                        <div class="btn-group btn-group-sm">
+                            <button class="btn btn-primary" id="prevPage">Previous</button>
+                            <button class="btn btn-primary" id="pageInfo" disabled>Page 1 of 1</button>
+                            <button class="btn btn-primary" id="nextPage">Next</button>
+                        </div>
+                    </div>
+                    <div class="col">
+                        <h2>Stats</h2>
+                    </div>
+                </div>
+            </div>
+        </section>
+        <script src="{{url_for('static', filename='index.js')}}"></script>
+{% endblock content %}

app/templates/log_viewer.html

@@ -0,0 +1,3 @@
+{% extends 'base.html' %}
+{% block content %}
+{% endblock content %}