From a5c15f211ecab72cfdcb6ba7eb2eef07d9ff33eb Mon Sep 17 00:00:00 2001 From: Michael Beck Date: Thu, 6 Feb 2025 22:37:39 +0100 Subject: [PATCH] adds status indicators to files view --- app.py | 96 ++++++++++++++++++----- static/download_results.js | 132 ++++++++++++++++---------------- templates/download_results.html | 49 +++++++----- 3 files changed, 174 insertions(+), 103 deletions(-) diff --git a/app.py b/app.py index 1b9d6fb..922d91e 100644 --- a/app.py +++ b/app.py @@ -41,14 +41,19 @@ for key in config['BOOTSTRAP']: continue print(f"Loaded config: {key} = {app.config[key]}") +# Global state +scraping_active = False +scraping_thread = None +data_file_name = None +log_file_name = "log/" + datetime.now().strftime('%Y-%m-%d-%H-%M') + '.log' + # Initialize the logger logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) # Adjust as needed # Make any logger.info() call go to both the log file and the queue. # 1) FILE HANDLER -logFile = "log/" + datetime.now().strftime('%Y-%m-%d-%H-%M') + '.log' -file_handler = logging.FileHandler(logFile, mode='w') +file_handler = logging.FileHandler(log_file_name, mode='w') file_handler.setLevel(logging.DEBUG) # or INFO, WARNING, etc. formatter = logging.Formatter('%(asctime)s - %(levelname)s: %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') @@ -62,9 +67,6 @@ queue_handler = QueueHandler(log_queue) queue_handler.setLevel(logging.DEBUG) logger.addHandler(queue_handler) -# Global state -scraping_active = False -scraping_thread = None def create_zip(file_paths, zip_name): @@ -107,8 +109,10 @@ def fetch_user_activity(user_id): def scrape_data(faction_id, fetch_interval, run_interval): global scraping_active + global data_file_name + end_time = datetime.now() + timedelta(days=run_interval) - filename = f"data/{faction_id}-{datetime.now().strftime('%Y-%m-%d-%H-%M')}.csv" + data_file_name = f"data/{faction_id}-{datetime.now().strftime('%Y-%m-%d-%H-%M')}.csv" while datetime.now() < end_time and scraping_active: logger.info(f"Fetching data at {datetime.now()}") @@ -122,7 +126,7 @@ def scrape_data(faction_id, fetch_interval, run_interval): 'user_id': user_id, 'name': user_activity.get('name', ''), 'last_action': user_activity.get('last_action', {}).get('timestamp', 0), - 'status': user_activity.get('status', {}).get('state', ''), + 'stadata_file_nametus': user_activity.get('status', {}).get('state', ''), 'timestamp': datetime.now().timestamp() }) logger.info(f"Fetched data for user {user_id} ({user_activity.get('name', '')})") @@ -132,12 +136,12 @@ def scrape_data(faction_id, fetch_interval, run_interval): df['last_action'] = pd.to_datetime(df['last_action'], unit='s') df['timestamp'] = pd.to_datetime(df['timestamp'], unit='s') - if not os.path.isfile(filename): - df.to_csv(filename, index=False) + if not os.path.isfile(data_file_name): + df.to_csv(data_file_name, index=False) else: - df.to_csv(filename, mode='a', header=False, index=False) + df.to_csv(data_file_name, mode='a', header=False, index=False) - logger.info(f"Data appended to {filename}") + logger.info(f"Data appended to {data_file_name}") time.sleep(fetch_interval) else: @@ -204,6 +208,29 @@ def tail(filename, n): bytes_to_read = offsets[i+1] - offset yield f.read(bytes_to_read) +def is_data_file_in_use(filename): + if(data_file_name == None): + return False + if os.path.join(app.root_path, filename.lstrip('/')) == os.path.join(app.root_path, data_file_name.lstrip('/')) and scraping_active: + return True + return False + +@app.route('/is_data_file_in_use/') +def is_data_file_in_use_json(filename): + return jsonify(is_data_file_in_use(filename)) + +def is_log_file_in_use(filename): + if(log_file_name == None): + return False + if os.path.join(app.root_path, filename.lstrip('/')) == os.path.join(app.root_path, log_file_name.lstrip('/')): + return True + return False + +@app.route('/is_log_file_in_use/') +def is_log_file_in_use_json(filename): + print(filename) + return jsonify(is_log_file_in_use(filename)) + @app.route('/') def index(): form = ScrapingForm() @@ -262,7 +289,7 @@ def logs(): def logfile(): page = int(request.args.get('page', 0)) # Page number lines_per_page = int(request.args.get('lines_per_page', config['LOGGING']['VIEW_PAGE_LINES'])) # Lines per page - log_file_path = logFile # Path to the current log file + log_file_path = log_file_name # Path to the current log file if not os.path.isfile(log_file_path): logging.error("Log file not found") @@ -282,6 +309,7 @@ def logfile(): "pages": (len(log_lines) + lines_per_page - 1) // lines_per_page, "start_line": len(log_lines) - start # Starting line number for the current page }) + @app.route('/results') def results(): # Assuming the scraping is done and data is saved somewhere @@ -298,7 +326,7 @@ def results(): def download_results(): data_files = glob.glob("data/*.csv") log_files = glob.glob("log/*.log") - + def get_file_info(file_path): return { "name": file_path, @@ -311,6 +339,18 @@ def download_results(): data_files_info = [get_file_info(file) for file in data_files] log_files_info = [get_file_info(file) for file in log_files] + for data_file in data_files_info: + if is_data_file_in_use(data_file['name']): + data_file['active'] = True + else: + data_file['active'] = False + + for log_file in log_files_info: + if is_log_file_in_use(log_file['name']): + log_file['active'] = True + else: + log_file['active'] = False + files = {"data": data_files_info, "log": log_files_info} return render_template('download_results.html', files=files) @@ -356,16 +396,36 @@ def delete_files(): errors = [] for file_path in file_paths: - file_path = os.path.join(app.root_path, file_path.lstrip('/')) - if not os.path.isfile(file_path): - errors.append({"file": file_path, "error": "File not found"}) + full_file_path = os.path.join(app.root_path, file_path.lstrip('/')) + print(f"Attempting to delete: {file_path}") # Debugging line + print(f"Full path: {full_file_path}") # Debugging line + print(f"file_path: {file_path}") # Debugging line + + # Check if the file is in either the logs or the data files folder + if not (full_file_path.startswith(os.path.join(app.root_path, 'log')) or + full_file_path.startswith(os.path.join(app.root_path, 'data'))): + errors.append({"file": file_path, "error": "File not in allowed directory"}) + continue + + # Check if it's the currently active log file + if is_log_file_in_use(file_path): + errors.append({"file": file_path, "error": "Cannot delete active log file."}) continue + # Check if it's an active data file + if is_data_file_in_use(file_path): + errors.append({"file": file_path, "error": "Cannot delete active data file."}) + continue + + if not os.path.isfile(full_file_path): + errors.append({"file": file_path, "error": "File not found"}) + continue + try: - os.remove(file_path) + os.remove(full_file_path) except Exception as e: errors.append({"file": file_path, "error": str(e)}) - + if errors: return jsonify({"errors": errors}), 207 # Multi-Status response return jsonify({"success": True}), 200 diff --git a/static/download_results.js b/static/download_results.js index 691df21..c58beb9 100644 --- a/static/download_results.js +++ b/static/download_results.js @@ -1,27 +1,34 @@ -function deleteFiles(filePaths) { - fetch('/delete_files', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - 'file_paths': filePaths - }) - }) - .then(response => response.json()) - .then(data => { +async function deleteFiles(filePaths) { + try { + const response = await fetch('/delete_files', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ file_paths: filePaths }) + }); + + const data = await response.json(); + if (data.success) { alert('Files deleted successfully'); location.reload(); } else { - alert('Error deleting files: ' + JSON.stringify(data.errors)); + alert(`Error deleting files: ${JSON.stringify(data.errors)}`); } - }); + } catch (error) { + console.error('Error:', error); + alert('An error occurred while deleting files.'); + } +} + +function getSelectedFiles() { + return Array.from(document.querySelectorAll('input[name="fileCheckbox"]:checked')) + .map(checkbox => checkbox.value); } function deleteSelectedFiles() { - const selectedFiles = Array.from(document.querySelectorAll('input[name="fileCheckbox"]:checked')) - .map(checkbox => checkbox.value); + const selectedFiles = getSelectedFiles(); if (selectedFiles.length > 0) { deleteFiles(selectedFiles); } else { @@ -29,39 +36,43 @@ function deleteSelectedFiles() { } } +async function downloadSelectedFiles() { + const selectedFiles = getSelectedFiles(); + if (selectedFiles.length === 0) { + alert('No files selected'); + return; + } -function downloadSelectedFiles() { - const selectedFiles = Array.from(document.querySelectorAll('input[name="fileCheckbox"]:checked')) - .map(checkbox => checkbox.value); - if (selectedFiles.length > 0) { - fetch('/download_files', { + try { + const response = await fetch('/download_files', { method: 'POST', headers: { 'Content-Type': 'application/json', }, - body: JSON.stringify({ 'file_paths': selectedFiles }) - }) - .then(response => { - if (!response.ok) { - return response.json().then(err => { throw new Error(err.error); }); - } - return response.blob(); - }) - .then(blob => { - if (blob.type !== 'application/zip') { - throw new Error("Received invalid ZIP file."); - } - const url = window.URL.createObjectURL(blob); - const a = document.createElement('a'); - a.href = url; - a.download = 'files.zip'; - document.body.appendChild(a); - a.click(); - a.remove(); - }) - .catch(error => alert('Error: ' + error.message)); - } else { - alert('No files selected'); + body: JSON.stringify({ file_paths: selectedFiles }) + }); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(errorData.error || 'Failed to download files.'); + } + + const blob = await response.blob(); + if (blob.type !== 'application/zip') { + throw new Error('Received invalid ZIP file.'); + } + + const url = window.URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = 'files.zip'; + document.body.appendChild(a); + a.click(); + a.remove(); + window.URL.revokeObjectURL(url); + } catch (error) { + console.error('Download error:', error); + alert(`Error: ${error.message}`); } } @@ -69,36 +80,25 @@ function sortTable(columnIndex, tableId) { const table = document.getElementById(tableId); const tbody = table.querySelector('tbody'); const rows = Array.from(tbody.rows); - const isAscending = table.getAttribute('data-sort-asc') === 'true'; - + const isAscending = table.dataset.sortAsc === 'true'; + rows.sort((rowA, rowB) => { - const cellA = rowA.cells[columnIndex].innerText.toLowerCase(); - const cellB = rowB.cells[columnIndex].innerText.toLowerCase(); - - if (cellA < cellB) { - return isAscending ? -1 : 1; - } - if (cellA > cellB) { - return isAscending ? 1 : -1; - } - return 0; + const cellA = rowA.cells[columnIndex].innerText.trim().toLowerCase(); + const cellB = rowB.cells[columnIndex].innerText.trim().toLowerCase(); + return cellA.localeCompare(cellB) * (isAscending ? 1 : -1); }); - // Toggle the sorting order for the next click - table.setAttribute('data-sort-asc', !isAscending); + // Toggle sorting order for next click + table.dataset.sortAsc = !isAscending; - // Append sorted rows back to the tbody + // Reinsert sorted rows rows.forEach(row => tbody.appendChild(row)); } -// Function to check or uncheck all checkboxes in a table by checking or unchecking the "CheckAll" checkbox function checkAllCheckboxes(tableId, checkAllCheckboxId) { const table = document.getElementById(tableId); - const checkboxes = table.querySelectorAll('input[type="checkbox"][name="fileCheckbox"]'); + const checkboxes = table.querySelectorAll('input[name="fileCheckbox"]'); const checkAllCheckbox = document.getElementById(checkAllCheckboxId); - const isChecked = checkAllCheckbox.checked; - checkboxes.forEach(checkbox => { - checkbox.checked = isChecked; - }); -} \ No newline at end of file + checkboxes.forEach(checkbox => checkbox.checked = checkAllCheckbox.checked); +} diff --git a/templates/download_results.html b/templates/download_results.html index e156439..d705991 100644 --- a/templates/download_results.html +++ b/templates/download_results.html @@ -1,4 +1,3 @@ - {% extends 'base.html' %} {% block content %}
@@ -25,18 +24,24 @@ Created Size Action + Status - + - {% for file in files.data %} + {% for file in files.data %} - + {{ file.name_display }} {{ file.last_modified | datetimeformat }} {{ file.created | datetimeformat }} {{ file.size }} - + + + + + {{ 'In Use' if file.active else 'Available' }} + {% endfor %} @@ -68,27 +73,33 @@ Created Size Action + Status {% for file in files.log %} - - - {{ file.name }} - {{ file.last_modified | datetimeformat }} - {{ file.created | datetimeformat }} - {{ file.size }} - - - - + + + {{ file.name_display }} + {{ file.last_modified | datetimeformat }} + {{ file.created | datetimeformat }} + {{ file.size }} + + + + + + {{ 'In Use' if file.active else 'Available' }} + + + {% endfor %}
- {% block scripts %} - {{ bootstrap.load_js() }} + {% block scripts %} + {{ bootstrap.load_js() }} - {% endblock %} -{% endblock content %} +{% endblock %} +{% endblock content %} \ No newline at end of file