mischbeck/TornActivityTracker
1
0
Fork 0
Code Issues 5 Pull Requests Packages Projects Releases Wiki Activity

adds status indicators to files view

Browse Source
This commit is contained in:
Michael Beck
2025-02-06 22:37:39 +01:00
parent 417a47d6a7
commit a5c15f211e
3 changed files with 174 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
app.py
View File

@@ -41,14 +41,19 @@ for key in config['BOOTSTRAP']:
continue
print(f"Loaded config: {key} = {app.config[key]}")
# Global state
scraping_active = False
scraping_thread = None
data_file_name = None
log_file_name = "log/" + datetime.now().strftime('%Y-%m-%d-%H-%M') + '.log'
# Initialize the logger
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG) # Adjust as needed
# Make any logger.info() call go to both the log file and the queue.
# 1) FILE HANDLER
logFile = "log/" + datetime.now().strftime('%Y-%m-%d-%H-%M') + '.log'
file_handler = logging.FileHandler(logFile, mode='w')
file_handler = logging.FileHandler(log_file_name, mode='w')
file_handler.setLevel(logging.DEBUG) # or INFO, WARNING, etc.
formatter = logging.Formatter('%(asctime)s - %(levelname)s: %(message)s',
datefmt='%m/%d/%Y %I:%M:%S %p')
@@ -62,9 +67,6 @@ queue_handler = QueueHandler(log_queue)
queue_handler.setLevel(logging.DEBUG)
logger.addHandler(queue_handler)
# Global state
scraping_active = False
scraping_thread = None
def create_zip(file_paths, zip_name):
@@ -107,8 +109,10 @@ def fetch_user_activity(user_id):
def scrape_data(faction_id, fetch_interval, run_interval):
global scraping_active
global data_file_name
end_time = datetime.now() + timedelta(days=run_interval)
filename = f"data/{faction_id}-{datetime.now().strftime('%Y-%m-%d-%H-%M')}.csv"
data_file_name = f"data/{faction_id}-{datetime.now().strftime('%Y-%m-%d-%H-%M')}.csv"
while datetime.now() < end_time and scraping_active:
logger.info(f"Fetching data at {datetime.now()}")
@@ -122,7 +126,7 @@ def scrape_data(faction_id, fetch_interval, run_interval):
'user_id': user_id,
'name': user_activity.get('name', ''),
'last_action': user_activity.get('last_action', {}).get('timestamp', 0),
'status': user_activity.get('status', {}).get('state', ''),
'stadata_file_nametus': user_activity.get('status', {}).get('state', ''),
'timestamp': datetime.now().timestamp()
})
logger.info(f"Fetched data for user {user_id} ({user_activity.get('name', '')})")
@@ -132,12 +136,12 @@ def scrape_data(faction_id, fetch_interval, run_interval):
df['last_action'] = pd.to_datetime(df['last_action'], unit='s')
df['timestamp'] = pd.to_datetime(df['timestamp'], unit='s')
if not os.path.isfile(filename):
df.to_csv(filename, index=False)
if not os.path.isfile(data_file_name):
df.to_csv(data_file_name, index=False)
else:
df.to_csv(filename, mode='a', header=False, index=False)
df.to_csv(data_file_name, mode='a', header=False, index=False)
logger.info(f"Data appended to {filename}")
logger.info(f"Data appended to {data_file_name}")
time.sleep(fetch_interval)
else:
@@ -204,6 +208,29 @@ def tail(filename, n):
bytes_to_read = offsets[i+1] - offset
yield f.read(bytes_to_read)
def is_data_file_in_use(filename):
if(data_file_name == None):
return False
if os.path.join(app.root_path, filename.lstrip('/')) == os.path.join(app.root_path, data_file_name.lstrip('/')) and scraping_active:
return True
return False
@app.route('/is_data_file_in_use/<path:filename>')
def is_data_file_in_use_json(filename):
return jsonify(is_data_file_in_use(filename))
def is_log_file_in_use(filename):
if(log_file_name == None):
return False
if os.path.join(app.root_path, filename.lstrip('/')) == os.path.join(app.root_path, log_file_name.lstrip('/')):
return True
return False
@app.route('/is_log_file_in_use/<path:filename>')
def is_log_file_in_use_json(filename):
print(filename)
return jsonify(is_log_file_in_use(filename))
@app.route('/')
def index():
form = ScrapingForm()
@@ -262,7 +289,7 @@ def logs():
def logfile():
page = int(request.args.get('page', 0)) # Page number
lines_per_page = int(request.args.get('lines_per_page', config['LOGGING']['VIEW_PAGE_LINES'])) # Lines per page
log_file_path = logFile # Path to the current log file
log_file_path = log_file_name # Path to the current log file
if not os.path.isfile(log_file_path):
logging.error("Log file not found")
@@ -282,6 +309,7 @@ def logfile():
"pages": (len(log_lines) + lines_per_page - 1) // lines_per_page,
"start_line": len(log_lines) - start # Starting line number for the current page
})
@app.route('/results')
def results():
# Assuming the scraping is done and data is saved somewhere
@@ -298,7 +326,7 @@ def results():
def download_results():
data_files = glob.glob("data/*.csv")
log_files = glob.glob("log/*.log")
def get_file_info(file_path):
return {
"name": file_path,
@@ -311,6 +339,18 @@ def download_results():
data_files_info = [get_file_info(file) for file in data_files]
log_files_info = [get_file_info(file) for file in log_files]
for data_file in data_files_info:
if is_data_file_in_use(data_file['name']):
data_file['active'] = True
else:
data_file['active'] = False
for log_file in log_files_info:
if is_log_file_in_use(log_file['name']):
log_file['active'] = True
else:
log_file['active'] = False
files = {"data": data_files_info, "log": log_files_info}
return render_template('download_results.html', files=files)
@@ -356,16 +396,36 @@ def delete_files():
errors = []
for file_path in file_paths:
file_path = os.path.join(app.root_path, file_path.lstrip('/'))
if not os.path.isfile(file_path):
errors.append({"file": file_path, "error": "File not found"})
full_file_path = os.path.join(app.root_path, file_path.lstrip('/'))
print(f"Attempting to delete: {file_path}") # Debugging line
print(f"Full path: {full_file_path}") # Debugging line
print(f"file_path: {file_path}") # Debugging line
# Check if the file is in either the logs or the data files folder
if not (full_file_path.startswith(os.path.join(app.root_path, 'log')) or
full_file_path.startswith(os.path.join(app.root_path, 'data'))):
errors.append({"file": file_path, "error": "File not in allowed directory"})
continue
# Check if it's the currently active log file
if is_log_file_in_use(file_path):
errors.append({"file": file_path, "error": "Cannot delete active log file."})
continue
# Check if it's an active data file
if is_data_file_in_use(file_path):
errors.append({"file": file_path, "error": "Cannot delete active data file."})
continue
if not os.path.isfile(full_file_path):
errors.append({"file": file_path, "error": "File not found"})
continue
try:
os.remove(file_path)
os.remove(full_file_path)
except Exception as e:
errors.append({"file": file_path, "error": str(e)})
if errors:
return jsonify({"errors": errors}), 207 # Multi-Status response
return jsonify({"success": True}), 200

132
static/download_results.js
View File

@@ -1,27 +1,34 @@
function deleteFiles(filePaths) {
fetch('/delete_files', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
'file_paths': filePaths
})
})
.then(response => response.json())
.then(data => {
async function deleteFiles(filePaths) {
try {
const response = await fetch('/delete_files', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ file_paths: filePaths })
});
const data = await response.json();
if (data.success) {
alert('Files deleted successfully');
location.reload();
} else {
alert('Error deleting files: ' + JSON.stringify(data.errors));
alert(`Error deleting files: ${JSON.stringify(data.errors)}`);
}
});
} catch (error) {
console.error('Error:', error);
alert('An error occurred while deleting files.');
}
}
function getSelectedFiles() {
return Array.from(document.querySelectorAll('input[name="fileCheckbox"]:checked'))
.map(checkbox => checkbox.value);
}
function deleteSelectedFiles() {
const selectedFiles = Array.from(document.querySelectorAll('input[name="fileCheckbox"]:checked'))
.map(checkbox => checkbox.value);
const selectedFiles = getSelectedFiles();
if (selectedFiles.length > 0) {
deleteFiles(selectedFiles);
} else {
@@ -29,39 +36,43 @@ function deleteSelectedFiles() {
}
}
async function downloadSelectedFiles() {
const selectedFiles = getSelectedFiles();
if (selectedFiles.length === 0) {
alert('No files selected');
return;
}
function downloadSelectedFiles() {
const selectedFiles = Array.from(document.querySelectorAll('input[name="fileCheckbox"]:checked'))
.map(checkbox => checkbox.value);
if (selectedFiles.length > 0) {
fetch('/download_files', {
try {
const response = await fetch('/download_files', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ 'file_paths': selectedFiles })
})
.then(response => {
if (!response.ok) {
return response.json().then(err => { throw new Error(err.error); });
}
return response.blob();
})
.then(blob => {
if (blob.type !== 'application/zip') {
throw new Error("Received invalid ZIP file.");
}
const url = window.URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = 'files.zip';
document.body.appendChild(a);
a.click();
a.remove();
})
.catch(error => alert('Error: ' + error.message));
} else {
alert('No files selected');
body: JSON.stringify({ file_paths: selectedFiles })
});
if (!response.ok) {
const errorData = await response.json();
throw new Error(errorData.error || 'Failed to download files.');
}
const blob = await response.blob();
if (blob.type !== 'application/zip') {
throw new Error('Received invalid ZIP file.');
}
const url = window.URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = 'files.zip';
document.body.appendChild(a);
a.click();
a.remove();
window.URL.revokeObjectURL(url);
} catch (error) {
console.error('Download error:', error);
alert(`Error: ${error.message}`);
}
}
@@ -69,36 +80,25 @@ function sortTable(columnIndex, tableId) {
const table = document.getElementById(tableId);
const tbody = table.querySelector('tbody');
const rows = Array.from(tbody.rows);
const isAscending = table.getAttribute('data-sort-asc') === 'true';
const isAscending = table.dataset.sortAsc === 'true';
rows.sort((rowA, rowB) => {
const cellA = rowA.cells[columnIndex].innerText.toLowerCase();
const cellB = rowB.cells[columnIndex].innerText.toLowerCase();
if (cellA < cellB) {
return isAscending ? -1 : 1;
}
if (cellA > cellB) {
return isAscending ? 1 : -1;
}
return 0;
const cellA = rowA.cells[columnIndex].innerText.trim().toLowerCase();
const cellB = rowB.cells[columnIndex].innerText.trim().toLowerCase();
return cellA.localeCompare(cellB) * (isAscending ? 1 : -1);
});
// Toggle the sorting order for the next click
table.setAttribute('data-sort-asc', !isAscending);
// Toggle sorting order for next click
table.dataset.sortAsc = !isAscending;
// Append sorted rows back to the tbody
// Reinsert sorted rows
rows.forEach(row => tbody.appendChild(row));
}
// Function to check or uncheck all checkboxes in a table by checking or unchecking the "CheckAll" checkbox
function checkAllCheckboxes(tableId, checkAllCheckboxId) {
const table = document.getElementById(tableId);
const checkboxes = table.querySelectorAll('input[type="checkbox"][name="fileCheckbox"]');
const checkboxes = table.querySelectorAll('input[name="fileCheckbox"]');
const checkAllCheckbox = document.getElementById(checkAllCheckboxId);
const isChecked = checkAllCheckbox.checked;
checkboxes.forEach(checkbox => {
checkbox.checked = isChecked;
});
}
checkboxes.forEach(checkbox => checkbox.checked = checkAllCheckbox.checked);
}

49
templates/download_results.html
View File

@@ -1,4 +1,3 @@
{% extends 'base.html' %}
{% block content %}
<section class="container-fluid d-flex justify-content-center">
@@ -25,18 +24,24 @@
<th onclick="sortTable(3, 'dataFilesTable')">Created</th>
<th onclick="sortTable(4, 'dataFilesTable')">Size</th>
<th>Action</th>
<th>Status</th>
</tr>
</thead>
</thead>
<tbody>
{% for file in files.data %}
{% for file in files.data %}
<tr>
<td class="d-sm-table-cell"><input type="checkbox" name="fileCheckbox" value="{{ url_for('download_data_file', filename=file.name_display) }}"></td>
<td><input type="checkbox" name="fileCheckbox" value="{{ url_for('download_log_file', filename=file.name_display) }}"{{ ' disabled' if file.active }}></td>
<td><a href="{{ url_for('download_data_file', filename=file.name_display) }}" target="_blank">{{ file.name_display }}</a></td>
<td>{{ file.last_modified | datetimeformat }}</td>
<td>{{ file.created | datetimeformat }}</td>
<td>{{ file.size }}</td>
<td>
<button class="btn btn-warning" onclick="deleteFiles(['{{ file.name }}'])">Delete</button>
<button class="btn btn-sm btn-warning" onclick="deleteFiles(['{{ file.name }}'])"{{ ' disabled' if file.active }}>Delete</button>
</td>
<td>
<span id="status-{{ file.name_display }}" class="badge {{ 'bg-danger' if file.active else 'bg-success' }}">
{{ 'In Use' if file.active else 'Available' }}
</span>
</td>
</tr>
{% endfor %}
@@ -68,27 +73,33 @@
<th onclick="sortTable(3, 'logFilesTable')">Created</th>
<th onclick="sortTable(4, 'logFilesTable')">Size</th>
<th>Action</th>
<th>Status</th>
</tr>
</thead>
<tbody>
{% for file in files.log %}
<tr>
<td><input type="checkbox" name="fileCheckbox" value="{{ url_for('download_log_file', filename=file.name_display) }}"></td>
<td><a href="{{ url_for('download_log_file', filename=file.name_display) }}" target="_blank">{{ file.name }}</a></td>
<td>{{ file.last_modified | datetimeformat }}</td>
<td>{{ file.created | datetimeformat }}</td>
<td>{{ file.size }}</td>
<td>
<button class="btn btn-warning" onclick="deleteFiles(['{{ file.name }}'])">Delete</button>
</td>
</tr>
<tr>
<td><input type="checkbox" name="fileCheckbox" value="{{ url_for('download_log_file', filename=file.name_display) }}"{{ ' disabled' if file.active }}></td>
<td><a href="{{ url_for('download_log_file', filename=file.name_display) }}" target="_blank">{{ file.name_display }}</a></td>
<td>{{ file.last_modified | datetimeformat }}</td>
<td>{{ file.created | datetimeformat }}</td>
<td>{{ file.size }}</td>
<td>
<button class="btn btn-sm btn-warning" onclick="deleteFiles(['{{ file.name }}'])"{{ ' disabled' if file.active }}>Delete</button>
</td>
<td>
<span id="status-{{ file.name_display }}" class="badge {{ 'bg-danger' if file.active else 'bg-success' }}">
{{ 'In Use' if file.active else 'Available' }}
</span>
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</section>
{% block scripts %}
{{ bootstrap.load_js() }}
{% block scripts %}
{{ bootstrap.load_js() }}
<script src="{{url_for('.static', filename='download_results.js')}}"></script>
{% endblock %}
{% endblock content %}
{% endblock %}
{% endblock content %}