Adds multiple file download/delete option. Fixes css-filename.

app.py

@@ -203,33 +203,44 @@ def results():
 def download_results():
     data_files = glob.glob("data/*.csv")
     log_files = glob.glob("log/*.log")
-    
+
     def get_file_info(file_path):
         return {
             "name": file_path,
+            "name_display": os.path.basename(file_path),
             "last_modified": os.path.getmtime(file_path),
             "created": os.path.getctime(file_path),
             "size": get_size(file_path)
         }
-    
+       
     data_files_info = [get_file_info(file) for file in data_files]
     log_files_info = [get_file_info(file) for file in log_files]
     
     files = {"data": data_files_info, "log": log_files_info}
+    
     return render_template('download_results.html', files=files)
 
-@app.route('/delete_file', methods=['POST'])
+@app.route('/delete_files', methods=['POST'])
-def delete_file():
+def delete_files():
-    file_path = request.form.get('file_path')
+    file_paths = request.form.getlist('file_paths')
     
-    if not file_path or not os.path.isfile(file_path):
+    if not file_paths:
-        return jsonify({"error": "File not found"}), 404
+        return jsonify({"error": "No files specified"}), 400
     
-    try:
+    errors = []
-        os.remove(file_path)
+    for file_path in file_paths:
-        return jsonify({"success": True}), 200
+        if not os.path.isfile(file_path):
-    except Exception as e:
+            errors.append({"file": file_path, "error": "File not found"})
-        return jsonify({"error": str(e)}), 500
+            continue
+        
+        try:
+            os.remove(file_path)
+        except Exception as e:
+            errors.append({"file": file_path, "error": str(e)})
+    
+    if errors:
+        return jsonify({"errors": errors}), 207  # Multi-Status response
+    return jsonify({"success": True}), 200
 
 @app.template_filter('datetimeformat')
 def datetimeformat(value):

static/index.js

@@ -0,0 +1,108 @@
+document.addEventListener('DOMContentLoaded', () => {
+    const form = document.getElementById('scrapingForm');
+    const stopButton = document.getElementById('stopButton');
+    const logsElement = document.getElementById('logs');
+    const prevPageButton = document.getElementById('prevPage');
+    const nextPageButton = document.getElementById('nextPage');
+    let currentPage = 0;
+    const linesPerPage = 50;
+    let autoRefreshInterval;
+
+    if (form) {
+        console.log('Form:', form);
+        console.log('Submit button:', form.querySelector('button[type="submit"]'));
+    }
+
+    const fetchLogs = (page) => {
+        fetch(`/logfile?lines=${linesPerPage * (page + 1)}`)
+            .then(response => response.json())
+            .then(data => {
+                if (data.error) {
+                    logsElement.textContent = data.error;
+                } else {
+                    // Reverse the order of log lines
+                    const reversedLogs = data.log.reverse();
+                    logsElement.textContent = reversedLogs.join('');
+                }
+            });
+    };
+
+    const startAutoRefresh = () => {
+        autoRefreshInterval = setInterval(() => {
+            fetchLogs(currentPage);
+        }, 5000); // Refresh every 5 seconds
+    };
+
+    const stopAutoRefresh = () => {
+        clearInterval(autoRefreshInterval);
+    };
+
+    // Check scraping status on page load
+    fetch('/scraping_status')
+        .then(response => response.json())
+        .then(data => {
+            if (data.scraping_active) {
+                startButton.disabled = true;
+                stopButton.disabled = false;
+                startAutoRefresh();  // Start auto-refresh if scraping is active
+            } else {
+                startButton.disabled = false;
+                stopButton.disabled = true;
+            }
+            fetchLogs(currentPage);
+        });
+
+    prevPageButton.addEventListener('click', () => {
+        if (currentPage > 0) {
+            currentPage--;
+            fetchLogs(currentPage);
+        }
+    });
+
+    nextPageButton.addEventListener('click', () => {
+        currentPage++;
+        fetchLogs(currentPage);
+    });
+
+    form.addEventListener('submit', function(e) {
+        e.preventDefault();
+        const formData = new FormData(this);
+
+        fetch('/start_scraping', {
+            method: 'POST',
+            body: formData
+        }).then(response => response.json())
+          .then(data => {
+              console.log(data);
+              const submitButton = form.querySelector('button[type="submit"]');
+              if (data.status === "Scraping started") {
+                  if (submitButton) {
+                      submitButton.disabled = true;
+                  }
+                  stopButton.disabled = false;
+                  startAutoRefresh();  // Start auto-refresh when scraping starts
+              } else {
+                  // Handle errors or other statuses
+              }
+          });
+    });
+
+    stopButton.addEventListener('click', function() {
+        fetch('/stop_scraping', {
+            method: 'POST'
+        }).then(response => response.json())
+          .then(data => {
+              console.log(data);
+              const submitButton = form.querySelector('button[type="submit"]');
+              if (data.status === "Scraping stopped") {
+                  if (submitButton) {
+                      submitButton.disabled = false;
+                  }
+                  stopButton.disabled = true;
+                  stopAutoRefresh();  // Stop auto-refresh when scraping stops
+              } else {
+                  // Handle errors or other statuses
+              }
+          });
+    });
+});

templates/download_results.html

@@ -7,7 +7,68 @@
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
     {{ bootstrap.load_css() }}
-    <link rel="stylesheet" href="{{url_for('.static', filename='styles.css')}}">
+    <link rel="stylesheet" href="{{url_for('.static', filename='style.css')}}">
+    <script>
+        function deleteFiles(filePaths) {
+            fetch('/delete_files', {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/x-www-form-urlencoded',
+                },
+                body: new URLSearchParams({
+                    'file_paths': filePaths
+                })
+            })
+            .then(response => response.json())
+            .then(data => {
+                if (data.success) {
+                    alert('Files deleted successfully');
+                    location.reload();
+                } else {
+                    alert('Error deleting files: ' + JSON.stringify(data.errors));
+                }
+            });
+        }
+
+        function deleteSelectedFiles() {
+            const selectedFiles = Array.from(document.querySelectorAll('input[name="fileCheckbox"]:checked'))
+                .map(checkbox => checkbox.value);
+            if (selectedFiles.length > 0) {
+                deleteFiles(selectedFiles);
+            } else {
+                alert('No files selected');
+            }
+        }
+
+        function downloadSelectedFiles() {
+            const selectedFiles = Array.from(document.querySelectorAll('input[name="fileCheckbox"]:checked'))
+                .map(checkbox => checkbox.value);
+            if (selectedFiles.length > 0) {
+                selectedFiles.forEach(file => {
+                    const link = document.createElement('a');
+                    link.href = file;
+                    link.download = file.split('/').pop();
+                    document.body.appendChild(link);
+                    link.click();
+                    document.body.removeChild(link);
+                });
+            } else {
+                alert('No files selected');
+            }
+        }
+        
+        // Function to check or uncheck all checkboxes in a table by checking or unchecking the "CheckAll" checkbox
+        function checkAllCheckboxes(tableId, checkAllCheckboxId) {
+            const table = document.getElementById(tableId);
+            const checkboxes = table.querySelectorAll('input[type="checkbox"][name="fileCheckbox"]');
+            const checkAllCheckbox = document.getElementById(checkAllCheckboxId);
+            const isChecked = checkAllCheckbox.checked;
+        
+            checkboxes.forEach(checkbox => {
+                checkbox.checked = isChecked;
+            });
+        }
+    </script>
 </head>
 <body>
     <header>
@@ -25,50 +86,74 @@
     <main>
         <section id="scrapingFormContainer" class="container-fluid d-flex justify-content-center">
             <div class="container-md my-5 mx-2 shadow-lg p-4 ">
-                <h2>Data</h2>
+                <div class="container-sm">
+                    <div class="row">
+                        <div class="col">
+                            <h2>Data Files</h2>
+                        </div>
+                        <div class="col btn-group" >
+                            <button class="btn-danger" onclick="deleteSelectedFiles()">Delete Selected Files</button>
+                            <button class="btn-success" onclick="downloadSelectedFiles()">Download Selected Files</button>
+                        </div>
+                    </div>
+                </div>
                 <table id="dataFilesTable" class="table table-striped table-bordered table-hover">
                     <thead>
                         <tr>
-                            <th onclick="sortTable(0, 'dataFilesTable')">File Name</th>
+                            <th><input type="checkbox" id="checkAllData" onclick="checkAllCheckboxes('dataFilesTable', 'checkAllData')">Select</th>
-                            <th onclick="sortTable(1, 'dataFilesTable')">Last Modified</th>
+                            <th onclick="sortTable(1, 'dataFilesTable')">File Name</th>
-                            <th onclick="sortTable(2, 'dataFilesTable')">Created</th>
+                            <th onclick="sortTable(2, 'dataFilesTable')">Last Modified</th>
-                            <th onclick="sortTable(3, 'dataFilesTable')">Size</th>
+                            <th onclick="sortTable(3, 'dataFilesTable')">Created</th>
+                            <th onclick="sortTable(4, 'dataFilesTable')">Size</th>
                             <th>Action</th>
                         </tr>
                     </thead>    
-                    {% for file in files.data %}
+                    <tbody>
-                    <tr>
+                        {% for file in files.data %}                
-                        <td><a href="{{ url_for('download_data_file', filename=file.name.split('/')[-1]) }}" target="_blank">{{ file.name }}</a></td>
+                        <tr>
-                        <td>{{ file.last_modified | datetimeformat }}</td>
+                            <td class="d-sm-table-cell"><input type="checkbox" name="fileCheckbox" value="{{ url_for('download_data_file', filename=file.name_display) }}"></td>
-                        <td>{{ file.created | datetimeformat }}</td>
+                            <td><a href="{{ url_for('download_data_file', filename=file.name_display) }}" target="_blank">{{ file.name_display }}</a></td>
-                        <td>{{ file.size }}</td>
+                            <td>{{ file.last_modified | datetimeformat }}</td>
-                        <td>
+                            <td>{{ file.created | datetimeformat }}</td>
-                            <button onclick="deleteFile('{{ file.name }}')">Delete</button>
+                            <td>{{ file.size }}</td>
-                        </td>
+                            <td>
-                    </tr>
+                                <button onclick="deleteFiles(['{{ file.name }}'])">Delete</button>
-                    {% endfor %}
+                            </td>
+                        </tr>
+                        {% endfor %}
                     </tbody>
                 </table>
-                <h2>Log</h2>
+                <div class="container-sm">
-                <table id="logFilesTable" class="table table-striped table-bordered table-hover">
+                    <div class="row">
+                        <div class="col">
+                            <h2>Log Files</h2>
+                        </div>
+                        <div class="col btn-group" >
+                            <button class="btn-danger" onclick="deleteSelectedFiles()">Delete Selected Files</button>
+                            <button class="btn-success" onclick="downloadSelectedFiles()">Download Selected Files</button>
+                        </div>
+                    </div>
+                </div>                <table id="logFilesTable" class="table table-striped table-bordered table-hover">
                     <thead>
                         <tr>
-                            <th onclick="sortTable(0, 'logFilesTable')">File Name</th>
+                            <th><input type="checkbox" id="checkAllLog" onclick="checkAllCheckboxes('logFilesTable', 'checkAllLog')"> Select</th>
-                            <th onclick="sortTable(1, 'logFilesTable')">Last Modified</th>
+                            <th onclick="sortTable(1, 'logFilesTable')">File Name</th>
-                            <th onclick="sortTable(2, 'logFilesTable')">Created</th>
+                            <th onclick="sortTable(2, 'logFilesTable')">Last Modified</th>
-                            <th onclick="sortTable(3, 'logFilesTable')">Size</th>
+                            <th onclick="sortTable(3, 'logFilesTable')">Created</th>
+                            <th onclick="sortTable(4, 'logFilesTable')">Size</th>
                             <th>Action</th>
                         </tr>
                     </thead>
                     <tbody>
                         {% for file in files.log %}
                             <tr>
-                                <td><a href="{{ url_for('download_log_file', filename=file.name.split('/')[-1]) }}" target="_blank">{{ file.name }}</a></td>
+                                <td><input type="checkbox" name="fileCheckbox" value="{{ url_for('download_log_file', filename=file.name_display) }}"></td>
+                                <td><a href="{{ url_for('download_log_file', filename=file.name_display) }}" target="_blank">{{ file.name }}</a></td>
                                 <td>{{ file.last_modified | datetimeformat }}</td>
                                 <td>{{ file.created | datetimeformat }}</td>
                                 <td>{{ file.size }}</td>
                                 <td>
-                                    <button onclick="deleteFile('{{ file.name }}')">Delete</button>
+                                    <button onclick="deleteFiles(['{{ file.name }}'])">Delete</button>
                                 </td>
                             </tr>
                         {% endfor %}
@@ -77,9 +162,5 @@
             </div>
         </section>
     </main>
-    {% block scripts %}
-        {{ bootstrap.load_js() }}
-        <script src="{{url_for('.static', filename='app.js')}}"></script>
-    {% endblock %}
 </body>
 </html>

templates/index.html

@@ -7,7 +7,7 @@
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
     {{ bootstrap.load_css() }}
-    <link rel="stylesheet" href="{{url_for('.static', filename='styles.css')}}">
+    <link rel="stylesheet" href="{{url_for('.static', filename='style.css')}}">
 </head>
 <body>
     <header>