refactors to use redis & celery

app/api.py

@@ -1,4 +1,3 @@
-# filepath: /home/michaelb/Dokumente/TornActivityTracker/app/api.py
 from flask import jsonify, request, Response, send_from_directory, current_app
 import threading
 import os
@@ -10,6 +9,7 @@ from app.models import Scraper
 from app.util import create_zip, delete_old_zips, tail
 from app.config import load_config
 from app.forms import ScrapingForm
+from app.tasks import start_scraping_task, stop_scraping_task, get_redis
 
 scraping_thread = None
 scraper = None
@@ -18,42 +18,40 @@ scrape_lock = threading.Lock()
 def register_api(app):
     @app.route('/start_scraping', methods=['POST'])
     def start_scraping():
-        global scraping_thread, scraper
-        with scrape_lock:
-            scraper = current_app.config.get('SCRAPER')
-            if scraper is not None and scraper.scraping_active:
-                current_app.logger.warning("Can't start scraping process: scraping already in progress")
+        form = ScrapingForm()
+        if form.validate_on_submit():
+            redis_client = get_redis()
+            faction_id = form.faction_id.data
+            
+            # Check if scraping is already active
+            if redis_client.hget(f"scraper:{faction_id}", "scraping_active") == "1":
                 return jsonify({"status": "Scraping already in progress"})
-
-            form = ScrapingForm()
-            if form.validate_on_submit():
-                faction_id = form.faction_id.data
-                fetch_interval = form.fetch_interval.data
-                run_interval = form.run_interval.data
-
-                scraper = Scraper(faction_id, fetch_interval, run_interval, app)
-                scraper.scraping_active = True
-
-                scraping_thread = threading.Thread(target=scraper.start_scraping, args=(app,))
-                scraping_thread.daemon = True
-                scraping_thread.start()
-
-                current_app.config['SCRAPER'] = scraper
-                current_app.config['SCRAPING_THREAD'] = scraping_thread
-
-                return jsonify({"status": "Scraping started"})
-            return jsonify({"status": "Invalid form data"})
+            
+            # Convert config to a serializable dict with only needed values
+            config_dict = {
+                'DATA': {'DATA_DIR': current_app.config['DATA']['DATA_DIR']},
+                'DEFAULT': {'API_KEY': current_app.config['DEFAULT']['API_KEY']}
+            }
+            
+            start_scraping_task.delay(
+                faction_id,
+                int(form.fetch_interval.data),  # Ensure this is an int
+                int(form.run_interval.data),    # Ensure this is an int
+                config_dict
+            )
+            return jsonify({"status": "Scraping started"})
+        return jsonify({"status": "Invalid form data"})
 
     @app.route('/stop_scraping', methods=['POST'])
     def stop_scraping():
-        scraper = current_app.config.get('SCRAPER')
-        if scraper is None or not scraper.scraping_active:
-            return jsonify({"status": "Scraping is not running"})
+        redis_client = get_redis()
+        faction_id = redis_client.get("current_faction_id")
+        if not faction_id:
+            return jsonify({"status": "No active scraping session"})
+        
+        stop_scraping_task.delay(faction_id)
+        return jsonify({"status": "Stopping scraping"})
 
-        scraper.stop_scraping()
-        current_app.config['SCRAPING_ACTIVE'] = False
-        current_app.logger.debug("Scraping stopped by user")
-        return jsonify({"status": "Scraping stopped"})    
     @app.route('/logfile', methods=['GET'])
     def logfile():
         log_file_name = current_app.logger.handlers[0].baseFilename
@@ -196,20 +194,37 @@ def register_api(app):
 
     @app.route('/scraping_status', methods=['GET'])
     def scraping_status():
-        if scraper is None:
-            current_app.logger.debug("Scraper is not initialized.")
+        redis_client = get_redis()
+        current_faction_id = redis_client.get("current_faction_id")
+        
+        if not current_faction_id:
             return jsonify({"scraping_active": False})
 
-        if scraper.scraping_active:
-            current_app.logger.debug("Scraping is active.")
-            return jsonify({"scraping_active": True})
-        else:
-            current_app.logger.debug("Scraping is not active.")
+        scraping_active = redis_client.hget(f"scraper:{current_faction_id}", "scraping_active")
+        
+        # If we have a faction_id but scraping is not active, clean up the stale state
+        if not scraping_active or scraping_active == "0":
+            redis_client.delete("current_faction_id")
             return jsonify({"scraping_active": False})
+            
+        return jsonify({
+            "scraping_active": True,
+            "faction_id": current_faction_id
+        })
 
     @app.route('/scraping_get_end_time')
     def scraping_get_end_time():
-        if scraper is None:
-            current_app.logger.debug("Scraper is not initialized.")
-            return jsonify({"scraping_active":False})
-        return jsonify({"end_time": scraper.end_time})
+        redis_client = get_redis()
+        current_faction_id = redis_client.get("current_faction_id")
+        
+        if not current_faction_id:
+            return jsonify({"scraping_active": False})
+            
+        end_time = redis_client.hget(f"scraper:{current_faction_id}", "end_time")
+        if not end_time:
+            return jsonify({"scraping_active": False})
+            
+        return jsonify({
+            "end_time": end_time,
+            "faction_id": current_faction_id
+        })