refactors to use redis & celery

app/tasks.py

@@ -0,0 +1,93 @@
+from celery import Celery
+from app.models import Scraper
+import redis
+from datetime import timedelta
+from flask import current_app
+
+def create_celery():
+    celery = Celery('tasks', broker='redis://localhost:6379/0')
+    celery.conf.update(
+        task_serializer='json',
+        accept_content=['json'],
+        result_serializer='json',
+        timezone='UTC'
+    )
+    return celery
+
+def init_celery(app):
+    """Initialize Celery with Flask app context"""
+    celery = create_celery()
+    celery.conf.update(app.config)
+
+    class ContextTask(celery.Task):
+        def __call__(self, *args, **kwargs):
+            with app.app_context():
+                return self.run(*args, **kwargs)
+
+    celery.Task = ContextTask
+    return celery
+
+celery = create_celery()  # This will be initialized properly in app/__init__.py
+
+def get_redis():
+    return redis.StrictRedis(
+        host='localhost',
+        port=6379,
+        db=0,
+        decode_responses=True
+    )
+
+@celery.task
+def start_scraping_task(faction_id, fetch_interval, run_interval, config_dict):
+    """
+    Start scraping task with serializable parameters
+    Args:
+        faction_id: ID of the faction to scrape
+        fetch_interval: Interval between fetches in seconds
+        run_interval: How long to run the scraper in days
+        config_dict: Dictionary containing configuration
+    """
+    try:
+        redis_client = get_redis()
+        # Set current faction ID at task start
+        redis_client.set("current_faction_id", str(faction_id))
+        
+        scraper = Scraper(
+            faction_id=faction_id,
+            fetch_interval=int(fetch_interval),
+            run_interval=int(run_interval),
+            config=config_dict
+        )
+        scraper.start_scraping()
+        return {"status": "success"}
+    except Exception as e:
+        # Clean up Redis state on error
+        redis_client = get_redis()
+        redis_client.delete("current_faction_id")
+        return {"status": "error", "message": str(e)}
+
+@celery.task
+def stop_scraping_task(faction_id):
+    """Stop scraping task and clean up Redis state"""
+    try:
+        redis_client = get_redis()
+        
+        # Clean up Redis state
+        redis_client.hset(f"scraper:{faction_id}", "scraping_active", "0")
+        redis_client.delete(f"scraper:{faction_id}")
+        
+        # Clean up current_faction_id if it matches
+        current_id = redis_client.get("current_faction_id")
+        if current_id and current_id == str(faction_id):
+            redis_client.delete("current_faction_id")
+            
+        # Revoke any running tasks for this faction
+        celery.control.revoke(
+            celery.current_task.request.id,
+            terminate=True,
+            signal='SIGTERM'
+        )
+        
+        return {"status": "success", "message": f"Stopped scraping for faction {faction_id}"}
+    except Exception as e:
+        return {"status": "error", "message": str(e)}