diff --git a/README.md b/README.md deleted file mode 100644 index 4cfef25..0000000 --- a/README.md +++ /dev/null @@ -1,61 +0,0 @@ -# Torn User Activity Scraper - -This project is a web application that scrapes user activity data from the Torn API and displays the results. It includes features for starting and stopping the scraping process, viewing logs, and downloading results. - -## Features - -- Start and stop scraping user activity data -- View real-time logs -- Download data and log files -- View scraping results and statistics - -## Requirements - -- Python 3.8+ -- Flask -- Flask-Bootstrap -- Flask-WTF -- Pandas -- Requests - -## Installation - -1. Clone the repository: - -```sh -git clone https://github.com/yourusername/torn-user-activity-scraper.git -cd torn-user-activity-scraper -``` - -2. Create a virtual environment and activate it: -```sh -python3 -m venv venv -source venv/bin/activate -``` - -3. Install the required packages: -```sh -pip install -r requirements.txt -``` - -4. Set up your configuration file: -Create a `config.ini` file in the root directory of the project by renaming `example_config.ini` with the following content: - -```ini -[DEFAULT] -SECRET_KEY = your_secret_key -API_KEY = your_api_key -``` - -## Usage - -1. Run the Flask application: -```sh -flask run -``` - -2. Open your web browser and navigate to `http://127.0.0.1:5000/`. - -## License - -This project is licensed under the MIT License. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app.py b/app/app.py similarity index 99% rename from app.py rename to app/app.py index 9da94f2..ccbf241 100644 --- a/app.py +++ b/app/app.py @@ -1,6 +1,6 @@ from flask import Flask, request, render_template, Response, jsonify, url_for from flask_bootstrap import Bootstrap5 # from package boostrap_flask -from forms import ScrapingForm +from app.forms import ScrapingForm import requests import pandas as pd import time diff --git a/app/config.py b/app/config.py new file mode 100644 index 0000000..e69de29 diff --git a/forms.py b/app/forms.py similarity index 91% rename from forms.py rename to app/forms.py index b311862..b4c5039 100644 --- a/forms.py +++ b/app/forms.py @@ -6,4 +6,4 @@ class ScrapingForm(FlaskForm): faction_id = StringField('Faction ID', validators=[DataRequired()], default='9686') fetch_interval = IntegerField('Fetch Interval (seconds)', validators=[DataRequired()], default=60) run_interval = IntegerField('Run Interval (days)', validators=[DataRequired()], default=1) - submit = SubmitField('Start Scraping') \ No newline at end of file + submit = SubmitField('Start') \ No newline at end of file diff --git a/app/models.py b/app/models.py new file mode 100644 index 0000000..e69de29 diff --git a/static/color_mode.js b/app/static/color_mode.js similarity index 100% rename from static/color_mode.js rename to app/static/color_mode.js diff --git a/static/download_results.js b/app/static/download_results.js similarity index 100% rename from static/download_results.js rename to app/static/download_results.js diff --git a/static/index.js b/app/static/index.js similarity index 98% rename from static/index.js rename to app/static/index.js index d4b8389..8297074 100644 --- a/static/index.js +++ b/app/static/index.js @@ -6,7 +6,7 @@ class LogScraperApp { this.prevPageButton = document.getElementById('prevPage'); this.nextPageButton = document.getElementById('nextPage'); this.pageInfo = document.getElementById('pageInfo'); - this.startButton = document.querySelector('button[type="submit"]'); + this.startButton = document.getElementById('startButton'); this.currentPage = 0; this.linesPerPage = null; diff --git a/static/style.css b/app/static/style.css similarity index 100% rename from static/style.css rename to app/static/style.css diff --git a/templates/analyze.html b/app/templates/analyze.html similarity index 100% rename from templates/analyze.html rename to app/templates/analyze.html diff --git a/app/templates/base.html b/app/templates/base.html new file mode 100644 index 0000000..a85aa5e --- /dev/null +++ b/app/templates/base.html @@ -0,0 +1,29 @@ + + + + + {% block head %} + + TornActivityTracker{% block title %}{% endblock %} + + + {% block styles %} + {{ bootstrap.load_css() }} + + + {% endblock %} + {% endblock %} + + +
+ {% include 'includes/navigation.html' %} +
+
+ {% block content %} + {% endblock %} +
+ {% block scripts %} + {% include 'includes/scripts.html' %} + {% endblock %} + + \ No newline at end of file diff --git a/templates/download_results.html b/app/templates/download_results.html similarity index 100% rename from templates/download_results.html rename to app/templates/download_results.html diff --git a/app/templates/includes/navigation.html b/app/templates/includes/navigation.html new file mode 100644 index 0000000..66935c4 --- /dev/null +++ b/app/templates/includes/navigation.html @@ -0,0 +1,17 @@ + + \ No newline at end of file diff --git a/app/templates/includes/scripts.html b/app/templates/includes/scripts.html new file mode 100644 index 0000000..6715dd4 --- /dev/null +++ b/app/templates/includes/scripts.html @@ -0,0 +1,2 @@ +{{ bootstrap.load_js() }} + \ No newline at end of file diff --git a/templates/index.html b/app/templates/index.html similarity index 94% rename from templates/index.html rename to app/templates/index.html index 790ffd1..b42d230 100644 --- a/templates/index.html +++ b/app/templates/index.html @@ -20,7 +20,7 @@
{{ form.submit(class="btn btn-success", type="submit", id="startButton", form="scrapingForm") }} - {{ form.submit(class="btn btn-warning", type="submit", id="stopButton") }} +
@@ -42,7 +42,5 @@ - {% block scripts %} - {% endblock %} {% endblock content %} diff --git a/templates/log_viewer.html b/app/templates/log_viewer.html similarity index 100% rename from templates/log_viewer.html rename to app/templates/log_viewer.html diff --git a/app/util.py b/app/util.py new file mode 100644 index 0000000..e69de29 diff --git a/app/views.py b/app/views.py new file mode 100644 index 0000000..e69de29 diff --git a/example_config.ini b/example_config.ini deleted file mode 100644 index 9a9c88b..0000000 --- a/example_config.ini +++ /dev/null @@ -1,30 +0,0 @@ -[DEFAULT] -# Secret key for session management -SECRET_KEY = your_secret_key -# API key for accessing the TORN API. Public key should be enough -API_KEY = your_api_key - -[LOGGING] -# Maximum number of lines to display in the log viewer -VIEW_MAX_LINES = 500 -# Number of lines to display per page in the log viewer -VIEW_PAGE_LINES = 50 -# Number of bytes to read at a time -TAIL_PAGE_SIZE = 100 - -[BOOTSTRAP] -# See: -# https://bootstrap-flask.readthedocs.io/en/stable/basic/#configurations -BOOTSTRAP_SERVE_LOCAL = False -BOOTSTRAP_BTN_STYLE = 'primary' -BOOTSTRAP_BTN_SIZE = 'sm' -BOOTSTRAP_ICON_SIZE = '1em' -BOOTSTRAP_ICON_COLOR = None -BOOTSTRAP_BOOTSWATCH_THEME = litera -BOOTSTRAP_MSG_CATEGORY = 'primary' -BOOTSTRAP_TABLE_VIEW_TITLE = 'View' -BOOTSTRAP_TABLE_EDIT_TITLE = 'Edit' -BOOTSTRAP_TABLE_DELETE_TITLE = 'Delete' -BOOTSTRAP_TABLE_NEW_TITLE = 'New' -BOOTSTRAP_FORM_GROUP_CLASSES = 'mb-3' -BOOTSTRAP_FORM_INLINE_CLASSES = 'row row-cols-lg-auto g-3 align-items-center' \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index dcd3564..0000000 --- a/requirements.txt +++ /dev/null @@ -1,31 +0,0 @@ -blinker==1.9.0 -Bootstrap-Flask==2.4.1 -certifi==2025.1.31 -charset-normalizer==3.4.1 -click==8.1.8 -configparser==7.1.0 -DateTime==5.5 -distlib==0.3.9 -dominate==2.9.1 -filelock==3.17.0 -Flask==3.1.0 -Flask-WTF==1.2.2 -idna==3.10 -itsdangerous==2.2.0 -Jinja2==3.1.5 -MarkupSafe==3.0.2 -numpy==2.2.2 -pandas==2.2.3 -platformdirs==4.3.6 -python-dateutil==2.9.0.post0 -pytz==2025.1 -requests==2.32.3 -setuptools==75.8.0 -six==1.17.0 -tzdata==2025.1 -urllib3==2.3.0 -virtualenv==20.29.1 -visitor==0.1.3 -Werkzeug==3.1.3 -WTForms==3.2.1 -zope.interface==7.2 diff --git a/run.py b/run.py new file mode 100644 index 0000000..ca5709b --- /dev/null +++ b/run.py @@ -0,0 +1,4 @@ +from app.app import app + +if __name__ == '__main__': + app.run(debug=True, threaded=True) \ No newline at end of file diff --git a/templates/base.html b/templates/base.html deleted file mode 100644 index b8c66b3..0000000 --- a/templates/base.html +++ /dev/null @@ -1,48 +0,0 @@ - - - - {% block head %} - - TornActivityTracker{% block title %}{% endblock %} - - - - {% block styles %} - {{ bootstrap.load_css() }} - - - - {% endblock %} - {% endblock %} - - -
- -
-
- {% block content %} - {% endblock %} - - -
- {% block scripts %} - {{ bootstrap.load_js() }} - {% endblock %} - - - \ No newline at end of file diff --git a/templates/results.html b/templates/results.html deleted file mode 100644 index a9fb16a..0000000 --- a/templates/results.html +++ /dev/null @@ -1,22 +0,0 @@ - - - - - Scraping Results - - -

User Activity Statistics

- - - - - - {% for hour, count in stats.items() %} - - - - - {% endfor %} -
HourActivity Count
{{ hour }}{{ count }}
- -