1st restructure
This commit is contained in:
Michael Beck
61
README.md
61
README.md
@@ -1,61 +0,0 @@
|
|||||||
# Torn User Activity Scraper
|
|
||||||
|
|
||||||
This project is a web application that scrapes user activity data from the Torn API and displays the results. It includes features for starting and stopping the scraping process, viewing logs, and downloading results.
|
|
||||||
|
|
||||||
## Features
|
|
||||||
|
|
||||||
- Start and stop scraping user activity data
|
|
||||||
- View real-time logs
|
|
||||||
- Download data and log files
|
|
||||||
- View scraping results and statistics
|
|
||||||
|
|
||||||
## Requirements
|
|
||||||
|
|
||||||
- Python 3.8+
|
|
||||||
- Flask
|
|
||||||
- Flask-Bootstrap
|
|
||||||
- Flask-WTF
|
|
||||||
- Pandas
|
|
||||||
- Requests
|
|
||||||
|
|
||||||
## Installation
|
|
||||||
|
|
||||||
1. Clone the repository:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
git clone https://github.com/yourusername/torn-user-activity-scraper.git
|
|
||||||
cd torn-user-activity-scraper
|
|
||||||
```
|
|
||||||
|
|
||||||
2. Create a virtual environment and activate it:
|
|
||||||
```sh
|
|
||||||
python3 -m venv venv
|
|
||||||
source venv/bin/activate
|
|
||||||
```
|
|
||||||
|
|
||||||
3. Install the required packages:
|
|
||||||
```sh
|
|
||||||
pip install -r requirements.txt
|
|
||||||
```
|
|
||||||
|
|
||||||
4. Set up your configuration file:
|
|
||||||
Create a `config.ini` file in the root directory of the project by renaming `example_config.ini` with the following content:
|
|
||||||
|
|
||||||
```ini
|
|
||||||
[DEFAULT]
|
|
||||||
SECRET_KEY = your_secret_key
|
|
||||||
API_KEY = your_api_key
|
|
||||||
```
|
|
||||||
|
|
||||||
## Usage
|
|
||||||
|
|
||||||
1. Run the Flask application:
|
|
||||||
```sh
|
|
||||||
flask run
|
|
||||||
```
|
|
||||||
|
|
||||||
2. Open your web browser and navigate to `http://127.0.0.1:5000/`.
|
|
||||||
|
|
||||||
## License
|
|
||||||
|
|
||||||
This project is licensed under the MIT License.
|
|
||||||
0
app/__init__.py
Normal file
0
app/__init__.py
Normal file
@@ -1,6 +1,6 @@
|
|||||||
from flask import Flask, request, render_template, Response, jsonify, url_for
|
from flask import Flask, request, render_template, Response, jsonify, url_for
|
||||||
from flask_bootstrap import Bootstrap5 # from package boostrap_flask
|
from flask_bootstrap import Bootstrap5 # from package boostrap_flask
|
||||||
from forms import ScrapingForm
|
from app.forms import ScrapingForm
|
||||||
import requests
|
import requests
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import time
|
import time
|
||||||
0
app/config.py
Normal file
0
app/config.py
Normal file
@@ -6,4 +6,4 @@ class ScrapingForm(FlaskForm):
|
|||||||
faction_id = StringField('Faction ID', validators=[DataRequired()], default='9686')
|
faction_id = StringField('Faction ID', validators=[DataRequired()], default='9686')
|
||||||
fetch_interval = IntegerField('Fetch Interval (seconds)', validators=[DataRequired()], default=60)
|
fetch_interval = IntegerField('Fetch Interval (seconds)', validators=[DataRequired()], default=60)
|
||||||
run_interval = IntegerField('Run Interval (days)', validators=[DataRequired()], default=1)
|
run_interval = IntegerField('Run Interval (days)', validators=[DataRequired()], default=1)
|
||||||
submit = SubmitField('Start Scraping')
|
submit = SubmitField('Start')
|
||||||
0
app/models.py
Normal file
0
app/models.py
Normal file
@@ -6,7 +6,7 @@ class LogScraperApp {
|
|||||||
this.prevPageButton = document.getElementById('prevPage');
|
this.prevPageButton = document.getElementById('prevPage');
|
||||||
this.nextPageButton = document.getElementById('nextPage');
|
this.nextPageButton = document.getElementById('nextPage');
|
||||||
this.pageInfo = document.getElementById('pageInfo');
|
this.pageInfo = document.getElementById('pageInfo');
|
||||||
this.startButton = document.querySelector('button[type="submit"]');
|
this.startButton = document.getElementById('startButton');
|
||||||
|
|
||||||
this.currentPage = 0;
|
this.currentPage = 0;
|
||||||
this.linesPerPage = null;
|
this.linesPerPage = null;
|
||||||
29
app/templates/base.html
Normal file
29
app/templates/base.html
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
<!-- app/templates/layouts/base.html -->
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
{% block head %}
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<title>TornActivityTracker{% block title %}{% endblock %}</title>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
|
||||||
|
{% block styles %}
|
||||||
|
{{ bootstrap.load_css() }}
|
||||||
|
<link rel="stylesheet" href="{{url_for('static', filename='style.css')}}">
|
||||||
|
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons/font/bootstrap-icons.css">
|
||||||
|
{% endblock %}
|
||||||
|
{% endblock %}
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<header>
|
||||||
|
{% include 'includes/navigation.html' %}
|
||||||
|
</header>
|
||||||
|
<main>
|
||||||
|
{% block content %}
|
||||||
|
{% endblock %}
|
||||||
|
</main>
|
||||||
|
{% block scripts %}
|
||||||
|
{% include 'includes/scripts.html' %}
|
||||||
|
{% endblock %}
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
17
app/templates/includes/navigation.html
Normal file
17
app/templates/includes/navigation.html
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
<!-- app/templates/includes/navigation.html -->
|
||||||
|
<nav class="navbar navbar-nav navbar-expand-md bg-primary">
|
||||||
|
<div class="container-fluid">
|
||||||
|
<a class="navbar-brand" href="/">Torn User Activity Scraper</a>
|
||||||
|
{% from 'bootstrap4/nav.html' import render_nav_item %}
|
||||||
|
{{ render_nav_item('analyze', 'Analyze') }}
|
||||||
|
{{ render_nav_item('download_results', 'Files') }}
|
||||||
|
{{ render_nav_item('log_viewer', 'Logs') }}
|
||||||
|
<div class="d-flex" id="color-mode-toggle">
|
||||||
|
<input type="checkbox" id="bd-theme" />
|
||||||
|
<label for="bd-theme">
|
||||||
|
<span class="icon sun"><i class="bi bi-brightness-high"></i></span>
|
||||||
|
<span class="icon moon"><i class="bi bi-moon-stars"></i></span>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</nav>
|
||||||
2
app/templates/includes/scripts.html
Normal file
2
app/templates/includes/scripts.html
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
{{ bootstrap.load_js() }}
|
||||||
|
<script src="{{url_for('static', filename='color_mode.js')}}"></script>
|
||||||
@@ -20,7 +20,7 @@
|
|||||||
</form>
|
</form>
|
||||||
<div class="btn-group btn-group m-2" role="group">
|
<div class="btn-group btn-group m-2" role="group">
|
||||||
{{ form.submit(class="btn btn-success", type="submit", id="startButton", form="scrapingForm") }}
|
{{ form.submit(class="btn btn-success", type="submit", id="startButton", form="scrapingForm") }}
|
||||||
{{ form.submit(class="btn btn-warning", type="submit", id="stopButton") }}
|
<button class="btn btn-warning" type="button" id="stopButton">Stop</button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
@@ -42,7 +42,5 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
{% block scripts %}
|
|
||||||
<script src="{{url_for('static', filename='index.js')}}"></script>
|
<script src="{{url_for('static', filename='index.js')}}"></script>
|
||||||
{% endblock %}
|
|
||||||
{% endblock content %}
|
{% endblock content %}
|
||||||
0
app/util.py
Normal file
0
app/util.py
Normal file
0
app/views.py
Normal file
0
app/views.py
Normal file
@@ -1,30 +0,0 @@
|
|||||||
[DEFAULT]
|
|
||||||
# Secret key for session management
|
|
||||||
SECRET_KEY = your_secret_key
|
|
||||||
# API key for accessing the TORN API. Public key should be enough
|
|
||||||
API_KEY = your_api_key
|
|
||||||
|
|
||||||
[LOGGING]
|
|
||||||
# Maximum number of lines to display in the log viewer
|
|
||||||
VIEW_MAX_LINES = 500
|
|
||||||
# Number of lines to display per page in the log viewer
|
|
||||||
VIEW_PAGE_LINES = 50
|
|
||||||
# Number of bytes to read at a time
|
|
||||||
TAIL_PAGE_SIZE = 100
|
|
||||||
|
|
||||||
[BOOTSTRAP]
|
|
||||||
# See:
|
|
||||||
# https://bootstrap-flask.readthedocs.io/en/stable/basic/#configurations
|
|
||||||
BOOTSTRAP_SERVE_LOCAL = False
|
|
||||||
BOOTSTRAP_BTN_STYLE = 'primary'
|
|
||||||
BOOTSTRAP_BTN_SIZE = 'sm'
|
|
||||||
BOOTSTRAP_ICON_SIZE = '1em'
|
|
||||||
BOOTSTRAP_ICON_COLOR = None
|
|
||||||
BOOTSTRAP_BOOTSWATCH_THEME = litera
|
|
||||||
BOOTSTRAP_MSG_CATEGORY = 'primary'
|
|
||||||
BOOTSTRAP_TABLE_VIEW_TITLE = 'View'
|
|
||||||
BOOTSTRAP_TABLE_EDIT_TITLE = 'Edit'
|
|
||||||
BOOTSTRAP_TABLE_DELETE_TITLE = 'Delete'
|
|
||||||
BOOTSTRAP_TABLE_NEW_TITLE = 'New'
|
|
||||||
BOOTSTRAP_FORM_GROUP_CLASSES = 'mb-3'
|
|
||||||
BOOTSTRAP_FORM_INLINE_CLASSES = 'row row-cols-lg-auto g-3 align-items-center'
|
|
||||||
@@ -1,31 +0,0 @@
|
|||||||
blinker==1.9.0
|
|
||||||
Bootstrap-Flask==2.4.1
|
|
||||||
certifi==2025.1.31
|
|
||||||
charset-normalizer==3.4.1
|
|
||||||
click==8.1.8
|
|
||||||
configparser==7.1.0
|
|
||||||
DateTime==5.5
|
|
||||||
distlib==0.3.9
|
|
||||||
dominate==2.9.1
|
|
||||||
filelock==3.17.0
|
|
||||||
Flask==3.1.0
|
|
||||||
Flask-WTF==1.2.2
|
|
||||||
idna==3.10
|
|
||||||
itsdangerous==2.2.0
|
|
||||||
Jinja2==3.1.5
|
|
||||||
MarkupSafe==3.0.2
|
|
||||||
numpy==2.2.2
|
|
||||||
pandas==2.2.3
|
|
||||||
platformdirs==4.3.6
|
|
||||||
python-dateutil==2.9.0.post0
|
|
||||||
pytz==2025.1
|
|
||||||
requests==2.32.3
|
|
||||||
setuptools==75.8.0
|
|
||||||
six==1.17.0
|
|
||||||
tzdata==2025.1
|
|
||||||
urllib3==2.3.0
|
|
||||||
virtualenv==20.29.1
|
|
||||||
visitor==0.1.3
|
|
||||||
Werkzeug==3.1.3
|
|
||||||
WTForms==3.2.1
|
|
||||||
zope.interface==7.2
|
|
||||||
4
run.py
Normal file
4
run.py
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
from app.app import app
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
app.run(debug=True, threaded=True)
|
||||||
@@ -1,48 +0,0 @@
|
|||||||
<!DOCTYPE html>
|
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
{% block head %}
|
|
||||||
<meta charset="UTF-8">
|
|
||||||
<title>TornActivityTracker{% block title %}{% endblock %}</title>
|
|
||||||
<!-- Required meta tags -->
|
|
||||||
<meta charset="utf-8">
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
|
|
||||||
{% block styles %}
|
|
||||||
{{ bootstrap.load_css() }}
|
|
||||||
<link rel="stylesheet" href="{{url_for('static', filename='style.css')}}">
|
|
||||||
<!-- color mode js-->
|
|
||||||
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons/font/bootstrap-icons.css">
|
|
||||||
{% endblock %}
|
|
||||||
{% endblock %}
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<header>
|
|
||||||
<nav class="navbar navbar-nav navbar-expand-md bg-primary">
|
|
||||||
<div class="container-fluid">
|
|
||||||
<a class="navbar-brand" href="/">Torn User Activity Scraper</a>
|
|
||||||
{% from 'bootstrap4/nav.html' import render_nav_item %}
|
|
||||||
{{ render_nav_item('analyze', 'Analyze') }}
|
|
||||||
{{ render_nav_item('download_results', 'Files') }}
|
|
||||||
{{ render_nav_item('log_viewer', 'Logs') }}
|
|
||||||
<div class="d-flex" id="color-mode-toggle">
|
|
||||||
<input type="checkbox" id="bd-theme" />
|
|
||||||
<label for="bd-theme">
|
|
||||||
<span class="icon sun"><i class="bi bi-brightness-high"></i></span>
|
|
||||||
<span class="icon moon"><i class="bi bi-moon-stars"></i></span>
|
|
||||||
</label>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</nav>
|
|
||||||
</header>
|
|
||||||
<main>
|
|
||||||
{% block content %}
|
|
||||||
{% endblock %}
|
|
||||||
|
|
||||||
|
|
||||||
</main>
|
|
||||||
{% block scripts %}
|
|
||||||
{{ bootstrap.load_js() }}
|
|
||||||
{% endblock %}
|
|
||||||
<script src="{{url_for('static', filename='color_mode.js')}}"></script>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
@@ -1,22 +0,0 @@
|
|||||||
<!DOCTYPE html>
|
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<meta charset="UTF-8">
|
|
||||||
<title>Scraping Results</title>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<h1>User Activity Statistics</h1>
|
|
||||||
<table border="1">
|
|
||||||
<tr>
|
|
||||||
<th>Hour</th>
|
|
||||||
<th>Activity Count</th>
|
|
||||||
</tr>
|
|
||||||
{% for hour, count in stats.items() %}
|
|
||||||
<tr>
|
|
||||||
<td>{{ hour }}</td>
|
|
||||||
<td>{{ count }}</td>
|
|
||||||
</tr>
|
|
||||||
{% endfor %}
|
|
||||||
</table>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
Reference in New Issue
Block a user