mischbeck/TornActivityTracker
1
0
Fork 0
Code Issues 5 Pull Requests Packages Projects Releases Wiki Activity

1st restructure

Browse Source
This commit is contained in:
Michael Beck
2025-02-06 23:27:30 +01:00
parent f552601c4b
commit ceebbafed5
24 changed files with 56 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
README.md
View File

@@ -1,61 +0,0 @@
# Torn User Activity Scraper
This project is a web application that scrapes user activity data from the Torn API and displays the results. It includes features for starting and stopping the scraping process, viewing logs, and downloading results.
## Features
- Start and stop scraping user activity data
- View real-time logs
- Download data and log files
- View scraping results and statistics
## Requirements
- Python 3.8+
- Flask
- Flask-Bootstrap
- Flask-WTF
- Pandas
- Requests
## Installation
1. Clone the repository:
```sh
git clone https://github.com/yourusername/torn-user-activity-scraper.git
cd torn-user-activity-scraper
```
2. Create a virtual environment and activate it:
```sh
python3 -m venv venv
source venv/bin/activate
```
3. Install the required packages:
```sh
pip install -r requirements.txt
```
4. Set up your configuration file:
Create a `config.ini` file in the root directory of the project by renaming `example_config.ini` with the following content:
```ini
[DEFAULT]
SECRET_KEY = your_secret_key
API_KEY = your_api_key
```
## Usage
1. Run the Flask application:
```sh
flask run
```
2. Open your web browser and navigate to `http://127.0.0.1:5000/`.
## License
This project is licensed under the MIT License.

0
app/__init__.py Normal file
View File

2
app.py → app/app.py
View File

@@ -1,6 +1,6 @@
from flask import Flask, request, render_template, Response, jsonify, url_for
from flask_bootstrap import Bootstrap5 # from package boostrap_flask
from forms import ScrapingForm
from app.forms import ScrapingForm
import requests
import pandas as pd
import time

0
app/config.py Normal file
View File

2
forms.py → app/forms.py
View File

@@ -6,4 +6,4 @@ class ScrapingForm(FlaskForm):
faction_id = StringField('Faction ID', validators=[DataRequired()], default='9686')
fetch_interval = IntegerField('Fetch Interval (seconds)', validators=[DataRequired()], default=60)
run_interval = IntegerField('Run Interval (days)', validators=[DataRequired()], default=1)
submit = SubmitField('Start Scraping')
submit = SubmitField('Start')

0
app/models.py Normal file
View File

0
static/color_mode.js → app/static/color_mode.js
View File

0
static/download_results.js → app/static/download_results.js
View File

2
static/index.js → app/static/index.js
View File

@@ -6,7 +6,7 @@ class LogScraperApp {
this.prevPageButton = document.getElementById('prevPage');
this.nextPageButton = document.getElementById('nextPage');
this.pageInfo = document.getElementById('pageInfo');
this.startButton = document.querySelector('button[type="submit"]');
this.startButton = document.getElementById('startButton');
this.currentPage = 0;
this.linesPerPage = null;

0
static/style.css → app/static/style.css
View File

0
templates/analyze.html → app/templates/analyze.html
View File

29
app/templates/base.html Normal file
View File

@@ -0,0 +1,29 @@
<!-- app/templates/layouts/base.html -->
<!DOCTYPE html>
<html lang="en">
<head>
{% block head %}
<meta charset="UTF-8">
<title>TornActivityTracker{% block title %}{% endblock %}</title>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
{% block styles %}
{{ bootstrap.load_css() }}
<link rel="stylesheet" href="{{url_for('static', filename='style.css')}}">
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons/font/bootstrap-icons.css">
{% endblock %}
{% endblock %}
</head>
<body>
<header>
{% include 'includes/navigation.html' %}
</header>
<main>
{% block content %}
{% endblock %}
</main>
{% block scripts %}
{% include 'includes/scripts.html' %}
{% endblock %}
</body>
</html>

0
templates/download_results.html → app/templates/download_results.html
View File

17
app/templates/includes/navigation.html Normal file
View File

@@ -0,0 +1,17 @@
<!-- app/templates/includes/navigation.html -->
<nav class="navbar navbar-nav navbar-expand-md bg-primary">
<div class="container-fluid">
<a class="navbar-brand" href="/">Torn User Activity Scraper</a>
{% from 'bootstrap4/nav.html' import render_nav_item %}
{{ render_nav_item('analyze', 'Analyze') }}
{{ render_nav_item('download_results', 'Files') }}
{{ render_nav_item('log_viewer', 'Logs') }}
<div class="d-flex" id="color-mode-toggle">
<input type="checkbox" id="bd-theme" />
<label for="bd-theme">
<span class="icon sun"><i class="bi bi-brightness-high"></i></span>
<span class="icon moon"><i class="bi bi-moon-stars"></i></span>
</label>
</div>
</div>
</nav>

2
app/templates/includes/scripts.html Normal file
View File

@@ -0,0 +1,2 @@
{{ bootstrap.load_js() }}
<script src="{{url_for('static', filename='color_mode.js')}}"></script>

4
templates/index.html → app/templates/index.html
View File

@@ -20,7 +20,7 @@
</form>
<div class="btn-group btn-group m-2" role="group">
{{ form.submit(class="btn btn-success", type="submit", id="startButton", form="scrapingForm") }}
{{ form.submit(class="btn btn-warning", type="submit", id="stopButton") }}
<button class="btn btn-warning" type="button" id="stopButton">Stop</button>
</div>
</div>
</section>
@@ -42,7 +42,5 @@
</div>
</div>
</section>
{% block scripts %}
<script src="{{url_for('static', filename='index.js')}}"></script>
{% endblock %}
{% endblock content %}

0
templates/log_viewer.html → app/templates/log_viewer.html
View File

0
app/util.py Normal file
View File

0
app/views.py Normal file
View File

30
example_config.ini
View File

@@ -1,30 +0,0 @@
[DEFAULT]
# Secret key for session management
SECRET_KEY = your_secret_key
# API key for accessing the TORN API. Public key should be enough
API_KEY = your_api_key
[LOGGING]
# Maximum number of lines to display in the log viewer
VIEW_MAX_LINES = 500
# Number of lines to display per page in the log viewer
VIEW_PAGE_LINES = 50
# Number of bytes to read at a time
TAIL_PAGE_SIZE = 100
[BOOTSTRAP]
# See:
# https://bootstrap-flask.readthedocs.io/en/stable/basic/#configurations
BOOTSTRAP_SERVE_LOCAL = False
BOOTSTRAP_BTN_STYLE = 'primary'
BOOTSTRAP_BTN_SIZE = 'sm'
BOOTSTRAP_ICON_SIZE = '1em'
BOOTSTRAP_ICON_COLOR = None
BOOTSTRAP_BOOTSWATCH_THEME = litera
BOOTSTRAP_MSG_CATEGORY = 'primary'
BOOTSTRAP_TABLE_VIEW_TITLE = 'View'
BOOTSTRAP_TABLE_EDIT_TITLE = 'Edit'
BOOTSTRAP_TABLE_DELETE_TITLE = 'Delete'
BOOTSTRAP_TABLE_NEW_TITLE = 'New'
BOOTSTRAP_FORM_GROUP_CLASSES = 'mb-3'
BOOTSTRAP_FORM_INLINE_CLASSES = 'row row-cols-lg-auto g-3 align-items-center'

31
requirements.txt
View File

@@ -1,31 +0,0 @@
blinker==1.9.0
Bootstrap-Flask==2.4.1
certifi==2025.1.31
charset-normalizer==3.4.1
click==8.1.8
configparser==7.1.0
DateTime==5.5
distlib==0.3.9
dominate==2.9.1
filelock==3.17.0
Flask==3.1.0
Flask-WTF==1.2.2
idna==3.10
itsdangerous==2.2.0
Jinja2==3.1.5
MarkupSafe==3.0.2
numpy==2.2.2
pandas==2.2.3
platformdirs==4.3.6
python-dateutil==2.9.0.post0
pytz==2025.1
requests==2.32.3
setuptools==75.8.0
six==1.17.0
tzdata==2025.1
urllib3==2.3.0
virtualenv==20.29.1
visitor==0.1.3
Werkzeug==3.1.3
WTForms==3.2.1
zope.interface==7.2

4
run.py Normal file
View File

@@ -0,0 +1,4 @@
from app.app import app
if __name__ == '__main__':
app.run(debug=True, threaded=True)

48
templates/base.html
View File

@@ -1,48 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
{% block head %}
<meta charset="UTF-8">
<title>TornActivityTracker{% block title %}{% endblock %}</title>
<!-- Required meta tags -->
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
{% block styles %}
{{ bootstrap.load_css() }}
<link rel="stylesheet" href="{{url_for('static', filename='style.css')}}">
<!-- color mode js-->
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons/font/bootstrap-icons.css">
{% endblock %}
{% endblock %}
</head>
<body>
<header>
<nav class="navbar navbar-nav navbar-expand-md bg-primary">
<div class="container-fluid">
<a class="navbar-brand" href="/">Torn User Activity Scraper</a>
{% from 'bootstrap4/nav.html' import render_nav_item %}
{{ render_nav_item('analyze', 'Analyze') }}
{{ render_nav_item('download_results', 'Files') }}
{{ render_nav_item('log_viewer', 'Logs') }}
<div class="d-flex" id="color-mode-toggle">
<input type="checkbox" id="bd-theme" />
<label for="bd-theme">
<span class="icon sun"><i class="bi bi-brightness-high"></i></span>
<span class="icon moon"><i class="bi bi-moon-stars"></i></span>
</label>
</div>
</div>
</nav>
</header>
<main>
{% block content %}
{% endblock %}
</main>
{% block scripts %}
{{ bootstrap.load_js() }}
{% endblock %}
<script src="{{url_for('static', filename='color_mode.js')}}"></script>
</body>
</html>

22
templates/results.html
View File

@@ -1,22 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Scraping Results</title>
</head>
<body>
<h1>User Activity Statistics</h1>
<table border="1">
<tr>
<th>Hour</th>
<th>Activity Count</th>
</tr>
{% for hour, count in stats.items() %}
<tr>
<td>{{ hour }}</td>
<td>{{ count }}</td>
</tr>
{% endfor %}
</table>
</body>
</html>