tildes-community/tildes-statistics
tildes-community
/
tildes-statistics
1
Fork 0
Code Issues Packages Releases Activity

Compare commits

base: tildes-community:163502505a1e62ef85b1abe8618955983b2d2bbd
Branches Tags
tildes-community:main
..
compare: tildes-community:6f142f5c3949ab459657849c2c12028617d7ff64
Branches Tags
tildes-community:main

No commits in common. "163502505a1e62ef85b1abe8618955983b2d2bbd" and "6f142f5c3949ab459657849c2c12028617d7ff64" have entirely different histories.
163502505a ... 6f142f5c39

17 changed files with 0 additions and 4175 deletions
Show Stats Expand all files Collapse all files

3520
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

41
Cargo.toml
View File

@ -1,41 +0,0 @@
[package]
name = "tildes-statistics"
description = "Statistics for Tildes.net."
repository = "https://git.bauke.xyz/Bauke/tildes-statistics"
license = "AGPL-3.0-or-later"
version = "0.1.0"
authors = ["Bauke <me@bauke.xyz>"]
edition = "2021"
[[bin]]
name = "tildes-statistics"
path = "source/main.rs"
[dependencies]
async-std = "1.12.0"
chrono = "0.4.22"
color-eyre = "0.6.2"
dotenvy = "0.15.5"
sea-orm-migration = "0.9.3"
tracing = "0.1.36"
[dependencies.clap]
features = ["derive"]
version = "4.0.10"
[dependencies.sea-orm]
features = ["macros", "mock", "runtime-async-std-rustls", "sqlx-postgres"]
version = "0.9.3"
[dependencies.surf]
default-features = false
features = ["encoding", "h1-client-rustls", "middleware-logger"]
version = "2.3.2"
[dependencies.tildes-parser]
git = "https://git.bauke.xyz/Bauke/tildes-parser.git"
rev = "08bf7ed"
[dependencies.tracing-subscriber]
features = ["env-filter"]
version = "0.3.15"

25
Dockerfile
View File

@ -1,25 +0,0 @@
FROM rust:1.64 as builder
# Create a new empty project.
RUN USER=root cargo new --bin tildes-statistics
WORKDIR /tildes-statistics
RUN mv src source
# Copy the Cargo files and build in release, caching the dependencies.
COPY Cargo.* .
RUN cargo build --release
# Then copy our code. This way when only the source code changes, the
# dependencies don't have to be entirely rebuilt.
COPY source source
# Remove the cached tildes-statistics dependencies.
RUN rm target/release/deps/tildes_statistics*
# Build the executable with actual source code.
RUN cargo install --path .
# Copy the executable to a smaller final image.
FROM debian:bullseye-slim
COPY --from=builder /usr/local/cargo/bin/tildes-statistics /usr/local/bin
CMD ["tildes-statistics"]

11
README.md
View File

@ -1,11 +0,0 @@
# Tildes 📈 Statistics
> **Statistics for Tildes.net.**
## Feedback
Found a problem or want to request a new feature? Message [@Bauke](https://tildes.net/user/Bauke/new_message) on Tildes or email [me@bauke.xyz](mailto:me@bauke.xyz) and I'll see what I can do for you.
## License
Distributed under the [AGPL-3.0-or-later](https://spdx.org/licenses/AGPL-3.0-or-later.html) license, see [LICENSE](https://git.bauke.xyz/Bauke/tildes-statistics/src/branch/main/LICENSE) for more information.

88
source/cli/mod.rs
View File

@ -1,88 +0,0 @@
//! All CLI-related code.
use {
chrono::NaiveDate,
clap::{Parser, Subcommand},
};
mod run;
pub use run::run;
/// The Clap Derive CLI struct.
#[derive(Debug, Parser)]
#[command(author, version, about)]
#[command(propagate_version = true)]
pub struct Cli {
/// The CLI subcommand.
#[command(subcommand)]
pub command: MainSubcommands,
/// Don't run pending migrations automatically.
#[clap(long)]
pub no_migrate: bool,
/// Output SQL queries in logging.
#[clap(long, global = true)]
pub sql_logging: bool,
}
/// Main CLI subcommands.
#[derive(Debug, Subcommand)]
pub enum MainSubcommands {
/// Database migrations.
Migrate {
/// Database migrations.
#[command(subcommand)]
command: MigrateSubcommands,
},
/// Snapshot management.
Snapshot {
/// Snapshot management.
#[command(subcommand)]
command: SnapshotSubcommands,
},
}
/// Migrate subcommands.
#[derive(Debug, Subcommand)]
pub enum MigrateSubcommands {
/// Rollback applied migrations.
Down {
/// How many migrations to rollback.
#[clap(default_value = "1")]
amount: u32,
},
/// View the status of all migrations.
Status,
/// Apply pending migrations.
Up {
/// How many migrations to apply.
#[clap(default_value = "1")]
amount: u32,
},
}
/// Snapshot subcommands.
#[derive(Debug, Subcommand)]
pub enum SnapshotSubcommands {
/// Create a snapshot for today.
Create {
/// If a snapshot for today already exists, remove it and remake it.
#[clap(long)]
force: bool,
},
/// List available snapshots.
List {},
/// Show a snapshot.
Show {
/// The date of the snapshot to show, defaults to today.
#[clap(short, long)]
date: Option<NaiveDate>,
},
}

78
source/cli/run.rs
View File

@ -1,78 +0,0 @@
//! All logic for running the CLI.
use {
clap::Parser, color_eyre::Result, sea_orm_migration::MigratorTrait,
tracing::info,
};
use crate::{
cli::{Cli, MainSubcommands, MigrateSubcommands, SnapshotSubcommands},
group_data::get_all_by_snapshot,
migrations::Migrator,
snapshots::{self, get_by_date},
utilities::{create_db, today},
};
/// Run the CLI.
pub async fn run() -> Result<()> {
let cli = Cli::parse();
let db = create_db(cli.sql_logging).await?;
if !cli.no_migrate {
Migrator::up(&db, None).await?;
}
match cli.command {
MainSubcommands::Migrate {
command: migrate_command,
} => match migrate_command {
MigrateSubcommands::Down { amount } => {
Migrator::down(&db, Some(amount)).await?;
}
MigrateSubcommands::Status => {
Migrator::status(&db).await?;
}
MigrateSubcommands::Up { amount } => {
Migrator::up(&db, Some(amount)).await?;
}
},
MainSubcommands::Snapshot {
command: snapshot_command,
} => match snapshot_command {
SnapshotSubcommands::Create { force } => {
snapshots::create(&db, force).await?;
}
SnapshotSubcommands::List {} => {
for snapshot in snapshots::get_all(&db).await? {
info!("Snapshot {snapshot:?}")
}
}
SnapshotSubcommands::Show { date } => {
let date = date.unwrap_or_else(today);
let snapshot = if let Some(snapshot) = get_by_date(&db, date).await? {
info!("Snapshot {snapshot:?}");
snapshot
} else {
info!("No snapshot exists for {date}");
return Ok(());
};
let groups = get_all_by_snapshot(&db, &snapshot).await?;
for group in groups {
info!(
id = group.id,
name = group.name,
subscribers = group.subscribers,
);
}
}
},
}
Ok(())
}

36
source/entities/group_data.rs
View File

@ -1,36 +0,0 @@
//! SeaORM Entity. Generated by sea-orm-codegen 0.9.3
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, DeriveEntityModel)]
#[sea_orm(table_name = "group_data")]
pub struct Model {
#[sea_orm(primary_key)]
pub id: i64,
#[sea_orm(column_type = "Text", nullable)]
pub description: Option<String>,
#[sea_orm(column_type = "Text")]
pub name: String,
pub snapshot_id: i64,
pub subscribers: i64,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {
#[sea_orm(
belongs_to = "super::snapshot::Entity",
from = "Column::SnapshotId",
to = "super::snapshot::Column::Id",
on_update = "NoAction",
on_delete = "Cascade"
)]
Snapshot,
}
impl Related<super::snapshot::Entity> for Entity {
fn to() -> RelationDef {
Relation::Snapshot.def()
}
}
impl ActiveModelBehavior for ActiveModel {}

6
source/entities/mod.rs
View File

@ -1,6 +0,0 @@
//! SeaORM Entity. Generated by sea-orm-codegen 0.9.3
pub mod prelude;
pub mod group_data;
pub mod snapshot;

4
source/entities/prelude.rs
View File

@ -1,4 +0,0 @@
//! SeaORM Entity. Generated by sea-orm-codegen 0.9.3
pub use super::group_data::Entity as GroupData;
pub use super::snapshot::Entity as Snapshot;

25
source/entities/snapshot.rs
View File

@ -1,25 +0,0 @@
//! SeaORM Entity. Generated by sea-orm-codegen 0.9.3
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, DeriveEntityModel)]
#[sea_orm(table_name = "snapshot")]
pub struct Model {
#[sea_orm(primary_key)]
pub id: i64,
pub date: Date,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {
#[sea_orm(has_many = "super::group_data::Entity")]
GroupData,
}
impl Related<super::group_data::Entity> for Entity {
fn to() -> RelationDef {
Relation::GroupData.def()
}
}
impl ActiveModelBehavior for ActiveModel {}

14
source/group_data/mod.rs
View File

@ -1,14 +0,0 @@
//! All logic for group datas.
use {color_eyre::Result, sea_orm::prelude::*};
use crate::entities::{group_data, snapshot};
/// Get all group datas from a given snapshot.
pub async fn get_all_by_snapshot(
db: &DatabaseConnection,
snapshot: &snapshot::Model,
) -> Result<Vec<group_data::Model>> {
let groups = snapshot.find_related(group_data::Entity).all(db).await?;
Ok(groups)
}

40
source/main.rs
View File

@ -1,40 +0,0 @@
//! # Tildes Statistics
//!
//! > **Statistics for Tildes.net.**
#![forbid(unsafe_code)]
#![warn(missing_docs)]
use {
color_eyre::{install, Result},
dotenvy::dotenv,
tracing_subscriber::filter::{EnvFilter, LevelFilter},
};
pub mod cli;
pub mod group_data;
pub mod migrations;
pub mod snapshots;
pub mod utilities;
/// The entities code is auto-generated using `sea-orm-cli`. With a database
/// and `.env` file setup, run the following command.
///
/// ```
/// sea-orm-cli generate entity -o source/entities
/// ```
#[allow(missing_docs, clippy::derive_partial_eq_without_eq)]
pub mod entities;
/// The main function.
pub fn main() -> Result<()> {
install()?;
dotenv().ok();
let env_filter = EnvFilter::builder()
.with_default_directive(LevelFilter::INFO.into())
.from_env_lossy();
tracing_subscriber::fmt().with_env_filter(env_filter).init();
async_std::task::block_on(async { cli::run().await })
}

98
source/migrations/m20221004_000001_initial_setup.rs
View File

@ -1,98 +0,0 @@
//! The migration for initial setup.
use sea_orm_migration::prelude::*;
pub struct Migration;
impl MigrationName for Migration {
fn name(&self) -> &str {
"m20221004_000001_initial_setup"
}
}
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.create_table(
Table::create()
.table(Snapshot::Table)
.if_not_exists()
.col(
ColumnDef::new(Snapshot::Id)
.big_integer()
.not_null()
.auto_increment()
.primary_key(),
)
.col(ColumnDef::new(Snapshot::Date).date().not_null())
.to_owned(),
)
.await?;
manager
.create_table(
Table::create()
.table(GroupData::Table)
.if_not_exists()
.foreign_key(
ForeignKey::create()
.from(GroupData::Table, GroupData::SnapshotId)
.to(Snapshot::Table, Snapshot::Id)
.on_delete(ForeignKeyAction::Cascade),
)
.col(
ColumnDef::new(GroupData::Id)
.big_integer()
.not_null()
.auto_increment()
.primary_key(),
)
.col(ColumnDef::new(GroupData::Description).text())
.col(ColumnDef::new(GroupData::Name).text().not_null())
.col(
ColumnDef::new(GroupData::SnapshotId)
.big_integer()
.not_null(),
)
.col(
ColumnDef::new(GroupData::Subscribers)
.big_integer()
.not_null(),
)
.to_owned(),
)
.await?;
Ok(())
}
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.drop_table(Table::drop().table(GroupData::Table).to_owned())
.await?;
manager
.drop_table(Table::drop().table(Snapshot::Table).to_owned())
.await?;
Ok(())
}
}
#[derive(Iden)]
enum Snapshot {
Table,
Id,
Date,
}
#[derive(Iden)]
enum GroupData {
Table,
Id,
SnapshotId,
Name,
Description,
Subscribers,
}

17
source/migrations/mod.rs
View File

@ -1,17 +0,0 @@
//! Database migrations.
use sea_orm_migration::prelude::*;
mod m20221004_000001_initial_setup;
/// [`sea_orm_migration`] struct, see
/// [Migration (API)](https://www.sea-ql.org/sea-orm-tutorial/ch01-03-migration-api.html)
/// for details.
pub struct Migrator;
#[async_trait::async_trait]
impl MigratorTrait for Migrator {
fn migrations() -> Vec<Box<dyn MigrationTrait>> {
vec![Box::new(m20221004_000001_initial_setup::Migration)]
}
}

76
source/snapshots/create.rs
View File

@ -1,76 +0,0 @@
//! Code for creating a new snapshot.
use {
color_eyre::Result,
sea_orm::{prelude::*, ActiveValue::*, TransactionTrait},
tildes_parser::{Group, GroupList},
tracing::{debug, info},
};
use crate::{
entities::{group_data, snapshot},
snapshots::get_by_date,
utilities::{create_http_client, download_html, today},
};
/// Create a snapshot for today.
pub async fn create(db: &DatabaseConnection, force: bool) -> Result<()> {
let snapshot_date = today();
match (force, get_by_date(db, snapshot_date).await?) {
(true, Some(existing)) => {
info!("Removing existing snapshot {:?}", existing);
existing.delete(db).await?;
}
(false, Some(existing)) => {
info!("Snapshot for today already exists");
info!("Use --force to override snapshot {:?}", existing);
return Ok(());
}
(_, None) => (),
};
let transaction = db.begin().await?;
let snapshot = snapshot::ActiveModel {
date: Set(snapshot_date),
..Default::default()
}
.insert(&transaction)
.await?;
info!("Scraping data for snapshot {:?}", snapshot);
let http = create_http_client()?;
let group_list = GroupList::from_html(
&download_html(&http, "https://tildes.net/groups").await?,
)?;
let mut groups_to_insert = vec![];
for summary in group_list.summaries {
debug!(summary = ?summary);
let group = Group::from_html(
&download_html(&http, format!("https://tildes.net/{}", summary.name))
.await?,
)?;
debug!(group = ?group);
groups_to_insert.push(group_data::ActiveModel {
description: Set(group.description),
name: Set(group.name),
snapshot_id: Set(snapshot.id),
subscribers: Set(group.subscribers.into()),
..Default::default()
});
}
info!("Inserting {} groups", groups_to_insert.len());
group_data::Entity::insert_many(groups_to_insert)
.exec(&transaction)
.await?;
transaction.commit().await?;
Ok(())
}

33
source/snapshots/mod.rs
View File

@ -1,33 +0,0 @@
//! All logic for snapshots.
use {
color_eyre::Result,
sea_orm::{prelude::*, QueryOrder},
};
use crate::entities::snapshot;
mod create;
pub use create::create;
/// Get a snapshot for a given date.
pub async fn get_by_date(
db: &DatabaseConnection,
date: ChronoDate,
) -> Result<Option<snapshot::Model>> {
let existing = snapshot::Entity::find()
.filter(snapshot::Column::Date.eq(date))
.order_by_desc(snapshot::Column::Date)
.one(db)
.await?;
Ok(existing)
}
/// Get all snapshots.
pub async fn get_all(db: &DatabaseConnection) -> Result<Vec<snapshot::Model>> {
let snapshots = snapshot::Entity::find().all(db).await?;
Ok(snapshots)
}

63
source/utilities.rs
View File

@ -1,63 +0,0 @@
//! Helper functions and miscellaneous utilities.
use std::time::Duration;
use {
async_std::task::sleep,
chrono::{NaiveDate, Utc},
color_eyre::{
eyre::{eyre, WrapErr},
Result,
},
sea_orm::{ConnectOptions, Database, DatabaseConnection},
surf::{Client, Config},
tildes_parser::Html,
};
/// Creates the SeaQL [`DatabaseConnection`].
pub async fn create_db(sql_logging: bool) -> Result<DatabaseConnection> {
let database_url = get_env_var("DATABASE_URL")?;
let mut connect_options = ConnectOptions::new(database_url);
connect_options.sqlx_logging(sql_logging);
Database::connect(connect_options)
.await
.wrap_err("Failed to connect to database")
}
/// Creates the HTTP [`Client`].
pub fn create_http_client() -> Result<Client> {
let user_agent = get_env_var("USER_AGENT")?;
let http: Client = Config::default()
.add_header("User-Agent", user_agent)
.map_err(|err| eyre!(err))?
.try_into()?;
Ok(http)
}
/// Shorthand to download a URL and parse it to [`Html`].
pub async fn download_html(
http: &Client,
url: impl AsRef<str>,
) -> Result<Html> {
sleep(Duration::from_millis(500)).await;
let html = http
.get(url)
.recv_string()
.await
.map_err(|err| eyre!(err))?;
Ok(Html::parse_document(&html))
}
/// Shorthand for [`std::env::var`] with wrapped error message.
pub fn get_env_var(key: &str) -> Result<String> {
std::env::var(key).wrap_err(key.to_string())
}
/// Create a [`NaiveDate`] for today.
pub fn today() -> NaiveDate {
Utc::now().date().naive_utc()
}