From 4b3863e0d10e14acf13daacc2ea3e024c2d23968 Mon Sep 17 00:00:00 2001 From: Bauke Date: Fri, 7 Oct 2022 15:27:43 +0200 Subject: [PATCH] Big commit that adds and changes a bunch of stuff. A lot of this will be changed around again anyway. --- source/charts/mod.rs | 116 ++++++++++++++++++++++++++++++++++++ source/cli/mod.rs | 19 ++++++ source/cli/run.rs | 57 ++++++++++++++---- source/group_data/mod.rs | 59 +++++++++++++++--- source/main.rs | 2 + source/scss/common.scss | 47 +++++++++++++++ source/scss/index.scss | 35 +++++++++++ source/scss/mod.rs | 40 +++++++++++++ source/snapshots/create.rs | 112 +++++++++++++++++----------------- source/snapshots/mod.rs | 57 +++++++++++------- source/templates/base.html | 1 + source/templates/index.html | 43 +++++++++++++ source/templates/mod.rs | 43 +++++++++++++ 13 files changed, 536 insertions(+), 95 deletions(-) create mode 100644 source/charts/mod.rs create mode 100644 source/scss/common.scss create mode 100644 source/scss/index.scss create mode 100644 source/scss/mod.rs create mode 100644 source/templates/index.html diff --git a/source/charts/mod.rs b/source/charts/mod.rs new file mode 100644 index 0000000..03dc654 --- /dev/null +++ b/source/charts/mod.rs @@ -0,0 +1,116 @@ +//! All code for drawing the [`plotters`] charts. + +use { + async_std::{fs::create_dir_all, path::PathBuf}, + color_eyre::Result, + plotters::prelude::*, +}; + +use crate::group_data::GroupDataModel; + +const BACKGROUND_1: RGBColor = RGBColor(17, 17, 17); +const BACKGROUND_2: RGBColor = RGBColor(0, 0, 0); +const FOREGROUND: RGBColor = RGBColor(255, 255, 255); +const ACCENT_1: RGBColor = RGBColor(255, 0, 255); + +/// The chart for the user count. +#[derive(Debug)] +pub struct UserCountChart { + /// The groups to use for user counts. + pub groups: Vec, +} + +impl UserCountChart { + /// Render the chart and write it to file. + pub async fn render(&self, parent: &PathBuf, group_name: &str) -> Result<()> { + let parent = parent.join("charts"); + create_dir_all(&parent).await?; + + let (mut datapoints, mut min_count, mut max_count) = (vec![], i64::MAX, 0); + + for (index, group) in self.groups.iter().enumerate() { + datapoints.push(((index + 1) as isize, group.subscribers)); + + if group.subscribers > max_count { + max_count = group.subscribers; + } + + if group.subscribers < min_count { + min_count = group.subscribers; + } + } + + let datapoints_len = datapoints.len() as isize; + let min_count = min_count - 10; + let max_count = max_count + 10; + + let path = parent.join("user-count.svg"); + let chart_root = SVGBackend::new(&path, (1280, 720)).into_drawing_area(); + chart_root.fill(&BACKGROUND_1)?; + + let text_style = + |font_size: i32| ("sans-serif", font_size).into_font().color(&FOREGROUND); + + let chart_root = chart_root + .margin(20, 20, 20, 20) + .titled("Tildes User Count", text_style(30))?; + + chart_root.fill(&BACKGROUND_1)?; + + let mut chart = ChartBuilder::on(&chart_root) + .caption( + format!("Using the {group_name} subscriber count."), + text_style(20), + ) + .x_label_area_size(40) + .y_label_area_size(40) + .margin(10) + .build_cartesian_2d(0..(datapoints_len + 1), min_count..max_count)?; + + chart + .configure_mesh() + .x_labels(datapoints.len() + 2) + .x_label_formatter(&|x| format!("{:0}", datapoints_len - x)) + .x_desc("N days ago") + .y_labels(5) + .y_label_formatter(&|y| format!("{y:0}")) + .label_style(text_style(20)) + .axis_style(&BACKGROUND_2) + .light_line_style(&BACKGROUND_2) + .bold_line_style(&BACKGROUND_1) + .draw()?; + + chart + .draw_series(LineSeries::new( + datapoints.clone(), + ACCENT_1.stroke_width(2), + ))? + .label("User Count") + .legend(|(x, y)| { + PathElement::new(vec![(x, y), (x + 20, y)], ACCENT_1.stroke_width(4)) + }); + + chart.draw_series(PointSeries::of_element( + datapoints, + 5, + &ACCENT_1, + &|(x, y), size, style| { + EmptyElement::at((x, y)) + + Circle::new((0, 0), size, style.filled()) + + Text::new( + { + if (x - 1) % 2 != 0 { + String::new() + } else { + format!("{:0}", y) + } + }, + (-10, 15), + text_style(20), + ) + }, + ))?; + + Ok(()) + } +} diff --git a/source/cli/mod.rs b/source/cli/mod.rs index cd60a65..6ce328e 100644 --- a/source/cli/mod.rs +++ b/source/cli/mod.rs @@ -1,6 +1,7 @@ //! All CLI-related code. use { + async_std::path::PathBuf, chrono::NaiveDate, clap::{Parser, Subcommand}, }; @@ -43,6 +44,13 @@ pub enum MainSubcommands { #[command(subcommand)] command: SnapshotSubcommands, }, + + /// Website management. + Web { + /// Website management. + #[command(subcommand)] + command: WebSubcommands, + }, } /// Migrate subcommands. @@ -86,3 +94,14 @@ pub enum SnapshotSubcommands { date: Option, }, } + +/// Website subcommands. +#[derive(Debug, Subcommand)] +pub enum WebSubcommands { + /// Build the website. + Build { + /// The output directory for the website files. + #[clap(short, long, default_value = "public")] + output: PathBuf, + }, +} diff --git a/source/cli/run.rs b/source/cli/run.rs index bf7955f..9280273 100644 --- a/source/cli/run.rs +++ b/source/cli/run.rs @@ -1,15 +1,21 @@ //! All logic for running the CLI. use { - clap::Parser, color_eyre::Result, sea_orm_migration::MigratorTrait, - tracing::info, + async_std::fs::create_dir_all, clap::Parser, color_eyre::Result, + sea_orm_migration::MigratorTrait, tracing::info, }; use crate::{ - cli::{Cli, MainSubcommands, MigrateSubcommands, SnapshotSubcommands}, - group_data::get_all_by_snapshot, + charts::UserCountChart, + cli::{ + Cli, MainSubcommands, MigrateSubcommands, SnapshotSubcommands, + WebSubcommands, + }, + group_data::GroupDataModel, migrations::Migrator, - snapshots::{self, get_by_date}, + scss::generate_css, + snapshots::SnapshotModel, + templates::HomeTemplate, utilities::{create_db, today}, }; @@ -43,18 +49,20 @@ pub async fn run() -> Result<()> { command: snapshot_command, } => match snapshot_command { SnapshotSubcommands::Create { force } => { - snapshots::create(&db, force).await?; + SnapshotModel::create(&db, force).await?; } SnapshotSubcommands::List {} => { - for snapshot in snapshots::get_all(&db).await? { + for snapshot in SnapshotModel::get_all(&db).await? { info!("Snapshot {snapshot:?}") } } SnapshotSubcommands::Show { date } => { let date = date.unwrap_or_else(today); - let snapshot = if let Some(snapshot) = get_by_date(&db, date).await? { + let snapshot = if let Some(snapshot) = + SnapshotModel::get_by_date(&db, date).await? + { info!("Snapshot {snapshot:?}"); snapshot } else { @@ -62,8 +70,8 @@ pub async fn run() -> Result<()> { return Ok(()); }; - let groups = get_all_by_snapshot(&db, &snapshot).await?; - for group in groups { + for group in GroupDataModel::get_all_by_snapshot(&db, &snapshot).await? + { info!( id = group.id, name = group.name, @@ -72,6 +80,35 @@ pub async fn run() -> Result<()> { } } }, + + MainSubcommands::Web { + command: web_command, + } => match web_command { + WebSubcommands::Build { output } => { + let user_count_group = + if let Some(snapshot) = SnapshotModel::get_most_recent(&db).await? { + GroupDataModel::get_highest_subscribers(&db, &snapshot).await? + } else { + None + }; + + create_dir_all(&output).await?; + HomeTemplate::new( + user_count_group.as_ref().map(|group| group.subscribers), + ) + .render_to_file(&output) + .await?; + generate_css(&output).await?; + + if let Some(group) = user_count_group { + let groups = + GroupDataModel::get_n_most_recent(&db, 30, &group.name).await?; + UserCountChart { groups } + .render(&output, &group.name) + .await?; + } + } + }, } Ok(()) diff --git a/source/group_data/mod.rs b/source/group_data/mod.rs index 53bab65..f752912 100644 --- a/source/group_data/mod.rs +++ b/source/group_data/mod.rs @@ -1,14 +1,55 @@ //! All logic for group datas. -use {color_eyre::Result, sea_orm::prelude::*}; +use { + color_eyre::Result, + sea_orm::{prelude::*, QueryOrder, QuerySelect}, +}; -use crate::entities::{group_data, snapshot}; +pub use crate::{ + entities::group_data::{ + ActiveModel as GroupDataActiveModel, Column as GroupDataColumn, + Entity as GroupDataEntity, Model as GroupDataModel, + }, + snapshots::SnapshotModel, +}; -/// Get all group datas from a given snapshot. -pub async fn get_all_by_snapshot( - db: &DatabaseConnection, - snapshot: &snapshot::Model, -) -> Result> { - let groups = snapshot.find_related(group_data::Entity).all(db).await?; - Ok(groups) +impl GroupDataModel { + /// Get all group datas from a given snapshot. + pub async fn get_all_by_snapshot( + db: &DatabaseConnection, + snapshot: &SnapshotModel, + ) -> Result> { + let groups = snapshot.find_related(GroupDataEntity).all(db).await?; + Ok(groups) + } + + /// Get the group with the highest subscriber count from a given snapshot. + pub async fn get_highest_subscribers( + db: &DatabaseConnection, + snapshot: &SnapshotModel, + ) -> Result> { + let group = snapshot + .find_related(GroupDataEntity) + .order_by_desc(GroupDataColumn::Subscribers) + .one(db) + .await?; + + Ok(group) + } + + /// Get the N most recently saved group datas from a given group name. + pub async fn get_n_most_recent( + db: &DatabaseConnection, + amount: u64, + name: &str, + ) -> Result> { + let groups = GroupDataEntity::find() + .order_by_desc(GroupDataColumn::SnapshotId) + .filter(GroupDataColumn::Name.eq(name)) + .limit(amount) + .all(db) + .await?; + + Ok(groups) + } } diff --git a/source/main.rs b/source/main.rs index 5d9b167..086b0e9 100644 --- a/source/main.rs +++ b/source/main.rs @@ -11,9 +11,11 @@ use { tracing_subscriber::filter::{EnvFilter, LevelFilter}, }; +pub mod charts; pub mod cli; pub mod group_data; pub mod migrations; +pub mod scss; pub mod snapshots; pub mod templates; pub mod utilities; diff --git a/source/scss/common.scss b/source/scss/common.scss new file mode 100644 index 0000000..eab74af --- /dev/null +++ b/source/scss/common.scss @@ -0,0 +1,47 @@ +html { + font-size: 62.5%; +} + +body { + --small-spacing: 4px; + --medium-spacing: 8px; + --large-spacing: 16px; + --background-1: #222; + --background-2: #111; + --foreground-1: #fff; + --anchor-1: #f0f; + --anchor-2: #000; + + background-color: var(--background-1); + color: var(--foreground-1); + font-size: 2rem; +} + +a, +a:visited { + color: var(--anchor-1); + text-decoration: none; + + &:hover { + background-color: var(--anchor-1); + color: var(--anchor-2); + } +} + +h1, +h2, +h3, +p, +ol, +li { + margin: 0; + padding: 0; +} + +.bold { + font-weight: bold; +} + +.underline { + text-decoration: underline; +} diff --git a/source/scss/index.scss b/source/scss/index.scss new file mode 100644 index 0000000..1c047d0 --- /dev/null +++ b/source/scss/index.scss @@ -0,0 +1,35 @@ +@mixin responsive-container($breakpoint) { + margin-left: auto; + margin-right: auto; + width: $breakpoint; + + @media (max-width: $breakpoint) { + width: 100%; + } +} + +.page-header, +.page-main, +.page-footer { + @include responsive-container(1200px); + + margin-bottom: var(--large-spacing); + padding: var(--large-spacing); +} + +.page-header { + border-bottom: 4px solid var(--background-2); +} + +.page-main { + h2 { + margin-bottom: var(--medium-spacing); + } +} + +.page-footer { + background-color: var(--background-2); + display: flex; + flex-direction: column; + gap: var(--large-spacing); +} diff --git a/source/scss/mod.rs b/source/scss/mod.rs new file mode 100644 index 0000000..8776b46 --- /dev/null +++ b/source/scss/mod.rs @@ -0,0 +1,40 @@ +//! All SCSS files. + +use { + async_std::{ + fs::{create_dir_all, write}, + path::PathBuf, + }, + color_eyre::{eyre::Context, Result}, +}; + +const MODERN_NORMALIZE_CSS: &str = + include_str!("../../node_modules/modern-normalize/modern-normalize.css"); + +/// Generate the CSS files and write them. +pub async fn generate_css(parent: &PathBuf) -> Result<()> { + let parent = parent.join("css"); + create_dir_all(&parent).await?; + + let render = |scss: &str| -> Result { + grass::from_string(scss.to_string(), &grass::Options::default()) + .wrap_err("Failed SCSS render") + }; + + let css_to_create = vec![ + ("modern-normalize.css", MODERN_NORMALIZE_CSS, false), + ("common.css", include_str!("common.scss"), true), + ("index.css", include_str!("index.scss"), true), + ]; + + for (file, css, is_scss) in css_to_create { + let path = parent.join(file); + if is_scss { + write(path, render(css)?).await?; + } else { + write(path, css).await?; + } + } + + Ok(()) +} diff --git a/source/snapshots/create.rs b/source/snapshots/create.rs index 5f7f980..01734de 100644 --- a/source/snapshots/create.rs +++ b/source/snapshots/create.rs @@ -8,69 +8,71 @@ use { }; use crate::{ - entities::{group_data, snapshot}, - snapshots::get_by_date, + group_data::{GroupDataActiveModel, GroupDataEntity}, + snapshots::{SnapshotActiveModel, SnapshotModel}, utilities::{create_http_client, download_html, today}, }; -/// Create a snapshot for today. -pub async fn create(db: &DatabaseConnection, force: bool) -> Result<()> { - let snapshot_date = today(); - match (force, get_by_date(db, snapshot_date).await?) { - (true, Some(existing)) => { - info!("Removing existing snapshot {:?}", existing); - existing.delete(db).await?; - } +impl SnapshotModel { + /// Create a snapshot for today. + pub async fn create(db: &DatabaseConnection, force: bool) -> Result<()> { + let snapshot_date = today(); + match (force, Self::get_by_date(db, snapshot_date).await?) { + (true, Some(existing)) => { + info!("Removing existing snapshot {:?}", existing); + existing.delete(db).await?; + } - (false, Some(existing)) => { - info!("Snapshot for today already exists"); - info!("Use --force to override snapshot {:?}", existing); - return Ok(()); - } + (false, Some(existing)) => { + info!("Snapshot for today already exists"); + info!("Use --force to override snapshot {:?}", existing); + return Ok(()); + } - (_, None) => (), - }; + (_, None) => (), + }; - let transaction = db.begin().await?; - let snapshot = snapshot::ActiveModel { - date: Set(snapshot_date), - ..Default::default() - } - .insert(&transaction) - .await?; - - info!("Scraping data for snapshot {:?}", snapshot); - - let http = create_http_client()?; - let group_list = GroupList::from_html( - &download_html(&http, "https://tildes.net/groups").await?, - )?; - - let mut groups_to_insert = vec![]; - - for summary in group_list.summaries { - debug!(summary = ?summary); - let group = Group::from_html( - &download_html(&http, format!("https://tildes.net/{}", summary.name)) - .await?, - )?; - - debug!(group = ?group); - groups_to_insert.push(group_data::ActiveModel { - description: Set(group.description), - name: Set(group.name), - snapshot_id: Set(snapshot.id), - subscribers: Set(group.subscribers.into()), + let transaction = db.begin().await?; + let snapshot = SnapshotActiveModel { + date: Set(snapshot_date), ..Default::default() - }); - } - - info!("Inserting {} groups", groups_to_insert.len()); - group_data::Entity::insert_many(groups_to_insert) - .exec(&transaction) + } + .insert(&transaction) .await?; - transaction.commit().await?; + info!("Scraping data for snapshot {:?}", snapshot); - Ok(()) + let http = create_http_client()?; + let group_list = GroupList::from_html( + &download_html(&http, "https://tildes.net/groups").await?, + )?; + + let mut groups_to_insert = vec![]; + + for summary in group_list.summaries { + debug!(summary = ?summary); + let group = Group::from_html( + &download_html(&http, format!("https://tildes.net/{}", summary.name)) + .await?, + )?; + + debug!(group = ?group); + groups_to_insert.push(GroupDataActiveModel { + description: Set(group.description), + name: Set(group.name), + snapshot_id: Set(snapshot.id), + subscribers: Set(group.subscribers.into()), + ..Default::default() + }); + } + + info!("Inserting {} groups", groups_to_insert.len()); + GroupDataEntity::insert_many(groups_to_insert) + .exec(&transaction) + .await?; + + transaction.commit().await?; + + Ok(()) + } } diff --git a/source/snapshots/mod.rs b/source/snapshots/mod.rs index a8e1538..57d5c61 100644 --- a/source/snapshots/mod.rs +++ b/source/snapshots/mod.rs @@ -5,29 +5,44 @@ use { sea_orm::{prelude::*, QueryOrder}, }; -use crate::entities::snapshot; - mod create; -pub use create::create; +pub use crate::entities::snapshot::{ + ActiveModel as SnapshotActiveModel, Column as SnapshotColumn, + Entity as SnapshotEntity, Model as SnapshotModel, +}; -/// Get a snapshot for a given date. -pub async fn get_by_date( - db: &DatabaseConnection, - date: ChronoDate, -) -> Result> { - let existing = snapshot::Entity::find() - .filter(snapshot::Column::Date.eq(date)) - .order_by_desc(snapshot::Column::Date) - .one(db) - .await?; +impl SnapshotModel { + /// Get a snapshot for a given date. + pub async fn get_by_date( + db: &DatabaseConnection, + date: ChronoDate, + ) -> Result> { + let existing = SnapshotEntity::find() + .filter(SnapshotColumn::Date.eq(date)) + .order_by_desc(SnapshotColumn::Date) + .one(db) + .await?; - Ok(existing) -} - -/// Get all snapshots. -pub async fn get_all(db: &DatabaseConnection) -> Result> { - let snapshots = snapshot::Entity::find().all(db).await?; - - Ok(snapshots) + Ok(existing) + } + + /// Get all snapshots. + pub async fn get_all(db: &DatabaseConnection) -> Result> { + let snapshots = SnapshotEntity::find().all(db).await?; + + Ok(snapshots) + } + + /// Get the most recent snapshot. + pub async fn get_most_recent( + db: &DatabaseConnection, + ) -> Result> { + let snapshot = SnapshotEntity::find() + .order_by_desc(SnapshotColumn::Date) + .one(db) + .await?; + + Ok(snapshot) + } } diff --git a/source/templates/base.html b/source/templates/base.html index 9fc0056..dcd2fa1 100644 --- a/source/templates/base.html +++ b/source/templates/base.html @@ -7,6 +7,7 @@ {{ page_title }} + {% block head %}{% endblock %} diff --git a/source/templates/index.html b/source/templates/index.html new file mode 100644 index 0000000..fda8b18 --- /dev/null +++ b/source/templates/index.html @@ -0,0 +1,43 @@ +{% extends "base.html" %} + +{% block head %} + +{% endblock %} + +{% block body %} +
+

Tildes Statistics

+
+ +
+

General

+ +

+ There are currently an + estimated + {{ user_count }} + registered users on Tildes. +

+ + User Count Chart +
+ +
+

+ Last generated on + . +

+ +

+ © Code + AGPL-3.0-or-later, + charts & data + CC BY-NC 4.0. +

+ +

+ Consider joining Tildes, a non-profit + community site driven by its users' interests. +

+
+{% endblock %} diff --git a/source/templates/mod.rs b/source/templates/mod.rs index 26c533f..3e04c82 100644 --- a/source/templates/mod.rs +++ b/source/templates/mod.rs @@ -1 +1,44 @@ //! All HTML templates. + +use { + askama::Template, + async_std::{fs::write, path::PathBuf}, + chrono::NaiveDate, + color_eyre::Result, +}; + +use crate::utilities::today; + +/// The template for the home page. +#[derive(Template)] +#[template(path = "index.html")] +pub struct HomeTemplate { + /// The string for the `` element. + pub page_title: String, + + /// The date of today's snapshot. + pub today: NaiveDate, + + /// The user count from the group with the most subscribers. + pub user_count: String, +} + +impl HomeTemplate { + /// Create a new [`HomeTemplate`]. + pub fn new(user_count: Option<i64>) -> Self { + Self { + page_title: "Tildes Statistics".to_string(), + today: today(), + user_count: user_count + .map(|n| n.to_string()) + .unwrap_or_else(|| "unknown".to_string()), + } + } + + /// Render the template and write it to file. + pub async fn render_to_file(&self, parent: &PathBuf) -> Result<()> { + write(parent.join("index.html"), self.render()?).await?; + + Ok(()) + } +}