Big commit that adds and changes a bunch of stuff.

A lot of this will be changed around again anyway.

source/charts/mod.rs

@@ -0,0 +1,116 @@
+//! All code for drawing the [`plotters`] charts.
+
+use {
+  async_std::{fs::create_dir_all, path::PathBuf},
+  color_eyre::Result,
+  plotters::prelude::*,
+};
+
+use crate::group_data::GroupDataModel;
+
+const BACKGROUND_1: RGBColor = RGBColor(17, 17, 17);
+const BACKGROUND_2: RGBColor = RGBColor(0, 0, 0);
+const FOREGROUND: RGBColor = RGBColor(255, 255, 255);
+const ACCENT_1: RGBColor = RGBColor(255, 0, 255);
+
+/// The chart for the user count.
+#[derive(Debug)]
+pub struct UserCountChart {
+  /// The groups to use for user counts.
+  pub groups: Vec<GroupDataModel>,
+}
+
+impl UserCountChart {
+  /// Render the chart and write it to file.
+  pub async fn render(&self, parent: &PathBuf, group_name: &str) -> Result<()> {
+    let parent = parent.join("charts");
+    create_dir_all(&parent).await?;
+
+    let (mut datapoints, mut min_count, mut max_count) = (vec![], i64::MAX, 0);
+
+    for (index, group) in self.groups.iter().enumerate() {
+      datapoints.push(((index + 1) as isize, group.subscribers));
+
+      if group.subscribers > max_count {
+        max_count = group.subscribers;
+      }
+
+      if group.subscribers < min_count {
+        min_count = group.subscribers;
+      }
+    }
+
+    let datapoints_len = datapoints.len() as isize;
+    let min_count = min_count - 10;
+    let max_count = max_count + 10;
+
+    let path = parent.join("user-count.svg");
+    let chart_root = SVGBackend::new(&path, (1280, 720)).into_drawing_area();
+    chart_root.fill(&BACKGROUND_1)?;
+
+    let text_style =
+      |font_size: i32| ("sans-serif", font_size).into_font().color(&FOREGROUND);
+
+    let chart_root = chart_root
+      .margin(20, 20, 20, 20)
+      .titled("Tildes User Count", text_style(30))?;
+
+    chart_root.fill(&BACKGROUND_1)?;
+
+    let mut chart = ChartBuilder::on(&chart_root)
+      .caption(
+        format!("Using the {group_name} subscriber count."),
+        text_style(20),
+      )
+      .x_label_area_size(40)
+      .y_label_area_size(40)
+      .margin(10)
+      .build_cartesian_2d(0..(datapoints_len + 1), min_count..max_count)?;
+
+    chart
+      .configure_mesh()
+      .x_labels(datapoints.len() + 2)
+      .x_label_formatter(&|x| format!("{:0}", datapoints_len - x))
+      .x_desc("N days ago")
+      .y_labels(5)
+      .y_label_formatter(&|y| format!("{y:0}"))
+      .label_style(text_style(20))
+      .axis_style(&BACKGROUND_2)
+      .light_line_style(&BACKGROUND_2)
+      .bold_line_style(&BACKGROUND_1)
+      .draw()?;
+
+    chart
+      .draw_series(LineSeries::new(
+        datapoints.clone(),
+        ACCENT_1.stroke_width(2),
+      ))?
+      .label("User Count")
+      .legend(|(x, y)| {
+        PathElement::new(vec![(x, y), (x + 20, y)], ACCENT_1.stroke_width(4))
+      });
+
+    chart.draw_series(PointSeries::of_element(
+      datapoints,
+      5,
+      &ACCENT_1,
+      &|(x, y), size, style| {
+        EmptyElement::at((x, y))
+          + Circle::new((0, 0), size, style.filled())
+          + Text::new(
+            {
+              if (x - 1) % 2 != 0 {
+                String::new()
+              } else {
+                format!("{:0}", y)
+              }
+            },
+            (-10, 15),
+            text_style(20),
+          )
+      },
+    ))?;
+
+    Ok(())
+  }
+}

source/cli/mod.rs

@@ -1,6 +1,7 @@
 //! All CLI-related code.
 
 use {
+  async_std::path::PathBuf,
   chrono::NaiveDate,
   clap::{Parser, Subcommand},
 };
@@ -43,6 +44,13 @@ pub enum MainSubcommands {
     #[command(subcommand)]
     command: SnapshotSubcommands,
   },
+
+  /// Website management.
+  Web {
+    /// Website management.
+    #[command(subcommand)]
+    command: WebSubcommands,
+  },
 }
 
 /// Migrate subcommands.
@@ -86,3 +94,14 @@ pub enum SnapshotSubcommands {
     date: Option<NaiveDate>,
   },
 }
+
+/// Website subcommands.
+#[derive(Debug, Subcommand)]
+pub enum WebSubcommands {
+  /// Build the website.
+  Build {
+    /// The output directory for the website files.
+    #[clap(short, long, default_value = "public")]
+    output: PathBuf,
+  },
+}

source/cli/run.rs

@@ -1,15 +1,21 @@
 //! All logic for running the CLI.
 
 use {
-  clap::Parser, color_eyre::Result, sea_orm_migration::MigratorTrait,
+  async_std::fs::create_dir_all, clap::Parser, color_eyre::Result,
-  tracing::info,
+  sea_orm_migration::MigratorTrait, tracing::info,
 };
 
 use crate::{
-  cli::{Cli, MainSubcommands, MigrateSubcommands, SnapshotSubcommands},
+  charts::UserCountChart,
-  group_data::get_all_by_snapshot,
+  cli::{
+    Cli, MainSubcommands, MigrateSubcommands, SnapshotSubcommands,
+    WebSubcommands,
+  },
+  group_data::GroupDataModel,
   migrations::Migrator,
-  snapshots::{self, get_by_date},
+  scss::generate_css,
+  snapshots::SnapshotModel,
+  templates::HomeTemplate,
   utilities::{create_db, today},
 };
 
@@ -43,18 +49,20 @@ pub async fn run() -> Result<()> {
       command: snapshot_command,
     } => match snapshot_command {
       SnapshotSubcommands::Create { force } => {
-        snapshots::create(&db, force).await?;
+        SnapshotModel::create(&db, force).await?;
       }
 
       SnapshotSubcommands::List {} => {
-        for snapshot in snapshots::get_all(&db).await? {
+        for snapshot in SnapshotModel::get_all(&db).await? {
           info!("Snapshot {snapshot:?}")
         }
       }
 
       SnapshotSubcommands::Show { date } => {
         let date = date.unwrap_or_else(today);
-        let snapshot = if let Some(snapshot) = get_by_date(&db, date).await? {
+        let snapshot = if let Some(snapshot) =
+          SnapshotModel::get_by_date(&db, date).await?
+        {
           info!("Snapshot {snapshot:?}");
           snapshot
         } else {
@@ -62,8 +70,8 @@ pub async fn run() -> Result<()> {
           return Ok(());
         };
 
-        let groups = get_all_by_snapshot(&db, &snapshot).await?;
+        for group in GroupDataModel::get_all_by_snapshot(&db, &snapshot).await?
-        for group in groups {
+        {
           info!(
             id = group.id,
             name = group.name,
@@ -72,6 +80,35 @@ pub async fn run() -> Result<()> {
         }
       }
     },
+
+    MainSubcommands::Web {
+      command: web_command,
+    } => match web_command {
+      WebSubcommands::Build { output } => {
+        let user_count_group =
+          if let Some(snapshot) = SnapshotModel::get_most_recent(&db).await? {
+            GroupDataModel::get_highest_subscribers(&db, &snapshot).await?
+          } else {
+            None
+          };
+
+        create_dir_all(&output).await?;
+        HomeTemplate::new(
+          user_count_group.as_ref().map(|group| group.subscribers),
+        )
+        .render_to_file(&output)
+        .await?;
+        generate_css(&output).await?;
+
+        if let Some(group) = user_count_group {
+          let groups =
+            GroupDataModel::get_n_most_recent(&db, 30, &group.name).await?;
+          UserCountChart { groups }
+            .render(&output, &group.name)
+            .await?;
+        }
+      }
+    },
   }
 
   Ok(())

source/group_data/mod.rs

@@ -1,14 +1,55 @@
 //! All logic for group datas.
 
-use {color_eyre::Result, sea_orm::prelude::*};
+use {
+  color_eyre::Result,
+  sea_orm::{prelude::*, QueryOrder, QuerySelect},
+};
 
-use crate::entities::{group_data, snapshot};
+pub use crate::{
+  entities::group_data::{
+    ActiveModel as GroupDataActiveModel, Column as GroupDataColumn,
+    Entity as GroupDataEntity, Model as GroupDataModel,
+  },
+  snapshots::SnapshotModel,
+};
 
-/// Get all group datas from a given snapshot.
+impl GroupDataModel {
-pub async fn get_all_by_snapshot(
+  /// Get all group datas from a given snapshot.
-  db: &DatabaseConnection,
+  pub async fn get_all_by_snapshot(
-  snapshot: &snapshot::Model,
+    db: &DatabaseConnection,
-) -> Result<Vec<group_data::Model>> {
+    snapshot: &SnapshotModel,
-  let groups = snapshot.find_related(group_data::Entity).all(db).await?;
+  ) -> Result<Vec<Self>> {
-  Ok(groups)
+    let groups = snapshot.find_related(GroupDataEntity).all(db).await?;
+    Ok(groups)
+  }
+
+  /// Get the group with the highest subscriber count from a given snapshot.
+  pub async fn get_highest_subscribers(
+    db: &DatabaseConnection,
+    snapshot: &SnapshotModel,
+  ) -> Result<Option<Self>> {
+    let group = snapshot
+      .find_related(GroupDataEntity)
+      .order_by_desc(GroupDataColumn::Subscribers)
+      .one(db)
+      .await?;
+
+    Ok(group)
+  }
+
+  /// Get the N most recently saved group datas from a given group name.
+  pub async fn get_n_most_recent(
+    db: &DatabaseConnection,
+    amount: u64,
+    name: &str,
+  ) -> Result<Vec<Self>> {
+    let groups = GroupDataEntity::find()
+      .order_by_desc(GroupDataColumn::SnapshotId)
+      .filter(GroupDataColumn::Name.eq(name))
+      .limit(amount)
+      .all(db)
+      .await?;
+
+    Ok(groups)
+  }
 }

source/main.rs

@@ -11,9 +11,11 @@ use {
   tracing_subscriber::filter::{EnvFilter, LevelFilter},
 };
 
+pub mod charts;
 pub mod cli;
 pub mod group_data;
 pub mod migrations;
+pub mod scss;
 pub mod snapshots;
 pub mod templates;
 pub mod utilities;

source/scss/common.scss

@@ -0,0 +1,47 @@
+html {
+  font-size: 62.5%;
+}
+
+body {
+  --small-spacing: 4px;
+  --medium-spacing: 8px;
+  --large-spacing: 16px;
+  --background-1: #222;
+  --background-2: #111;
+  --foreground-1: #fff;
+  --anchor-1: #f0f;
+  --anchor-2: #000;
+
+  background-color: var(--background-1);
+  color: var(--foreground-1);
+  font-size: 2rem;
+}
+
+a,
+a:visited {
+  color: var(--anchor-1);
+  text-decoration: none;
+
+  &:hover {
+    background-color: var(--anchor-1);
+    color: var(--anchor-2);
+  }
+}
+
+h1,
+h2,
+h3,
+p,
+ol,
+li {
+  margin: 0;
+  padding: 0;
+}
+
+.bold {
+  font-weight: bold;
+}
+
+.underline {
+  text-decoration: underline;
+}

source/scss/index.scss

@@ -0,0 +1,35 @@
+@mixin responsive-container($breakpoint) {
+  margin-left: auto;
+  margin-right: auto;
+  width: $breakpoint;
+
+  @media (max-width: $breakpoint) {
+    width: 100%;
+  }
+}
+
+.page-header,
+.page-main,
+.page-footer {
+  @include responsive-container(1200px);
+
+  margin-bottom: var(--large-spacing);
+  padding: var(--large-spacing);
+}
+
+.page-header {
+  border-bottom: 4px solid var(--background-2);
+}
+
+.page-main {
+  h2 {
+    margin-bottom: var(--medium-spacing);
+  }
+}
+
+.page-footer {
+  background-color: var(--background-2);
+  display: flex;
+  flex-direction: column;
+  gap: var(--large-spacing);
+}

source/scss/mod.rs

@@ -0,0 +1,40 @@
+//! All SCSS files.
+
+use {
+  async_std::{
+    fs::{create_dir_all, write},
+    path::PathBuf,
+  },
+  color_eyre::{eyre::Context, Result},
+};
+
+const MODERN_NORMALIZE_CSS: &str =
+  include_str!("../../node_modules/modern-normalize/modern-normalize.css");
+
+/// Generate the CSS files and write them.
+pub async fn generate_css(parent: &PathBuf) -> Result<()> {
+  let parent = parent.join("css");
+  create_dir_all(&parent).await?;
+
+  let render = |scss: &str| -> Result<String> {
+    grass::from_string(scss.to_string(), &grass::Options::default())
+      .wrap_err("Failed SCSS render")
+  };
+
+  let css_to_create = vec![
+    ("modern-normalize.css", MODERN_NORMALIZE_CSS, false),
+    ("common.css", include_str!("common.scss"), true),
+    ("index.css", include_str!("index.scss"), true),
+  ];
+
+  for (file, css, is_scss) in css_to_create {
+    let path = parent.join(file);
+    if is_scss {
+      write(path, render(css)?).await?;
+    } else {
+      write(path, css).await?;
+    }
+  }
+
+  Ok(())
+}

source/snapshots/create.rs

@@ -8,69 +8,71 @@ use {
 };
 
 use crate::{
-  entities::{group_data, snapshot},
+  group_data::{GroupDataActiveModel, GroupDataEntity},
-  snapshots::get_by_date,
+  snapshots::{SnapshotActiveModel, SnapshotModel},
   utilities::{create_http_client, download_html, today},
 };
 
-/// Create a snapshot for today.
+impl SnapshotModel {
-pub async fn create(db: &DatabaseConnection, force: bool) -> Result<()> {
+  /// Create a snapshot for today.
-  let snapshot_date = today();
+  pub async fn create(db: &DatabaseConnection, force: bool) -> Result<()> {
-  match (force, get_by_date(db, snapshot_date).await?) {
+    let snapshot_date = today();
-    (true, Some(existing)) => {
+    match (force, Self::get_by_date(db, snapshot_date).await?) {
-      info!("Removing existing snapshot {:?}", existing);
+      (true, Some(existing)) => {
-      existing.delete(db).await?;
+        info!("Removing existing snapshot {:?}", existing);
-    }
+        existing.delete(db).await?;
+      }
 
-    (false, Some(existing)) => {
+      (false, Some(existing)) => {
-      info!("Snapshot for today already exists");
+        info!("Snapshot for today already exists");
-      info!("Use --force to override snapshot {:?}", existing);
+        info!("Use --force to override snapshot {:?}", existing);
-      return Ok(());
+        return Ok(());
-    }
+      }
 
-    (_, None) => (),
+      (_, None) => (),
-  };
+    };
 
-  let transaction = db.begin().await?;
+    let transaction = db.begin().await?;
-  let snapshot = snapshot::ActiveModel {
+    let snapshot = SnapshotActiveModel {
-    date: Set(snapshot_date),
+      date: Set(snapshot_date),
-    ..Default::default()
-  }
-  .insert(&transaction)
-  .await?;
-
-  info!("Scraping data for snapshot {:?}", snapshot);
-
-  let http = create_http_client()?;
-  let group_list = GroupList::from_html(
-    &download_html(&http, "https://tildes.net/groups").await?,
-  )?;
-
-  let mut groups_to_insert = vec![];
-
-  for summary in group_list.summaries {
-    debug!(summary = ?summary);
-    let group = Group::from_html(
-      &download_html(&http, format!("https://tildes.net/{}", summary.name))
-        .await?,
-    )?;
-
-    debug!(group = ?group);
-    groups_to_insert.push(group_data::ActiveModel {
-      description: Set(group.description),
-      name: Set(group.name),
-      snapshot_id: Set(snapshot.id),
-      subscribers: Set(group.subscribers.into()),
       ..Default::default()
-    });
+    }
-  }
+    .insert(&transaction)
-
-  info!("Inserting {} groups", groups_to_insert.len());
-  group_data::Entity::insert_many(groups_to_insert)
-    .exec(&transaction)
     .await?;
 
-  transaction.commit().await?;
+    info!("Scraping data for snapshot {:?}", snapshot);
 
-  Ok(())
+    let http = create_http_client()?;
+    let group_list = GroupList::from_html(
+      &download_html(&http, "https://tildes.net/groups").await?,
+    )?;
+
+    let mut groups_to_insert = vec![];
+
+    for summary in group_list.summaries {
+      debug!(summary = ?summary);
+      let group = Group::from_html(
+        &download_html(&http, format!("https://tildes.net/{}", summary.name))
+          .await?,
+      )?;
+
+      debug!(group = ?group);
+      groups_to_insert.push(GroupDataActiveModel {
+        description: Set(group.description),
+        name: Set(group.name),
+        snapshot_id: Set(snapshot.id),
+        subscribers: Set(group.subscribers.into()),
+        ..Default::default()
+      });
+    }
+
+    info!("Inserting {} groups", groups_to_insert.len());
+    GroupDataEntity::insert_many(groups_to_insert)
+      .exec(&transaction)
+      .await?;
+
+    transaction.commit().await?;
+
+    Ok(())
+  }
 }

source/snapshots/mod.rs

@@ -5,29 +5,44 @@ use {
   sea_orm::{prelude::*, QueryOrder},
 };
 
-use crate::entities::snapshot;
-
 mod create;
 
-pub use create::create;
+pub use crate::entities::snapshot::{
+  ActiveModel as SnapshotActiveModel, Column as SnapshotColumn,
+  Entity as SnapshotEntity, Model as SnapshotModel,
+};
 
-/// Get a snapshot for a given date.
+impl SnapshotModel {
-pub async fn get_by_date(
+  /// Get a snapshot for a given date.
-  db: &DatabaseConnection,
+  pub async fn get_by_date(
-  date: ChronoDate,
+    db: &DatabaseConnection,
-) -> Result<Option<snapshot::Model>> {
+    date: ChronoDate,
-  let existing = snapshot::Entity::find()
+  ) -> Result<Option<Self>> {
-    .filter(snapshot::Column::Date.eq(date))
+    let existing = SnapshotEntity::find()
-    .order_by_desc(snapshot::Column::Date)
+      .filter(SnapshotColumn::Date.eq(date))
-    .one(db)
+      .order_by_desc(SnapshotColumn::Date)
-    .await?;
+      .one(db)
+      .await?;
 
-  Ok(existing)
+    Ok(existing)
-}
+  }
-
+
-/// Get all snapshots.
+  /// Get all snapshots.
-pub async fn get_all(db: &DatabaseConnection) -> Result<Vec<snapshot::Model>> {
+  pub async fn get_all(db: &DatabaseConnection) -> Result<Vec<Self>> {
-  let snapshots = snapshot::Entity::find().all(db).await?;
+    let snapshots = SnapshotEntity::find().all(db).await?;
-
+
-  Ok(snapshots)
+    Ok(snapshots)
+  }
+
+  /// Get the most recent snapshot.
+  pub async fn get_most_recent(
+    db: &DatabaseConnection,
+  ) -> Result<Option<Self>> {
+    let snapshot = SnapshotEntity::find()
+      .order_by_desc(SnapshotColumn::Date)
+      .one(db)
+      .await?;
+
+    Ok(snapshot)
+  }
 }

source/templates/base.html

@@ -7,6 +7,7 @@
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
   <title>{{ page_title }}</title>
   <link rel="stylesheet" href="/css/modern-normalize.css">
+  <link rel="stylesheet" href="/css/common.css">
   {% block head %}{% endblock %}
 </head>
 

source/templates/index.html

@@ -0,0 +1,43 @@
+{% extends "base.html" %}
+
+{% block head %}
+<link rel="stylesheet" href="/css/index.css">
+{% endblock %}
+
+{% block body %}
+<header class="page-header">
+  <h1>Tildes Statistics</h1>
+</header>
+
+<main class="page-main">
+  <h2>General</h2>
+
+  <p>
+    There are currently an
+    <abbr title="Based on the group with the highest subscriber count.">estimated</abbr>
+    <span class="underline">{{ user_count }}</span>
+    registered users on Tildes.
+  </p>
+
+  <img src="/charts/user-count.svg" alt="User Count Chart">
+</main>
+
+<footer class="page-footer">
+  <p>
+    Last generated on
+    <time class="underline" datetime="{{ today }}">{{- today -}}</time>.
+  </p>
+
+  <p>
+    &copy; Code
+    <a href="https://git.bauke.xyz/Bauke/tildes-statistics">AGPL-3.0-or-later</a>,
+    charts & data
+    <a href="https://creativecommons.org/licenses/by-nc/4.0/">CC BY-NC 4.0</a>.
+  </p>
+
+  <p class="bold">
+    Consider joining <a href="https://tildes.net">Tildes</a>, a non-profit
+    community site driven by its users' interests.
+  </p>
+</footer>
+{% endblock %}

source/templates/mod.rs

@@ -1 +1,44 @@
 //! All HTML templates.
+
+use {
+  askama::Template,
+  async_std::{fs::write, path::PathBuf},
+  chrono::NaiveDate,
+  color_eyre::Result,
+};
+
+use crate::utilities::today;
+
+/// The template for the home page.
+#[derive(Template)]
+#[template(path = "index.html")]
+pub struct HomeTemplate {
+  /// The string for the `<title>` element.
+  pub page_title: String,
+
+  /// The date of today's snapshot.
+  pub today: NaiveDate,
+
+  /// The user count from the group with the most subscribers.
+  pub user_count: String,
+}
+
+impl HomeTemplate {
+  /// Create a new [`HomeTemplate`].
+  pub fn new(user_count: Option<i64>) -> Self {
+    Self {
+      page_title: "Tildes Statistics".to_string(),
+      today: today(),
+      user_count: user_count
+        .map(|n| n.to_string())
+        .unwrap_or_else(|| "unknown".to_string()),
+    }
+  }
+
+  /// Render the template and write it to file.
+  pub async fn render_to_file(&self, parent: &PathBuf) -> Result<()> {
+    write(parent.join("index.html"), self.render()?).await?;
+
+    Ok(())
+  }
+}