Big commit that adds and changes a bunch of stuff.

A lot of this will be changed around again anyway.
This commit is contained in:
Bauke 2022-10-07 15:27:43 +02:00
parent 99b24d0eac
commit 4b3863e0d1
Signed by: Bauke
GPG Key ID: C1C0F29952BCF558
13 changed files with 536 additions and 95 deletions

116
source/charts/mod.rs Normal file
View File

@ -0,0 +1,116 @@
//! All code for drawing the [`plotters`] charts.
use {
async_std::{fs::create_dir_all, path::PathBuf},
color_eyre::Result,
plotters::prelude::*,
};
use crate::group_data::GroupDataModel;
const BACKGROUND_1: RGBColor = RGBColor(17, 17, 17);
const BACKGROUND_2: RGBColor = RGBColor(0, 0, 0);
const FOREGROUND: RGBColor = RGBColor(255, 255, 255);
const ACCENT_1: RGBColor = RGBColor(255, 0, 255);
/// The chart for the user count.
#[derive(Debug)]
pub struct UserCountChart {
/// The groups to use for user counts.
pub groups: Vec<GroupDataModel>,
}
impl UserCountChart {
/// Render the chart and write it to file.
pub async fn render(&self, parent: &PathBuf, group_name: &str) -> Result<()> {
let parent = parent.join("charts");
create_dir_all(&parent).await?;
let (mut datapoints, mut min_count, mut max_count) = (vec![], i64::MAX, 0);
for (index, group) in self.groups.iter().enumerate() {
datapoints.push(((index + 1) as isize, group.subscribers));
if group.subscribers > max_count {
max_count = group.subscribers;
}
if group.subscribers < min_count {
min_count = group.subscribers;
}
}
let datapoints_len = datapoints.len() as isize;
let min_count = min_count - 10;
let max_count = max_count + 10;
let path = parent.join("user-count.svg");
let chart_root = SVGBackend::new(&path, (1280, 720)).into_drawing_area();
chart_root.fill(&BACKGROUND_1)?;
let text_style =
|font_size: i32| ("sans-serif", font_size).into_font().color(&FOREGROUND);
let chart_root = chart_root
.margin(20, 20, 20, 20)
.titled("Tildes User Count", text_style(30))?;
chart_root.fill(&BACKGROUND_1)?;
let mut chart = ChartBuilder::on(&chart_root)
.caption(
format!("Using the {group_name} subscriber count."),
text_style(20),
)
.x_label_area_size(40)
.y_label_area_size(40)
.margin(10)
.build_cartesian_2d(0..(datapoints_len + 1), min_count..max_count)?;
chart
.configure_mesh()
.x_labels(datapoints.len() + 2)
.x_label_formatter(&|x| format!("{:0}", datapoints_len - x))
.x_desc("N days ago")
.y_labels(5)
.y_label_formatter(&|y| format!("{y:0}"))
.label_style(text_style(20))
.axis_style(&BACKGROUND_2)
.light_line_style(&BACKGROUND_2)
.bold_line_style(&BACKGROUND_1)
.draw()?;
chart
.draw_series(LineSeries::new(
datapoints.clone(),
ACCENT_1.stroke_width(2),
))?
.label("User Count")
.legend(|(x, y)| {
PathElement::new(vec![(x, y), (x + 20, y)], ACCENT_1.stroke_width(4))
});
chart.draw_series(PointSeries::of_element(
datapoints,
5,
&ACCENT_1,
&|(x, y), size, style| {
EmptyElement::at((x, y))
+ Circle::new((0, 0), size, style.filled())
+ Text::new(
{
if (x - 1) % 2 != 0 {
String::new()
} else {
format!("{:0}", y)
}
},
(-10, 15),
text_style(20),
)
},
))?;
Ok(())
}
}

View File

@ -1,6 +1,7 @@
//! All CLI-related code. //! All CLI-related code.
use { use {
async_std::path::PathBuf,
chrono::NaiveDate, chrono::NaiveDate,
clap::{Parser, Subcommand}, clap::{Parser, Subcommand},
}; };
@ -43,6 +44,13 @@ pub enum MainSubcommands {
#[command(subcommand)] #[command(subcommand)]
command: SnapshotSubcommands, command: SnapshotSubcommands,
}, },
/// Website management.
Web {
/// Website management.
#[command(subcommand)]
command: WebSubcommands,
},
} }
/// Migrate subcommands. /// Migrate subcommands.
@ -86,3 +94,14 @@ pub enum SnapshotSubcommands {
date: Option<NaiveDate>, date: Option<NaiveDate>,
}, },
} }
/// Website subcommands.
#[derive(Debug, Subcommand)]
pub enum WebSubcommands {
/// Build the website.
Build {
/// The output directory for the website files.
#[clap(short, long, default_value = "public")]
output: PathBuf,
},
}

View File

@ -1,15 +1,21 @@
//! All logic for running the CLI. //! All logic for running the CLI.
use { use {
clap::Parser, color_eyre::Result, sea_orm_migration::MigratorTrait, async_std::fs::create_dir_all, clap::Parser, color_eyre::Result,
tracing::info, sea_orm_migration::MigratorTrait, tracing::info,
}; };
use crate::{ use crate::{
cli::{Cli, MainSubcommands, MigrateSubcommands, SnapshotSubcommands}, charts::UserCountChart,
group_data::get_all_by_snapshot, cli::{
Cli, MainSubcommands, MigrateSubcommands, SnapshotSubcommands,
WebSubcommands,
},
group_data::GroupDataModel,
migrations::Migrator, migrations::Migrator,
snapshots::{self, get_by_date}, scss::generate_css,
snapshots::SnapshotModel,
templates::HomeTemplate,
utilities::{create_db, today}, utilities::{create_db, today},
}; };
@ -43,18 +49,20 @@ pub async fn run() -> Result<()> {
command: snapshot_command, command: snapshot_command,
} => match snapshot_command { } => match snapshot_command {
SnapshotSubcommands::Create { force } => { SnapshotSubcommands::Create { force } => {
snapshots::create(&db, force).await?; SnapshotModel::create(&db, force).await?;
} }
SnapshotSubcommands::List {} => { SnapshotSubcommands::List {} => {
for snapshot in snapshots::get_all(&db).await? { for snapshot in SnapshotModel::get_all(&db).await? {
info!("Snapshot {snapshot:?}") info!("Snapshot {snapshot:?}")
} }
} }
SnapshotSubcommands::Show { date } => { SnapshotSubcommands::Show { date } => {
let date = date.unwrap_or_else(today); let date = date.unwrap_or_else(today);
let snapshot = if let Some(snapshot) = get_by_date(&db, date).await? { let snapshot = if let Some(snapshot) =
SnapshotModel::get_by_date(&db, date).await?
{
info!("Snapshot {snapshot:?}"); info!("Snapshot {snapshot:?}");
snapshot snapshot
} else { } else {
@ -62,8 +70,8 @@ pub async fn run() -> Result<()> {
return Ok(()); return Ok(());
}; };
let groups = get_all_by_snapshot(&db, &snapshot).await?; for group in GroupDataModel::get_all_by_snapshot(&db, &snapshot).await?
for group in groups { {
info!( info!(
id = group.id, id = group.id,
name = group.name, name = group.name,
@ -72,6 +80,35 @@ pub async fn run() -> Result<()> {
} }
} }
}, },
MainSubcommands::Web {
command: web_command,
} => match web_command {
WebSubcommands::Build { output } => {
let user_count_group =
if let Some(snapshot) = SnapshotModel::get_most_recent(&db).await? {
GroupDataModel::get_highest_subscribers(&db, &snapshot).await?
} else {
None
};
create_dir_all(&output).await?;
HomeTemplate::new(
user_count_group.as_ref().map(|group| group.subscribers),
)
.render_to_file(&output)
.await?;
generate_css(&output).await?;
if let Some(group) = user_count_group {
let groups =
GroupDataModel::get_n_most_recent(&db, 30, &group.name).await?;
UserCountChart { groups }
.render(&output, &group.name)
.await?;
}
}
},
} }
Ok(()) Ok(())

View File

@ -1,14 +1,55 @@
//! All logic for group datas. //! All logic for group datas.
use {color_eyre::Result, sea_orm::prelude::*}; use {
color_eyre::Result,
sea_orm::{prelude::*, QueryOrder, QuerySelect},
};
use crate::entities::{group_data, snapshot}; pub use crate::{
entities::group_data::{
ActiveModel as GroupDataActiveModel, Column as GroupDataColumn,
Entity as GroupDataEntity, Model as GroupDataModel,
},
snapshots::SnapshotModel,
};
/// Get all group datas from a given snapshot. impl GroupDataModel {
pub async fn get_all_by_snapshot( /// Get all group datas from a given snapshot.
db: &DatabaseConnection, pub async fn get_all_by_snapshot(
snapshot: &snapshot::Model, db: &DatabaseConnection,
) -> Result<Vec<group_data::Model>> { snapshot: &SnapshotModel,
let groups = snapshot.find_related(group_data::Entity).all(db).await?; ) -> Result<Vec<Self>> {
Ok(groups) let groups = snapshot.find_related(GroupDataEntity).all(db).await?;
Ok(groups)
}
/// Get the group with the highest subscriber count from a given snapshot.
pub async fn get_highest_subscribers(
db: &DatabaseConnection,
snapshot: &SnapshotModel,
) -> Result<Option<Self>> {
let group = snapshot
.find_related(GroupDataEntity)
.order_by_desc(GroupDataColumn::Subscribers)
.one(db)
.await?;
Ok(group)
}
/// Get the N most recently saved group datas from a given group name.
pub async fn get_n_most_recent(
db: &DatabaseConnection,
amount: u64,
name: &str,
) -> Result<Vec<Self>> {
let groups = GroupDataEntity::find()
.order_by_desc(GroupDataColumn::SnapshotId)
.filter(GroupDataColumn::Name.eq(name))
.limit(amount)
.all(db)
.await?;
Ok(groups)
}
} }

View File

@ -11,9 +11,11 @@ use {
tracing_subscriber::filter::{EnvFilter, LevelFilter}, tracing_subscriber::filter::{EnvFilter, LevelFilter},
}; };
pub mod charts;
pub mod cli; pub mod cli;
pub mod group_data; pub mod group_data;
pub mod migrations; pub mod migrations;
pub mod scss;
pub mod snapshots; pub mod snapshots;
pub mod templates; pub mod templates;
pub mod utilities; pub mod utilities;

47
source/scss/common.scss Normal file
View File

@ -0,0 +1,47 @@
html {
font-size: 62.5%;
}
body {
--small-spacing: 4px;
--medium-spacing: 8px;
--large-spacing: 16px;
--background-1: #222;
--background-2: #111;
--foreground-1: #fff;
--anchor-1: #f0f;
--anchor-2: #000;
background-color: var(--background-1);
color: var(--foreground-1);
font-size: 2rem;
}
a,
a:visited {
color: var(--anchor-1);
text-decoration: none;
&:hover {
background-color: var(--anchor-1);
color: var(--anchor-2);
}
}
h1,
h2,
h3,
p,
ol,
li {
margin: 0;
padding: 0;
}
.bold {
font-weight: bold;
}
.underline {
text-decoration: underline;
}

35
source/scss/index.scss Normal file
View File

@ -0,0 +1,35 @@
@mixin responsive-container($breakpoint) {
margin-left: auto;
margin-right: auto;
width: $breakpoint;
@media (max-width: $breakpoint) {
width: 100%;
}
}
.page-header,
.page-main,
.page-footer {
@include responsive-container(1200px);
margin-bottom: var(--large-spacing);
padding: var(--large-spacing);
}
.page-header {
border-bottom: 4px solid var(--background-2);
}
.page-main {
h2 {
margin-bottom: var(--medium-spacing);
}
}
.page-footer {
background-color: var(--background-2);
display: flex;
flex-direction: column;
gap: var(--large-spacing);
}

40
source/scss/mod.rs Normal file
View File

@ -0,0 +1,40 @@
//! All SCSS files.
use {
async_std::{
fs::{create_dir_all, write},
path::PathBuf,
},
color_eyre::{eyre::Context, Result},
};
const MODERN_NORMALIZE_CSS: &str =
include_str!("../../node_modules/modern-normalize/modern-normalize.css");
/// Generate the CSS files and write them.
pub async fn generate_css(parent: &PathBuf) -> Result<()> {
let parent = parent.join("css");
create_dir_all(&parent).await?;
let render = |scss: &str| -> Result<String> {
grass::from_string(scss.to_string(), &grass::Options::default())
.wrap_err("Failed SCSS render")
};
let css_to_create = vec![
("modern-normalize.css", MODERN_NORMALIZE_CSS, false),
("common.css", include_str!("common.scss"), true),
("index.css", include_str!("index.scss"), true),
];
for (file, css, is_scss) in css_to_create {
let path = parent.join(file);
if is_scss {
write(path, render(css)?).await?;
} else {
write(path, css).await?;
}
}
Ok(())
}

View File

@ -8,69 +8,71 @@ use {
}; };
use crate::{ use crate::{
entities::{group_data, snapshot}, group_data::{GroupDataActiveModel, GroupDataEntity},
snapshots::get_by_date, snapshots::{SnapshotActiveModel, SnapshotModel},
utilities::{create_http_client, download_html, today}, utilities::{create_http_client, download_html, today},
}; };
/// Create a snapshot for today. impl SnapshotModel {
pub async fn create(db: &DatabaseConnection, force: bool) -> Result<()> { /// Create a snapshot for today.
let snapshot_date = today(); pub async fn create(db: &DatabaseConnection, force: bool) -> Result<()> {
match (force, get_by_date(db, snapshot_date).await?) { let snapshot_date = today();
(true, Some(existing)) => { match (force, Self::get_by_date(db, snapshot_date).await?) {
info!("Removing existing snapshot {:?}", existing); (true, Some(existing)) => {
existing.delete(db).await?; info!("Removing existing snapshot {:?}", existing);
} existing.delete(db).await?;
}
(false, Some(existing)) => { (false, Some(existing)) => {
info!("Snapshot for today already exists"); info!("Snapshot for today already exists");
info!("Use --force to override snapshot {:?}", existing); info!("Use --force to override snapshot {:?}", existing);
return Ok(()); return Ok(());
} }
(_, None) => (), (_, None) => (),
}; };
let transaction = db.begin().await?; let transaction = db.begin().await?;
let snapshot = snapshot::ActiveModel { let snapshot = SnapshotActiveModel {
date: Set(snapshot_date), date: Set(snapshot_date),
..Default::default()
}
.insert(&transaction)
.await?;
info!("Scraping data for snapshot {:?}", snapshot);
let http = create_http_client()?;
let group_list = GroupList::from_html(
&download_html(&http, "https://tildes.net/groups").await?,
)?;
let mut groups_to_insert = vec![];
for summary in group_list.summaries {
debug!(summary = ?summary);
let group = Group::from_html(
&download_html(&http, format!("https://tildes.net/{}", summary.name))
.await?,
)?;
debug!(group = ?group);
groups_to_insert.push(group_data::ActiveModel {
description: Set(group.description),
name: Set(group.name),
snapshot_id: Set(snapshot.id),
subscribers: Set(group.subscribers.into()),
..Default::default() ..Default::default()
}); }
} .insert(&transaction)
info!("Inserting {} groups", groups_to_insert.len());
group_data::Entity::insert_many(groups_to_insert)
.exec(&transaction)
.await?; .await?;
transaction.commit().await?; info!("Scraping data for snapshot {:?}", snapshot);
Ok(()) let http = create_http_client()?;
let group_list = GroupList::from_html(
&download_html(&http, "https://tildes.net/groups").await?,
)?;
let mut groups_to_insert = vec![];
for summary in group_list.summaries {
debug!(summary = ?summary);
let group = Group::from_html(
&download_html(&http, format!("https://tildes.net/{}", summary.name))
.await?,
)?;
debug!(group = ?group);
groups_to_insert.push(GroupDataActiveModel {
description: Set(group.description),
name: Set(group.name),
snapshot_id: Set(snapshot.id),
subscribers: Set(group.subscribers.into()),
..Default::default()
});
}
info!("Inserting {} groups", groups_to_insert.len());
GroupDataEntity::insert_many(groups_to_insert)
.exec(&transaction)
.await?;
transaction.commit().await?;
Ok(())
}
} }

View File

@ -5,29 +5,44 @@ use {
sea_orm::{prelude::*, QueryOrder}, sea_orm::{prelude::*, QueryOrder},
}; };
use crate::entities::snapshot;
mod create; mod create;
pub use create::create; pub use crate::entities::snapshot::{
ActiveModel as SnapshotActiveModel, Column as SnapshotColumn,
Entity as SnapshotEntity, Model as SnapshotModel,
};
/// Get a snapshot for a given date. impl SnapshotModel {
pub async fn get_by_date( /// Get a snapshot for a given date.
db: &DatabaseConnection, pub async fn get_by_date(
date: ChronoDate, db: &DatabaseConnection,
) -> Result<Option<snapshot::Model>> { date: ChronoDate,
let existing = snapshot::Entity::find() ) -> Result<Option<Self>> {
.filter(snapshot::Column::Date.eq(date)) let existing = SnapshotEntity::find()
.order_by_desc(snapshot::Column::Date) .filter(SnapshotColumn::Date.eq(date))
.one(db) .order_by_desc(SnapshotColumn::Date)
.await?; .one(db)
.await?;
Ok(existing) Ok(existing)
} }
/// Get all snapshots. /// Get all snapshots.
pub async fn get_all(db: &DatabaseConnection) -> Result<Vec<snapshot::Model>> { pub async fn get_all(db: &DatabaseConnection) -> Result<Vec<Self>> {
let snapshots = snapshot::Entity::find().all(db).await?; let snapshots = SnapshotEntity::find().all(db).await?;
Ok(snapshots) Ok(snapshots)
}
/// Get the most recent snapshot.
pub async fn get_most_recent(
db: &DatabaseConnection,
) -> Result<Option<Self>> {
let snapshot = SnapshotEntity::find()
.order_by_desc(SnapshotColumn::Date)
.one(db)
.await?;
Ok(snapshot)
}
} }

View File

@ -7,6 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>{{ page_title }}</title> <title>{{ page_title }}</title>
<link rel="stylesheet" href="/css/modern-normalize.css"> <link rel="stylesheet" href="/css/modern-normalize.css">
<link rel="stylesheet" href="/css/common.css">
{% block head %}{% endblock %} {% block head %}{% endblock %}
</head> </head>

View File

@ -0,0 +1,43 @@
{% extends "base.html" %}
{% block head %}
<link rel="stylesheet" href="/css/index.css">
{% endblock %}
{% block body %}
<header class="page-header">
<h1>Tildes Statistics</h1>
</header>
<main class="page-main">
<h2>General</h2>
<p>
There are currently an
<abbr title="Based on the group with the highest subscriber count.">estimated</abbr>
<span class="underline">{{ user_count }}</span>
registered users on Tildes.
</p>
<img src="/charts/user-count.svg" alt="User Count Chart">
</main>
<footer class="page-footer">
<p>
Last generated on
<time class="underline" datetime="{{ today }}">{{- today -}}</time>.
</p>
<p>
&copy; Code
<a href="https://git.bauke.xyz/Bauke/tildes-statistics">AGPL-3.0-or-later</a>,
charts & data
<a href="https://creativecommons.org/licenses/by-nc/4.0/">CC BY-NC 4.0</a>.
</p>
<p class="bold">
Consider joining <a href="https://tildes.net">Tildes</a>, a non-profit
community site driven by its users' interests.
</p>
</footer>
{% endblock %}

View File

@ -1 +1,44 @@
//! All HTML templates. //! All HTML templates.
use {
askama::Template,
async_std::{fs::write, path::PathBuf},
chrono::NaiveDate,
color_eyre::Result,
};
use crate::utilities::today;
/// The template for the home page.
#[derive(Template)]
#[template(path = "index.html")]
pub struct HomeTemplate {
/// The string for the `<title>` element.
pub page_title: String,
/// The date of today's snapshot.
pub today: NaiveDate,
/// The user count from the group with the most subscribers.
pub user_count: String,
}
impl HomeTemplate {
/// Create a new [`HomeTemplate`].
pub fn new(user_count: Option<i64>) -> Self {
Self {
page_title: "Tildes Statistics".to_string(),
today: today(),
user_count: user_count
.map(|n| n.to_string())
.unwrap_or_else(|| "unknown".to_string()),
}
}
/// Render the template and write it to file.
pub async fn render_to_file(&self, parent: &PathBuf) -> Result<()> {
write(parent.join("index.html"), self.render()?).await?;
Ok(())
}
}