.gitignore

@@ -1,9 +1,8 @@
-# Generated by Cargo
+# Compiled files and executables.
-debug/
+/target/
-target/
 
-# Code coverage results
+# Backup files generated by rustfmt.
-coverage/
+**/*.rs.bk
 
-# Markdown output directory
+# The actual Sitemap, to be copied to https://tildes.net/~tildes/wiki/sitemap.
-output/
+sitemap.md

Cargo.toml

@@ -2,24 +2,17 @@
 
 [package]
 name = "tildes-wiki-sitemap"
-description = "Tildes Wiki sitemap generator."
+version = "0.1.0"
-repository = "https://git.bauke.xyz/Bauke/tildes-parser"
-license = "AGPL-3.0-or-later"
-version = "0.2.0"
 authors = ["Bauke <me@bauke.xyz>"]
-edition = "2021"
+edition = "2018"
 
 [[bin]]
 name = "tildes-wiki-sitemap"
 path = "source/main.rs"
 
 [dependencies]
-askama = "0.11.1"
+scraper = "0.12.0"
-color-eyre = "0.6.2"
-indicatif = "0.17.1"
-regex = "1.6.0"
-ureq = "2.5.0"
 
-[dependencies.tildes-parser]
+[dependencies.reqwest]
-git = "https://git.bauke.xyz/Bauke/tildes-parser.git"
+version = "0.10.7"
-rev = "08bf7ed"
+features = ["blocking"]

Makefile.toml

@@ -1,39 +0,0 @@
-[tasks.fmt]
-command = "cargo"
-args = ["fmt", "${@}"]
-
-[tasks.check]
-command = "cargo"
-args = ["check", "${@}"]
-
-[tasks.clippy]
-command = "cargo"
-args = ["clippy", "${@}"]
-
-[tasks.test]
-command = "cargo"
-args = ["test", "${@}"]
-
-[tasks.doc]
-command = "cargo"
-args = ["doc", "${@}"]
-
-[tasks.build]
-command = "cargo"
-args = ["build", "${@}"]
-
-[tasks.complete-check]
-dependencies = ["fmt", "check", "clippy", "test", "doc", "build"]
-
-[tasks.code-coverage]
-workspace = false
-install_crate = "cargo-tarpaulin"
-command = "cargo"
-args = [
-  "tarpaulin",
-  "--exclude-files=target/*",
-  "--out=html",
-  "--output-dir=coverage",
-  "--skip-clean",
-  "--target-dir=target/tarpaulin"
-]

README.md

@@ -1,17 +1,38 @@
-# Tildes 📍 Wiki Sitemap
+# Tildes Wiki Sitemap
 
-> **Tildes Wiki sitemap generator.**
+> Generates a Markdown file with all group wiki pages of Tildes.
 
 ## Installation
 
-### Binaries
+### Binary
 
-Precompiled `x86_64-unknown-linux-gnu` binaries are available on the [Releases page](https://git.bauke.xyz/Bauke/tildes-wiki-sitemap/releases).
+Precompiled binaries are available [here](https://git.holllo.cc/Bauke/tildes-wiki-sitemap/releases).
 
-## Feedback
+### Source
 
-Found a problem or want to request a new feature? Message [@Bauke](https://tildes.net/user/Bauke/new_message) on Tildes or email [me@bauke.xyz](mailto:me@bauke.xyz) and I'll see what I can do for you.
+Requires [Rust and Cargo](https://www.rust-lang.org/tools/install) to be installed.
+
+```sh
+git clone https://git.holllo.cc/Bauke/tildes-wiki-sitemap.git
+cd tildes-wiki-sitemap
+cargo build --release
+mv target/release/tildes-wiki-sitemap ./
+```
+
+## Usage
+
+Make sure the file is executable, then run it. A `sitemap.md` file will be created with the results.
+
+```sh
+chmod +x ./tildes-wiki-sitemap
+./tildes-wiki-sitemap
+less sitemap.md
+```
+
+## Previous Version
+
+If you're looking for the previous version of this program written in Go, [click here](https://git.holllo.cc/Bauke/tildes-wiki-sitemap/src/commit/18a96e9d541fd1e231574ceec4d4bdf5783e3b5f) to go to the commit before the Rust rewrite.
 
 ## License
 
-Distributed under the [AGPL-3.0-or-later](https://spdx.org/licenses/AGPL-3.0-or-later.html) license, see [LICENSE](https://git.bauke.xyz/Bauke/tildes-wiki-sitemap/src/branch/main/LICENSE) for more information.
+Open-sourced with the [AGPL-3.0-or-later license](https://git.holllo.cc/Bauke/tildes-wiki-sitemap/src/branch/main/LICENSE).

askama.toml

@@ -1,2 +0,0 @@
-[general]
-dirs = ["source/templates"]

source/main.rs

@@ -1,75 +1,97 @@
-//! # Tildes Wiki Sitemap
+use std::{error::Error, fs, thread, time::Duration};
-//!
-//! > **Tildes Wiki sitemap generator.**
 
-#![forbid(unsafe_code)]
+use reqwest::blocking::Client;
-#![warn(missing_docs, clippy::missing_docs_in_private_items)]
+use scraper::{ElementRef, Html, Selector};
 
-use {
+fn main() -> Result<(), Box<dyn Error>> {
-  askama::Template,
+  let client = Client::builder()
-  color_eyre::Result,
-  indicatif::{ProgressIterator, ProgressStyle},
-  regex::Regex,
-  tildes_parser::{Group, GroupList, Html},
-};
-
-mod templates;
-
-fn main() -> Result<()> {
-  color_eyre::install()?;
-
-  let http = ureq::AgentBuilder::new()
     .user_agent("Tildes Wiki Sitemap")
-    .build();
+    .build()?;
 
-  // Shorthand to download a URL and parse it to `Html`.
+  // Get the HTML from the groups list.
-  let download_html = |url: &str| -> Result<Html> {
+  let response = client.get("https://tildes.net/groups").send()?;
-    Ok(Html::parse_document(&http.get(url).call()?.into_string()?))
+  let body = response.text()?;
-  };
 
-  let group_list =
+  // Parse the HTML.
-    GroupList::from_html(&download_html("https://tildes.net/groups")?)?;
+  let html = Html::parse_document(&body);
 
-  let groups = group_list
+  // Create a selector to grab all anchors that link to a group.
-    .summaries
+  let selector = Selector::parse(".group-list .link-group").unwrap();
-    .into_iter()
-    .progress_with_style(ProgressStyle::with_template(
-      "{spinner} {pos}/{len} {bar}",
-    )?)
-    .map(|summary| {
-      // Sleep 500 milliseconds between HTTP requests.
-      std::thread::sleep(std::time::Duration::from_millis(500));
 
-      Group::from_html(&download_html(&format!(
+  // Get all the group link elements from the HTML.
-        "https://tildes.net/{}",
+  let group_links = html.select(&selector).collect::<Vec<ElementRef>>();
-        summary.name
-      ))?)
-    })
-    .collect::<Result<Vec<_>>>()?;
 
-  let wiki_link_count = groups
+  // Create the sitemap with the info.
-    .iter()
+  let mut sitemap = "# Tildes Wiki Sitemap\n\n".to_string();
-    .map(|group| group.wiki_links.len())
+  sitemap += "Automatically generated by \
-    .sum::<usize>();
+  [this program](https://git.holllo.cc/Bauke/tildes-wiki-sitemap). \
+  [message @Bauke](https://tildes.net/user/Bauke/new_message?subject=Tildes%20Wiki%20Sitemap\
+  &message=Update%20the%20sitemap%20you%20doofus!) if this page is outdated and \
+  you can't run the program yourself.\n\n\
+  This page is a temporary placeholder to help wiki contributors navigate. \
+  Find this page easily by bookmarking it!\n\n";
 
-  println!(
+  for group_link in group_links {
-    "Collected {} groups and {} wiki links.",
+    // Get the group name without the tilde.
-    groups.len(),
+    let group_name = group_link.inner_html()[1..].to_string();
-    wiki_link_count,
+    println!("┌ Processing ~{}!", group_name);
-  );
 
-  std::fs::create_dir_all("output")?;
+    // Get the HTML from the group page.
-  std::fs::write("output/sitemap.md", render_sitemap(groups)?)?;
+    let response = client
+      .get(&format!("https://tildes.net/~{}", group_name))
+      .send()?;
+    let body = response.text()?;
 
+    // Parse the HTML.
+    let html = Html::parse_document(&body);
+
+    // Create a selector to grab all the anchors in the sidebar that lead to a wiki page.
+    let selector =
+      Selector::parse("#sidebar .nav a[href*=\"/wiki/\"]").unwrap();
+
+    // Get all the wiki URL elements from the HTML.
+    let wiki_links = html.select(&selector).collect::<Vec<ElementRef>>();
+    let wiki_links_amount = wiki_links.len();
+
+    // Create a selector to grab the group description.
+    let selector =
+      Selector::parse("#sidebar .group-short-description").unwrap();
+
+    // Get the group description from the HTML.
+    let group_description = html.select(&selector).collect::<Vec<ElementRef>>();
+
+    // Add the group as a new header.
+    sitemap += format!("## ~{}\n\n", group_name).as_str();
+
+    // If a group description is found, add it to the Markdown.
+    if !group_description.is_empty() {
+      let description = group_description.first().unwrap().inner_html();
+      sitemap += format!("> {}\n\n", description).as_str();
+    }
+
+    // If there's no wiki pages, add a little blurb with a link to create one.
+    if wiki_links.is_empty() {
+      sitemap += format!("There are no wiki pages for ~{} yet, \
+      [click here and be the first to create one](https://tildes.net/~{}/wiki/new_page), \
+      if you were granted the necessary permission to do so!\n", group_name, group_name).as_str();
+    }
+
+    // Loop over the links and add them in a list.
+    for wiki_link in wiki_links {
+      let wiki_page_title = wiki_link.inner_html();
+      let wiki_page_link = wiki_link.value().attr("href").unwrap_or("");
+      sitemap += &format!("* [{}]({})\n", wiki_page_title, wiki_page_link);
+    }
+
+    sitemap += "\n";
+    println!("└ Processed {} wiki links.", wiki_links_amount);
+
+    // Sleep 500ms between HTTP requests.
+    thread::sleep(Duration::from_millis(500));
+  }
+
+  sitemap = sitemap.trim_end().to_string() + "\n";
+  fs::write("./sitemap.md", sitemap)?;
+  println!("✓ Done!");
   Ok(())
 }
-
-fn render_sitemap(groups: Vec<Group>) -> Result<String> {
-  let duplicate_newline_re = Regex::new("\n\n\n+").unwrap();
-
-  Ok(
-    duplicate_newline_re
-      .replace_all(&templates::SitemapTemplate { groups }.render()?, "\n\n")
-      .to_string(),
-  )
-}

source/templates/mod.rs

@@ -1,77 +0,0 @@
-//! Askama templates.
-
-use {askama::Template, tildes_parser::Group};
-
-/// The template for `sitemap.md`.
-#[derive(Debug, Template)]
-#[template(path = "sitemap.md")]
-pub struct SitemapTemplate {
-  /// All groups to render in the sitemap.
-  pub groups: Vec<Group>,
-}
-
-#[test]
-fn test_sitemap_template() -> color_eyre::Result<()> {
-  let groups = vec![
-    Group {
-      description: Some("Example group description.".to_string()),
-      name: "~example".to_string(),
-      sub_groups: vec![],
-      subscribers: 12345,
-      wiki_links: vec![
-        tildes_parser::GroupWikiLink {
-          name: "Example Page".to_string(),
-          url: "https://example.org/~example/wiki/example_page".to_string(),
-        },
-        tildes_parser::GroupWikiLink {
-          name: "Example Page".to_string(),
-          url: "https://example.org/~example/wiki/example_page".to_string(),
-        },
-      ],
-    },
-    Group {
-      description: Some("Example group description.".to_string()),
-      name: "~example".to_string(),
-      sub_groups: vec![],
-      subscribers: 12345,
-      wiki_links: vec![
-        tildes_parser::GroupWikiLink {
-          name: "Example Page".to_string(),
-          url: "https://example.org/~example/wiki/example_page".to_string(),
-        },
-        tildes_parser::GroupWikiLink {
-          name: "Example Page".to_string(),
-          url: "https://example.org/~example/wiki/example_page".to_string(),
-        },
-      ],
-    },
-    Group {
-      description: None,
-      name: "~example".to_string(),
-      sub_groups: vec![],
-      subscribers: 12345,
-      wiki_links: vec![],
-    },
-    Group {
-      description: None,
-      name: "~example".to_string(),
-      sub_groups: vec![],
-      subscribers: 12345,
-      wiki_links: vec![
-        tildes_parser::GroupWikiLink {
-          name: "Example Page".to_string(),
-          url: "https://example.org/~example/wiki/example_page".to_string(),
-        },
-        tildes_parser::GroupWikiLink {
-          name: "Example Page".to_string(),
-          url: "https://example.org/~example/wiki/example_page".to_string(),
-        },
-      ],
-    },
-  ];
-
-  std::fs::create_dir_all("output")?;
-  std::fs::write("output/sitemap-test.md", crate::render_sitemap(groups)?)?;
-
-  Ok(())
-}

source/templates/sitemap.md

@@ -1,27 +0,0 @@
-# Tildes Wiki Sitemap
-
-This page is a temporary placeholder to help wiki contributors navigate. Find this page easily by bookmarking it!
-
-<details>
-<summary>Generating this page</summary>
-
-The sitemap can be automatically generated by [tildes-wiki-sitemap](https://git.bauke.xyz/Bauke/tildes-wiki-sitemap).
-
-If this page is outdated and you can't update it yourself, feel free to [message @Bauke](https://tildes.net/user/Bauke/new_message?subject=Tildes%20Wiki%20Sitemap&message=Update%20the%20sitemap%20you%20doofus!).
-</details>
-
-{% for group in groups %}
-## {{ group.name }}
-
-{% if let Some(description) = group.description -%}
-> **{{ description }}**
-{%- endif %}
-
-{% if group.wiki_links.is_empty() -%}
-There are no wiki pages for {{ group.name }} yet, [click here](https://tildes.net/{{ group.name }}/wiki/new_page) if you want to create one.
-{%- endif %}
-
-{% for link in group.wiki_links -%}
-* [{{ link.name }}]({{ link.url }})
-{% endfor -%}
-{% endfor %}