add basic parallelization to site building

Cargo.lock

@@ -298,6 +298,25 @@ dependencies = [
  "crossbeam-utils",
 ]
 
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+dependencies = [
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "crossbeam-utils"
 version = "0.8.20"
@@ -1640,6 +1659,26 @@ dependencies = [
  "rand_core 0.5.1",
 ]
 
+[[package]]
+name = "rayon"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.5.7"
@@ -2551,6 +2590,7 @@ dependencies = [
  "minifier",
  "percent-encoding",
  "pulldown-cmark",
+ "rayon",
  "rss",
  "serde",
  "serde_yml",

Cargo.toml

@@ -21,6 +21,7 @@ pulldown-cmark = {version = "0.12", default-features = false, features = [
   "simd",
   "html",
 ]}
+rayon = "1"
 rss = {version = "2", features = ["validation"]}
 serde = {version = "1", features = ["derive"]}
 serde_yml = "0.0.12"

src/builder.rs

@@ -117,16 +117,32 @@ impl<'a> SiteBuilder<'a> {
 			std::fs::create_dir(images_path).wrap_err("Failed to create images path")?;
 		}
 
-		self.images_builder
+		self.reload_images_builder()?;
-			.load_all(&self)
+		self.reload_blog_builder()?;
-			.wrap_err("Failed to load images metadata")?;
-		self.blog_builder
-			.load_all(&self)
-			.wrap_err("Failed to load blog metadata")?;
 
 		Ok(self)
 	}
 
+	/// Reloads the images builder's metadata.
+	pub fn reload_images_builder(&mut self) -> eyre::Result<()> {
+		let mut images_builder = std::mem::take(&mut self.images_builder);
+		images_builder
+			.load_all(self)
+			.wrap_err("Failed to load images metadata")?;
+		self.images_builder = images_builder;
+		Ok(())
+	}
+
+	/// Reloads the blog builder's metadata.
+	pub fn reload_blog_builder(&mut self) -> eyre::Result<()> {
+		let mut blog_builder = std::mem::take(&mut self.blog_builder);
+		blog_builder
+			.load_all(self)
+			.wrap_err("Failed to load blog metadata")?;
+		self.blog_builder = blog_builder;
+		Ok(())
+	}
+
 	/// Function to rewrite HTML wow.
 	pub fn rewrite_html(&self, html: String) -> eyre::Result<String> {
 		let mut output = Vec::new();

src/extras.rs

@@ -52,12 +52,12 @@ fn index(page: String, builder: &SiteBuilder) -> eyre::Result<String> {
 		resources: Vec<ResourceTemplateData<'r, BlogPostMetadata, ()>>,
 	}
 
-	let lmd = builder.blog_builder.loaded_metadata.borrow();
-
 	let sidebar = builder.reg.render(
 		"extras/index-injection",
 		&SidebarTemplateData {
-			resources: lmd
+			resources: builder
+				.blog_builder
+				.loaded_metadata
 				.iter()
 				.take(3)
 				.map(|(id, v)| ResourceTemplateData {

src/lib.rs

@@ -14,6 +14,7 @@ use std::{
 };
 
 use eyre::Context;
+use rayon::prelude::*;
 use resource::EmbedMetadata;
 use serde::Deserialize;
 use url::Url;
@@ -162,9 +163,10 @@ impl Site {
 
 	/// Helper method to build all available pages.
 	fn build_all_pages(&self, builder: &SiteBuilder) -> eyre::Result<()> {
-		for page_name in self.page_index.keys() {
+		self.page_index
-			builder.build_page(page_name)?;
+			.keys()
-		}
+			.par_bridge()
+			.try_for_each(|page_name| builder.build_page(page_name))?;
 
 		Ok(())
 	}

src/resource.rs

@@ -1,5 +1,4 @@
 use std::{
-	cell::RefCell,
 	collections::BTreeMap,
 	marker::PhantomData,
 	path::{Path, PathBuf},
@@ -146,7 +145,7 @@ struct ExtraResourceRenderData {
 }
 
 /// Config for the resource builder.
-#[derive(Debug)]
+#[derive(Debug, Default)]
 pub struct ResourceBuilderConfig {
 	/// Path to where the resources should be loaded from.
 	pub source_path: String,
@@ -180,7 +179,7 @@ pub struct ResourceBuilder<M, E> {
 	/// The builder's config.
 	pub config: ResourceBuilderConfig,
 	/// The currently loaded resource metadata.
-	pub loaded_metadata: RefCell<Vec<(String, ResourceMetadata<M>)>>,
+	pub loaded_metadata: Vec<(String, ResourceMetadata<M>)>,
 	_extra: PhantomData<E>,
 }
 
@@ -228,8 +227,8 @@ where
 	}
 
 	/// Loads all resource metadata from the given config.
-	pub fn load_all(&self, builder: &SiteBuilder) -> eyre::Result<()> {
+	pub fn load_all(&mut self, builder: &SiteBuilder) -> eyre::Result<()> {
-		let mut lmd = self.loaded_metadata.borrow_mut();
+		let lmd = &mut self.loaded_metadata;
 		lmd.clear();
 		for e in builder
 			.site
@@ -302,7 +301,7 @@ where
 			std::fs::create_dir_all(&out_long)?;
 		}
 
-		let lmd = self.loaded_metadata.borrow();
+		let lmd = &self.loaded_metadata;
 
 		for (id, resource) in lmd.iter() {
 			self.build(builder, id.clone(), resource)?;
@@ -473,3 +472,13 @@ where
 		Ok(())
 	}
 }
+
+impl<M, E> Default for ResourceBuilder<M, E> {
+	fn default() -> Self {
+		Self {
+			config: Default::default(),
+			loaded_metadata: Default::default(),
+			_extra: Default::default(),
+		}
+	}
+}

src/serving.rs

@@ -72,11 +72,11 @@ fn create(
 		std::fs::copy(path, builder.build_path.join(root_path))?;
 	} else if let Ok(_image_path) = relative_path.strip_prefix(crate::images::IMAGES_PATH) {
 		// HACK: this could get very inefficient with a larger number of images. should definitely optimize
-		builder.images_builder.load_all(builder)?;
+		builder.reload_images_builder()?;
 		builder.build_images()?;
 	} else if let Ok(_blog_path) = relative_path.strip_prefix(crate::blog::BLOG_PATH) {
 		// HACK: same as above
-		builder.blog_builder.load_all(builder)?;
+		builder.reload_blog_builder()?;
 		builder.build_blog()?;
 	}
 
@@ -108,11 +108,11 @@ fn remove(builder: &mut SiteBuilder, path: &Path, relative_path: &Path) -> eyre:
 		std::fs::remove_file(builder.build_path.join(root_path))?;
 	} else if let Ok(_image_path) = relative_path.strip_prefix(crate::images::IMAGES_PATH) {
 		// HACK: same as in `create`
-		builder.images_builder.load_all(builder)?;
+		builder.reload_images_builder()?;
 		builder.build_images()?;
 	} else if let Ok(_blog_path) = relative_path.strip_prefix(crate::blog::BLOG_PATH) {
 		// HACK: same as above
-		builder.blog_builder.load_all(builder)?;
+		builder.reload_blog_builder()?;
 		builder.build_blog()?;
 	}