From 487129370807803b87d264480197390ef72d9bc7 Mon Sep 17 00:00:00 2001 From: zyl Date: Wed, 30 Oct 2024 14:42:43 -0700 Subject: [PATCH] add basic parallelization to site building --- Cargo.lock | 40 ++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + src/builder.rs | 28 ++++++++++++++++++++++------ src/extras.rs | 6 +++--- src/lib.rs | 8 +++++--- src/resource.rs | 21 +++++++++++++++------ src/serving.rs | 8 ++++---- 7 files changed, 90 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 00df189..2145437 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -298,6 +298,25 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.20" @@ -1640,6 +1659,26 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.7" @@ -2551,6 +2590,7 @@ dependencies = [ "minifier", "percent-encoding", "pulldown-cmark", + "rayon", "rss", "serde", "serde_yml", diff --git a/Cargo.toml b/Cargo.toml index ecdc583..13f2c63 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ pulldown-cmark = {version = "0.12", default-features = false, features = [ "simd", "html", ]} +rayon = "1" rss = {version = "2", features = ["validation"]} serde = {version = "1", features = ["derive"]} serde_yml = "0.0.12" diff --git a/src/builder.rs b/src/builder.rs index 0fa487d..8d29625 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -117,16 +117,32 @@ impl<'a> SiteBuilder<'a> { std::fs::create_dir(images_path).wrap_err("Failed to create images path")?; } - self.images_builder - .load_all(&self) - .wrap_err("Failed to load images metadata")?; - self.blog_builder - .load_all(&self) - .wrap_err("Failed to load blog metadata")?; + self.reload_images_builder()?; + self.reload_blog_builder()?; Ok(self) } + /// Reloads the images builder's metadata. + pub fn reload_images_builder(&mut self) -> eyre::Result<()> { + let mut images_builder = std::mem::take(&mut self.images_builder); + images_builder + .load_all(self) + .wrap_err("Failed to load images metadata")?; + self.images_builder = images_builder; + Ok(()) + } + + /// Reloads the blog builder's metadata. + pub fn reload_blog_builder(&mut self) -> eyre::Result<()> { + let mut blog_builder = std::mem::take(&mut self.blog_builder); + blog_builder + .load_all(self) + .wrap_err("Failed to load blog metadata")?; + self.blog_builder = blog_builder; + Ok(()) + } + /// Function to rewrite HTML wow. pub fn rewrite_html(&self, html: String) -> eyre::Result { let mut output = Vec::new(); diff --git a/src/extras.rs b/src/extras.rs index 4ec3590..9525585 100644 --- a/src/extras.rs +++ b/src/extras.rs @@ -52,12 +52,12 @@ fn index(page: String, builder: &SiteBuilder) -> eyre::Result { resources: Vec>, } - let lmd = builder.blog_builder.loaded_metadata.borrow(); - let sidebar = builder.reg.render( "extras/index-injection", &SidebarTemplateData { - resources: lmd + resources: builder + .blog_builder + .loaded_metadata .iter() .take(3) .map(|(id, v)| ResourceTemplateData { diff --git a/src/lib.rs b/src/lib.rs index 2b7703f..531d244 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,6 +14,7 @@ use std::{ }; use eyre::Context; +use rayon::prelude::*; use resource::EmbedMetadata; use serde::Deserialize; use url::Url; @@ -162,9 +163,10 @@ impl Site { /// Helper method to build all available pages. fn build_all_pages(&self, builder: &SiteBuilder) -> eyre::Result<()> { - for page_name in self.page_index.keys() { - builder.build_page(page_name)?; - } + self.page_index + .keys() + .par_bridge() + .try_for_each(|page_name| builder.build_page(page_name))?; Ok(()) } diff --git a/src/resource.rs b/src/resource.rs index f16bde4..eb87085 100644 --- a/src/resource.rs +++ b/src/resource.rs @@ -1,5 +1,4 @@ use std::{ - cell::RefCell, collections::BTreeMap, marker::PhantomData, path::{Path, PathBuf}, @@ -146,7 +145,7 @@ struct ExtraResourceRenderData { } /// Config for the resource builder. -#[derive(Debug)] +#[derive(Debug, Default)] pub struct ResourceBuilderConfig { /// Path to where the resources should be loaded from. pub source_path: String, @@ -180,7 +179,7 @@ pub struct ResourceBuilder { /// The builder's config. pub config: ResourceBuilderConfig, /// The currently loaded resource metadata. - pub loaded_metadata: RefCell)>>, + pub loaded_metadata: Vec<(String, ResourceMetadata)>, _extra: PhantomData, } @@ -228,8 +227,8 @@ where } /// Loads all resource metadata from the given config. - pub fn load_all(&self, builder: &SiteBuilder) -> eyre::Result<()> { - let mut lmd = self.loaded_metadata.borrow_mut(); + pub fn load_all(&mut self, builder: &SiteBuilder) -> eyre::Result<()> { + let lmd = &mut self.loaded_metadata; lmd.clear(); for e in builder .site @@ -302,7 +301,7 @@ where std::fs::create_dir_all(&out_long)?; } - let lmd = self.loaded_metadata.borrow(); + let lmd = &self.loaded_metadata; for (id, resource) in lmd.iter() { self.build(builder, id.clone(), resource)?; @@ -473,3 +472,13 @@ where Ok(()) } } + +impl Default for ResourceBuilder { + fn default() -> Self { + Self { + config: Default::default(), + loaded_metadata: Default::default(), + _extra: Default::default(), + } + } +} diff --git a/src/serving.rs b/src/serving.rs index d0d502f..ef44df6 100644 --- a/src/serving.rs +++ b/src/serving.rs @@ -72,11 +72,11 @@ fn create( std::fs::copy(path, builder.build_path.join(root_path))?; } else if let Ok(_image_path) = relative_path.strip_prefix(crate::images::IMAGES_PATH) { // HACK: this could get very inefficient with a larger number of images. should definitely optimize - builder.images_builder.load_all(builder)?; + builder.reload_images_builder()?; builder.build_images()?; } else if let Ok(_blog_path) = relative_path.strip_prefix(crate::blog::BLOG_PATH) { // HACK: same as above - builder.blog_builder.load_all(builder)?; + builder.reload_blog_builder()?; builder.build_blog()?; } @@ -108,11 +108,11 @@ fn remove(builder: &mut SiteBuilder, path: &Path, relative_path: &Path) -> eyre: std::fs::remove_file(builder.build_path.join(root_path))?; } else if let Ok(_image_path) = relative_path.strip_prefix(crate::images::IMAGES_PATH) { // HACK: same as in `create` - builder.images_builder.load_all(builder)?; + builder.reload_images_builder()?; builder.build_images()?; } else if let Ok(_blog_path) = relative_path.strip_prefix(crate::blog::BLOG_PATH) { // HACK: same as above - builder.blog_builder.load_all(builder)?; + builder.reload_blog_builder()?; builder.build_blog()?; }