From 93e2b45fd5fca966ec477b7ec80173746010e064 Mon Sep 17 00:00:00 2001 From: Martin Vassor Date: Fri, 19 Jul 2024 11:19:06 +0100 Subject: [PATCH 1/6] Add feature to disable robots.txt and sitemap.xml from the config file. Addresses feature request #2248 --- components/config/src/config/mod.rs | 41 +++++++++++++++++++++++++++++ components/site/src/lib.rs | 12 ++++++--- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/components/config/src/config/mod.rs b/components/config/src/config/mod.rs index 57ff1a8056..467eabec1c 100644 --- a/components/config/src/config/mod.rs +++ b/components/config/src/config/mod.rs @@ -98,6 +98,10 @@ pub struct Config { pub markdown: markup::Markdown, /// All user params set in `[extra]` in the config pub extra: HashMap, + /// Disable the generation of Sitemap.xml + pub no_sitemap: bool, + /// Disable the generation of robots.txt + pub no_robots: bool, } #[derive(Serialize)] @@ -117,6 +121,8 @@ pub struct SerializedConfig<'a> { extra: &'a HashMap, markdown: &'a markup::Markdown, search: search::SerializedSearch<'a>, + no_sitemap: bool, + no_robots: bool, } impl Config { @@ -332,6 +338,8 @@ impl Config { extra: &self.extra, markdown: &self.markdown, search: self.search.serialize(), + no_sitemap: self.no_sitemap, + no_robots: self.no_robots, } } } @@ -395,6 +403,8 @@ impl Default for Config { search: search::Search::default(), markdown: markup::Markdown::default(), extra: HashMap::new(), + no_sitemap: false, + no_robots: false, } } } @@ -992,4 +1002,35 @@ feed_filename = "test.xml" Config::parse(config).unwrap(); } + + #[test] + fn parse_no_sitemap() { + let config = r#" +title = "My Site" +base_url = "example.com" +no_sitemap = true +"#; + let config = Config::parse(config).unwrap(); + assert!(config.no_sitemap); + } + + #[test] + fn default_no_sitemap_false() { + let config = r#" +title = "My Site" +base_url = "example.com" +"#; + let config = Config::parse(config).unwrap(); + assert!(!config.no_sitemap); + } + + #[test] + fn default_no_robots_false() { + let config = r#" +title = "My Site" +base_url = "example.com" +"#; + let config = Config::parse(config).unwrap(); + assert!(!config.no_robots); + } } diff --git a/components/site/src/lib.rs b/components/site/src/lib.rs index 46c62c9598..27a332dc38 100644 --- a/components/site/src/lib.rs +++ b/components/site/src/lib.rs @@ -742,8 +742,10 @@ impl Site { start = log_time(start, "Rendered sections"); self.render_orphan_pages()?; start = log_time(start, "Rendered orphan pages"); - self.render_sitemap()?; - start = log_time(start, "Rendered sitemap"); + if !self.config.no_sitemap { + self.render_sitemap()?; + start = log_time(start, "Rendered sitemap"); + } let library = self.library.read().unwrap(); if self.config.generate_feeds { @@ -769,8 +771,10 @@ impl Site { start = log_time(start, "Rendered themes css"); self.render_404()?; start = log_time(start, "Rendered 404"); - self.render_robots()?; - start = log_time(start, "Rendered robots.txt"); + if !self.config.no_robots { + self.render_robots()?; + start = log_time(start, "Rendered robots.txt"); + } self.render_taxonomies()?; start = log_time(start, "Rendered taxonomies"); // We process images at the end as we might have picked up images to process from markdown From 7834870aab4c86a86a283e39f32b328c11551560 Mon Sep 17 00:00:00 2001 From: Martin Vassor Date: Fri, 19 Jul 2024 11:29:18 +0100 Subject: [PATCH 2/6] Add documentation for no_sitemap & no_robots --- docs/content/documentation/getting-started/configuration.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/content/documentation/getting-started/configuration.md b/docs/content/documentation/getting-started/configuration.md index 103ae07300..5cddfd8619 100644 --- a/docs/content/documentation/getting-started/configuration.md +++ b/docs/content/documentation/getting-started/configuration.md @@ -100,6 +100,12 @@ taxonomies = [] # content for `default_language`. build_search_index = false +# When set to "true", Sitemap.xml is not generated (default: false) +no_sitemap = false + +# When set to "true", robots.txt is not generated (default: false) +no_robots = false + # Configuration of the Markdown rendering [markdown] # When set to "true", all code blocks are highlighted. From 63ef902ad8f5ecfc849137f24a98dddf5bed055a Mon Sep 17 00:00:00 2001 From: Martin Vassor Date: Sun, 21 Jul 2024 22:00:30 +0100 Subject: [PATCH 3/6] Rename no_robots and no_sitemap into generate_robots_txt and generate_sitemap (default to true) --- components/config/src/config/mod.rs | 64 ++++++++++++++----- components/site/src/lib.rs | 4 +- .../getting-started/configuration.md | 8 +-- 3 files changed, 55 insertions(+), 21 deletions(-) diff --git a/components/config/src/config/mod.rs b/components/config/src/config/mod.rs index 467eabec1c..5581d1a483 100644 --- a/components/config/src/config/mod.rs +++ b/components/config/src/config/mod.rs @@ -99,9 +99,9 @@ pub struct Config { /// All user params set in `[extra]` in the config pub extra: HashMap, /// Disable the generation of Sitemap.xml - pub no_sitemap: bool, + pub generate_sitemap: bool, /// Disable the generation of robots.txt - pub no_robots: bool, + pub generate_robots_txt: bool, } #[derive(Serialize)] @@ -121,8 +121,8 @@ pub struct SerializedConfig<'a> { extra: &'a HashMap, markdown: &'a markup::Markdown, search: search::SerializedSearch<'a>, - no_sitemap: bool, - no_robots: bool, + generate_sitemap: bool, + generate_robots_txt: bool, } impl Config { @@ -338,8 +338,8 @@ impl Config { extra: &self.extra, markdown: &self.markdown, search: self.search.serialize(), - no_sitemap: self.no_sitemap, - no_robots: self.no_robots, + generate_sitemap: self.generate_sitemap, + generate_robots_txt: self.generate_robots_txt, } } } @@ -403,8 +403,8 @@ impl Default for Config { search: search::Search::default(), markdown: markup::Markdown::default(), extra: HashMap::new(), - no_sitemap: false, - no_robots: false, + generate_sitemap: true, + generate_robots_txt: true, } } } @@ -1004,33 +1004,67 @@ feed_filename = "test.xml" } #[test] - fn parse_no_sitemap() { + fn parse_generate_sitemap_true() { let config = r#" title = "My Site" base_url = "example.com" -no_sitemap = true +generate_sitemap = true "#; let config = Config::parse(config).unwrap(); - assert!(config.no_sitemap); + assert!(config.generate_sitemap); } #[test] - fn default_no_sitemap_false() { + fn parse_generate_sitemap_false() { let config = r#" title = "My Site" base_url = "example.com" +generate_sitemap = false "#; let config = Config::parse(config).unwrap(); - assert!(!config.no_sitemap); + assert!(!config.generate_sitemap); } #[test] - fn default_no_robots_false() { + fn default_no_sitemap_true() { let config = r#" title = "My Site" base_url = "example.com" "#; let config = Config::parse(config).unwrap(); - assert!(!config.no_robots); + assert!(config.generate_sitemap); + } + + #[test] + fn parse_generate_robots_true() { + let config = r#" +title = "My Site" +base_url = "example.com" +generate_robots_txt = true +"#; + let config = Config::parse(config).unwrap(); + assert!(config.generate_robots_txt); + } + + #[test] + fn parse_generate_robots_false() { + let config = r#" +title = "My Site" +base_url = "example.com" +generate_robots_txt = false +"#; + let config = Config::parse(config).unwrap(); + assert!(!config.generate_robots_txt); + } + + + #[test] + fn default_no_robots_true() { + let config = r#" +title = "My Site" +base_url = "example.com" +"#; + let config = Config::parse(config).unwrap(); + assert!(config.generate_robots_txt); } } diff --git a/components/site/src/lib.rs b/components/site/src/lib.rs index 27a332dc38..edb4f4e7e2 100644 --- a/components/site/src/lib.rs +++ b/components/site/src/lib.rs @@ -742,7 +742,7 @@ impl Site { start = log_time(start, "Rendered sections"); self.render_orphan_pages()?; start = log_time(start, "Rendered orphan pages"); - if !self.config.no_sitemap { + if self.config.generate_sitemap{ self.render_sitemap()?; start = log_time(start, "Rendered sitemap"); } @@ -771,7 +771,7 @@ impl Site { start = log_time(start, "Rendered themes css"); self.render_404()?; start = log_time(start, "Rendered 404"); - if !self.config.no_robots { + if self.config.generate_robots_txt { self.render_robots()?; start = log_time(start, "Rendered robots.txt"); } diff --git a/docs/content/documentation/getting-started/configuration.md b/docs/content/documentation/getting-started/configuration.md index 5cddfd8619..95eef65a86 100644 --- a/docs/content/documentation/getting-started/configuration.md +++ b/docs/content/documentation/getting-started/configuration.md @@ -100,11 +100,11 @@ taxonomies = [] # content for `default_language`. build_search_index = false -# When set to "true", Sitemap.xml is not generated (default: false) -no_sitemap = false +# When set to "false", Sitemap.xml is not generated (default: true) +generate_sitemap = false -# When set to "true", robots.txt is not generated (default: false) -no_robots = false +# When set to "false", robots.txt is not generated (default: true) +generate_robots_txt = false # Configuration of the Markdown rendering [markdown] From 52b61bc9961228a75f7eb381dc827aa91db73b36 Mon Sep 17 00:00:00 2001 From: Martin Vassor Date: Sun, 21 Jul 2024 22:15:52 +0100 Subject: [PATCH 4/6] fix rustfmt issues --- components/config/src/config/mod.rs | 1 - components/site/src/lib.rs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/components/config/src/config/mod.rs b/components/config/src/config/mod.rs index 5581d1a483..ee0d2813f1 100644 --- a/components/config/src/config/mod.rs +++ b/components/config/src/config/mod.rs @@ -1057,7 +1057,6 @@ generate_robots_txt = false assert!(!config.generate_robots_txt); } - #[test] fn default_no_robots_true() { let config = r#" diff --git a/components/site/src/lib.rs b/components/site/src/lib.rs index edb4f4e7e2..cd48c219e4 100644 --- a/components/site/src/lib.rs +++ b/components/site/src/lib.rs @@ -742,7 +742,7 @@ impl Site { start = log_time(start, "Rendered sections"); self.render_orphan_pages()?; start = log_time(start, "Rendered orphan pages"); - if self.config.generate_sitemap{ + if self.config.generate_sitemap { self.render_sitemap()?; start = log_time(start, "Rendered sitemap"); } From c3f85c300978441fc775e4937a039e293c6a2e78 Mon Sep 17 00:00:00 2001 From: Martin Vassor Date: Wed, 24 Jul 2024 00:21:11 +0100 Subject: [PATCH 5/6] Change documentation to show defaults --- .../documentation/getting-started/configuration.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/content/documentation/getting-started/configuration.md b/docs/content/documentation/getting-started/configuration.md index 95eef65a86..2565e0913f 100644 --- a/docs/content/documentation/getting-started/configuration.md +++ b/docs/content/documentation/getting-started/configuration.md @@ -100,11 +100,11 @@ taxonomies = [] # content for `default_language`. build_search_index = false -# When set to "false", Sitemap.xml is not generated (default: true) -generate_sitemap = false +# When set to "false", Sitemap.xml is not generated +generate_sitemap = true -# When set to "false", robots.txt is not generated (default: true) -generate_robots_txt = false +# When set to "false", robots.txt is not generated +generate_robots_txt = true # Configuration of the Markdown rendering [markdown] From 621c917b98a572ed5bff5951d388d2c89abc457f Mon Sep 17 00:00:00 2001 From: Martin Vassor Date: Wed, 24 Jul 2024 00:24:55 +0100 Subject: [PATCH 6/6] Update documentation for the fields generate_sitemaps (resp. robots_txt), now that the default is true and false is needed to disable --- components/config/src/config/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/config/src/config/mod.rs b/components/config/src/config/mod.rs index ee0d2813f1..88d949fc21 100644 --- a/components/config/src/config/mod.rs +++ b/components/config/src/config/mod.rs @@ -98,9 +98,9 @@ pub struct Config { pub markdown: markup::Markdown, /// All user params set in `[extra]` in the config pub extra: HashMap, - /// Disable the generation of Sitemap.xml + /// Enables the generation of Sitemap.xml pub generate_sitemap: bool, - /// Disable the generation of robots.txt + /// Enables the generation of robots.txt pub generate_robots_txt: bool, }