[#155836076] update search.gov sitemap (#14)

* [#155838182] update search.gov sitemap - add noindex robots tag to /tagged/ pages, allow crawling - use jekyll-sitemap gem to generate sitemap - disallow drafts folder in robots.txt - remove old drafts - update sample.md to prevent drafts from being indexed - remove pinterest verification page - use search.gov instead of www - add pry * squashme: use HTML5 main element * squashme: remove stray space
GSA · Mar 16, 2018 · e48fc7d · e48fc7d
1 parent 21331b8
commit e48fc7d
Show file tree

Hide file tree

Showing 54 changed files with 46 additions and 1,113 deletions.
diff --git a/404.html b/404.html
@@ -1,6 +1,7 @@
 ---
 layout: article
 title: Oops! We can't find the file.
+sitemap: false
 ---
 
 <p>We're sorry, but the page you're looking for might have been removed, had its name changed, or is temporarily unavailable. What should you do?</p>
@@ -11,4 +12,4 @@
 <li>Use our <a href="http://search.usa.gov/search/news?utf8=%E2%9C%93&m=true&affiliate=usasearch&channel=1745">search</a> to find the information you want.</li>
 </ul>
 
-<p>We're here to answer your questions. Email us at <a href="mailto:[email protected]">[email protected]</a>, or give us a call at 202-505-5315.</p>
+<p>We're here to answer your questions. Email us at <a href="mailto:[email protected]">[email protected]</a>, or give us a call at 202-505-5315.</p>
diff --git a/Gemfile b/Gemfile
@@ -7,3 +7,9 @@ gem 'rdiscount'
 gem 'rspec'
 gem 'sass', '~> 3.2.19'
 gem 'therubyracer'
+gem 'jekyll-sitemap'
+
+group :test, :development do
+  gem 'pry'
+  gem 'rb-readline'
+end
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -4,6 +4,7 @@ GEM
     addressable (2.5.2)
       public_suffix (>= 2.0.2, < 4.0)
     chunky_png (1.2.7)
+    coderay (1.1.1)
     colorator (1.1.0)
     compass (0.12.2)
       chunky_png (~> 1.2)
@@ -26,6 +27,8 @@ GEM
       safe_yaml (~> 1.0)
     jekyll-sass-converter (1.3.0)
       sass (~> 3.2)
+    jekyll-sitemap (1.2.0)
+      jekyll (~> 3.3)
     jekyll-watch (1.5.0)
       listen (~> 3.0, < 3.1)
     kramdown (1.16.1)
@@ -35,13 +38,19 @@ GEM
       rb-fsevent (~> 0.9, >= 0.9.4)
       rb-inotify (~> 0.9, >= 0.9.7)
     mercenary (0.3.6)
+    method_source (0.8.2)
     pathutil (0.16.0)
       forwardable-extended (~> 2.6)
+    pry (0.10.4)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
     public_suffix (3.0.1)
     rake (10.0.3)
     rb-fsevent (0.10.2)
     rb-inotify (0.9.10)
       ffi (>= 0.5.0, < 2)
+    rb-readline (0.5.5)
     rdiscount (2.1.7)
     ref (1.0.5)
     rouge (2.2.1)
@@ -60,6 +69,7 @@ GEM
     rspec-support (3.2.2)
     safe_yaml (1.0.4)
     sass (3.2.19)
+    slop (3.6.0)
     therubyracer (0.12.2)
       libv8 (~> 3.16.14.0)
       ref
@@ -70,7 +80,10 @@ PLATFORMS
 DEPENDENCIES
   compass
   jekyll
+  jekyll-sitemap
+  pry
   rake
+  rb-readline
   rdiscount
   rspec
   sass (~> 3.2.19)

diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-# www.search.gov
+# search.gov
 
 ## How to add a new post
 

diff --git a/_config.yml b/_config.yml
@@ -1,6 +1,6 @@
 author: Search.gov
 title: Search.gov
-url: https://www.search.gov
+url: https://search.gov
 permalink: /:categories/:title.html
 lsi: false
 highlighter: pygments
@@ -22,3 +22,6 @@ paginate_per_page: 10
 posts_paginate_path: blog/page:num
 posts_paginate_layout: paginate
 posts_tag_layout: tag
+
+plugins:
+  - jekyll-sitemap
diff --git a/_includes/head.html b/_includes/head.html
@@ -1,6 +1,7 @@
 <head>
   <meta name=viewport content="width=device-width, initial-scale=1">
   <meta charset="utf-8">
+  {% if page.metatags %} <meta name="robots" content="{{page.metatags}}">{% endif %}
   <title>{{ page.title }}</title>
   <meta name="description" content="{{ page.description }}">
   <meta name="author" content="">

diff --git a/_layouts/default.html b/_layouts/default.html
@@ -7,9 +7,9 @@
 <div class="container-fluid">
   <div class="col-md-8 col-md-offset-2">
     <!-- do not remove as used to parse in usasearch -->
-    <div id="main-container">    
-      {{ content }}      
-    </div>
+    <main id="main-container">
+      {{ content }}
+    </main>
   </div>
 </div>
 

diff --git a/_layouts/feature.html b/_layouts/feature.html
@@ -5,11 +5,11 @@
 {% include navbar.html %}
 <div class="container-fluid">
   <!-- do not remove as used to parse in usasearch -->
-  <div id="main-container">    
+  <main id="main-container">    
     <div class="col-md-8 col-md-offset-2">      
       {{ content }}      
     </div>
-  </div>
+  </main>
 </div>
 {% include footer.html %}
 </body>

diff --git a/_layouts/landing-page.html b/_layouts/landing-page.html
@@ -4,11 +4,11 @@
 <body>
 {% include navbar.html %}
 <div class="container">
-  <div id="main-container">
+  <main id="main-container">
     <article id="landing-page" class="article">
       {{ content }}
     </article>
-  </div>
+  </main>
 </div>
 {% include footer.html %}
 </body>

diff --git a/_layouts/tag.html b/_layouts/tag.html
@@ -1,5 +1,7 @@
 ---
 layout: default
+metatags: noindex, follow
+sitemap: false
 ---
 
 <!-- begin tag layout -->
@@ -23,4 +25,4 @@
   </li>
   {% endif %}
 </ul>
-<!-- end tag layout -->
+<!-- end tag layout -->
diff --git a/blog/index.md b/blog/index.md
@@ -1,6 +1,7 @@
 ---
 layout: default
 title: Search.gov Blog
+sitemap: false
 ---
 
 <!-- begin /blog/index.md content -->

diff --git a/drafts/2013-02-11-west-palm-beach.md b/drafts/2013-02-11-west-palm-beach.md
diff --git a/drafts/2013-12-31-feature-CTR-top-queries.md b/drafts/2013-12-31-feature-CTR-top-queries.md
diff --git a/drafts/2013-12-31-feature-analytics-best-bets.md b/drafts/2013-12-31-feature-analytics-best-bets.md
diff --git a/drafts/2013-12-31-feature-improved-BB-relevancy.md b/drafts/2013-12-31-feature-improved-BB-relevancy.md
diff --git a/drafts/2014-01-31-feature-responsive-results-pages.md b/drafts/2014-01-31-feature-responsive-results-pages.md
diff --git a/drafts/2014-02-27-customers-cdfif.md b/drafts/2014-02-27-customers-cdfif.md
diff --git a/drafts/2014-02-27-customers-esa.md b/drafts/2014-02-27-customers-esa.md
diff --git a/drafts/2014-02-27-customers-laosup.md b/drafts/2014-02-27-customers-laosup.md
diff --git a/drafts/2014-02-27-customers-mioag.md b/drafts/2014-02-27-customers-mioag.md
diff --git a/drafts/2014-02-27-customers-ncnutrition.md b/drafts/2014-02-27-customers-ncnutrition.md
diff --git a/drafts/2014-02-27-customers-noaacsc.md b/drafts/2014-02-27-customers-noaacsc.md
diff --git a/drafts/2014-02-27-customers-sheriff.md b/drafts/2014-02-27-customers-sheriff.md
diff --git a/drafts/2014-02-27-customers-tigta.md b/drafts/2014-02-27-customers-tigta.md
diff --git a/drafts/2014-02-27-customers-treasdirect.md b/drafts/2014-02-27-customers-treasdirect.md
diff --git a/drafts/2014-03-05-sample.md b/drafts/2014-03-05-sample.md
@@ -3,6 +3,10 @@ layout: post
 title: My Sample page
 category: manual
 tags: how-to
+# Remove the following two lines when publishing, so the page
+# can be included in the sitemap and indexed by search engines
+sitemap: false
+metatags: noindex
 ---
 
 This will show up in /drafts/manual/sample.html.
diff --git a/drafts/2014-03-06-promos.md b/drafts/2014-03-06-promos.md
diff --git a/drafts/2014-03-13-customer-nctobacco.md b/drafts/2014-03-13-customer-nctobacco.md
diff --git a/drafts/2014-03-13-customer-pcphpv.md b/drafts/2014-03-13-customer-pcphpv.md
diff --git a/drafts/2014-03-13-customer-washingtonoria.md b/drafts/2014-03-13-customer-washingtonoria.md
diff --git a/drafts/2014-05-15-march-2014.md b/drafts/2014-05-15-march-2014.md