Rollup merge of rust-lang#41290 - GuillaumeGomez:put-back-hoedown, r=…

…steveklabnik

Hoedown big comeback!

```bash
> cargo +local test
   Compiling libc v0.2.20
   Compiling sysinfo v0.3.4 (file:///Users/imperio/rust/sysinfo)
    Finished dev [unoptimized + debuginfo] target(s) in 3.2 secs
     Running target/debug/deps/disk_list-dbd70897f1f7e080

running 1 test
test test_disks ... ok

test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured

     Running target/debug/deps/sysinfo-8ad11103abdf5941

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured

   Doc-tests sysinfo
WARNING: src/sysinfo.rs -  (line 45) test will be run in the next rustdoc version. If it's not supposed to, please update your documentation and make it compliant to common mark specifications.
WARNING: src/sysinfo.rs -  (line 48) test will be run in the next rustdoc version. If it's not supposed to, please update your documentation and make it compliant to common mark specifications.

running 1 test
test src/sysinfo.rs -  (line 14) ... ok

test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured
```

r? @rust-lang/docs

.gitmodules

@@ -5,6 +5,10 @@
 [submodule "src/compiler-rt"]
 	path = src/compiler-rt
 	url = https://github.com/rust-lang/compiler-rt.git
+[submodule "src/rt/hoedown"]
+	path = src/rt/hoedown
+	url = https://github.com/rust-lang/hoedown.git
+	branch = rust-2015-09-21-do-not-delete
 [submodule "src/jemalloc"]
 	path = src/jemalloc
 	url = https://github.com/rust-lang/jemalloc.git

COPYRIGHT

@@ -197,6 +197,28 @@ their own copyright notices and license terms:
     USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
     OF SUCH DAMAGE.
 
+* Hoedown, the markdown parser, under src/rt/hoedown, is
+  licensed as follows.
+
+    Copyright (c) 2008, Natacha Porté
+    Copyright (c) 2011, Vicent Martí
+    Copyright (c) 2013, Devin Torres and the Hoedown authors
+
+    Permission to use, copy, modify, and distribute this
+    software for any purpose with or without fee is hereby
+    granted, provided that the above copyright notice and
+    this permission notice appear in all copies.
+
+    THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR
+    DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE
+    INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+    FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+    SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR
+    ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA
+    OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+    OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+    CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
 * libbacktrace, under src/libbacktrace:
 
     Copyright (C) 2012-2014 Free Software Foundation, Inc.

src/librustdoc/Cargo.toml

@@ -2,6 +2,7 @@
 authors = ["The Rust Project Developers"]
 name = "rustdoc"
 version = "0.0.0"
+build = "build.rs"
 
 [lib]
 name = "rustdoc"

src/librustdoc/build.rs

@@ -0,0 +1,30 @@
+// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+extern crate build_helper;
+extern crate gcc;
+
+fn main() {
+    let src_dir = std::path::Path::new("../rt/hoedown/src");
+    build_helper::rerun_if_changed_anything_in_dir(src_dir);
+    let mut cfg = gcc::Config::new();
+    cfg.file("../rt/hoedown/src/autolink.c")
+       .file("../rt/hoedown/src/buffer.c")
+       .file("../rt/hoedown/src/document.c")
+       .file("../rt/hoedown/src/escape.c")
+       .file("../rt/hoedown/src/html.c")
+       .file("../rt/hoedown/src/html_blocks.c")
+       .file("../rt/hoedown/src/html_smartypants.c")
+       .file("../rt/hoedown/src/stack.c")
+       .file("../rt/hoedown/src/version.c")
+       .include(src_dir)
+       .compile("libhoedown.a");
+}
+

src/librustdoc/html/markdown.rs

@@ -25,6 +25,9 @@
 
 #![allow(non_camel_case_types)]
 
+use libc;
+use std::slice;
+
 use std::ascii::AsciiExt;
 use std::cell::RefCell;
 use std::collections::{HashMap, VecDeque};
@@ -357,6 +360,194 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for Footnotes<'a, I> {
     }
 }
 
+const DEF_OUNIT: libc::size_t = 64;
+const HOEDOWN_EXT_NO_INTRA_EMPHASIS: libc::c_uint = 1 << 11;
+const HOEDOWN_EXT_TABLES: libc::c_uint = 1 << 0;
+const HOEDOWN_EXT_FENCED_CODE: libc::c_uint = 1 << 1;
+const HOEDOWN_EXT_AUTOLINK: libc::c_uint = 1 << 3;
+const HOEDOWN_EXT_STRIKETHROUGH: libc::c_uint = 1 << 4;
+const HOEDOWN_EXT_SUPERSCRIPT: libc::c_uint = 1 << 8;
+const HOEDOWN_EXT_FOOTNOTES: libc::c_uint = 1 << 2;
+
+const HOEDOWN_EXTENSIONS: libc::c_uint =
+    HOEDOWN_EXT_NO_INTRA_EMPHASIS | HOEDOWN_EXT_TABLES |
+    HOEDOWN_EXT_FENCED_CODE | HOEDOWN_EXT_AUTOLINK |
+    HOEDOWN_EXT_STRIKETHROUGH | HOEDOWN_EXT_SUPERSCRIPT |
+    HOEDOWN_EXT_FOOTNOTES;
+
+enum hoedown_document {}
+
+type blockcodefn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
+                                 *const hoedown_buffer, *const hoedown_renderer_data,
+                                 libc::size_t);
+
+type blockquotefn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
+                                  *const hoedown_renderer_data, libc::size_t);
+
+type headerfn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
+                              libc::c_int, *const hoedown_renderer_data,
+                              libc::size_t);
+
+type blockhtmlfn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
+                                 *const hoedown_renderer_data, libc::size_t);
+
+type codespanfn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
+                                *const hoedown_renderer_data, libc::size_t) -> libc::c_int;
+
+type linkfn = extern "C" fn (*mut hoedown_buffer, *const hoedown_buffer,
+                             *const hoedown_buffer, *const hoedown_buffer,
+                             *const hoedown_renderer_data, libc::size_t) -> libc::c_int;
+
+type entityfn = extern "C" fn (*mut hoedown_buffer, *const hoedown_buffer,
+                               *const hoedown_renderer_data, libc::size_t);
+
+type normaltextfn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
+                                  *const hoedown_renderer_data, libc::size_t);
+
+#[repr(C)]
+struct hoedown_renderer_data {
+    opaque: *mut libc::c_void,
+}
+
+#[repr(C)]
+struct hoedown_renderer {
+    opaque: *mut libc::c_void,
+
+    blockcode: Option<blockcodefn>,
+    blockquote: Option<blockquotefn>,
+    header: Option<headerfn>,
+
+    other_block_level_callbacks: [libc::size_t; 11],
+
+    blockhtml: Option<blockhtmlfn>,
+
+    /* span level callbacks - NULL or return 0 prints the span verbatim */
+    autolink: libc::size_t, // unused
+    codespan: Option<codespanfn>,
+    other_span_level_callbacks_1: [libc::size_t; 7],
+    link: Option<linkfn>,
+    other_span_level_callbacks_2: [libc::size_t; 6],
+
+    /* low level callbacks - NULL copies input directly into the output */
+    entity: Option<entityfn>,
+    normal_text: Option<normaltextfn>,
+
+    /* header and footer */
+    other_callbacks: [libc::size_t; 2],
+}
+
+#[repr(C)]
+struct hoedown_html_renderer_state {
+    opaque: *mut libc::c_void,
+    toc_data: html_toc_data,
+    flags: libc::c_uint,
+    link_attributes: Option<extern "C" fn(*mut hoedown_buffer,
+                                          *const hoedown_buffer,
+                                          *const hoedown_renderer_data)>,
+}
+
+#[repr(C)]
+struct html_toc_data {
+    header_count: libc::c_int,
+    current_level: libc::c_int,
+    level_offset: libc::c_int,
+    nesting_level: libc::c_int,
+}
+
+#[repr(C)]
+struct hoedown_buffer {
+    data: *const u8,
+    size: libc::size_t,
+    asize: libc::size_t,
+    unit: libc::size_t,
+}
+
+extern {
+    fn hoedown_html_renderer_new(render_flags: libc::c_uint,
+                                 nesting_level: libc::c_int)
+        -> *mut hoedown_renderer;
+    fn hoedown_html_renderer_free(renderer: *mut hoedown_renderer);
+
+    fn hoedown_document_new(rndr: *const hoedown_renderer,
+                            extensions: libc::c_uint,
+                            max_nesting: libc::size_t) -> *mut hoedown_document;
+    fn hoedown_document_render(doc: *mut hoedown_document,
+                               ob: *mut hoedown_buffer,
+                               document: *const u8,
+                               doc_size: libc::size_t);
+    fn hoedown_document_free(md: *mut hoedown_document);
+
+    fn hoedown_buffer_new(unit: libc::size_t) -> *mut hoedown_buffer;
+    fn hoedown_buffer_free(b: *mut hoedown_buffer);
+}
+
+impl hoedown_buffer {
+    fn as_bytes(&self) -> &[u8] {
+        unsafe { slice::from_raw_parts(self.data, self.size as usize) }
+    }
+}
+
+pub fn old_find_testable_code(doc: &str, tests: &mut ::test::Collector, position: Span) {
+    extern fn block(_ob: *mut hoedown_buffer,
+                    text: *const hoedown_buffer,
+                    lang: *const hoedown_buffer,
+                    data: *const hoedown_renderer_data,
+                    line: libc::size_t) {
+        unsafe {
+            if text.is_null() { return }
+            let block_info = if lang.is_null() {
+                LangString::all_false()
+            } else {
+                let lang = (*lang).as_bytes();
+                let s = str::from_utf8(lang).unwrap();
+                LangString::parse(s)
+            };
+            if !block_info.rust { return }
+            let opaque = (*data).opaque as *mut hoedown_html_renderer_state;
+            let tests = &mut *((*opaque).opaque as *mut ::test::Collector);
+            let line = tests.get_line() + line;
+            let filename = tests.get_filename();
+            tests.add_old_test(line, filename);
+        }
+    }
+
+    extern fn header(_ob: *mut hoedown_buffer,
+                     text: *const hoedown_buffer,
+                     level: libc::c_int, data: *const hoedown_renderer_data,
+                     _: libc::size_t) {
+        unsafe {
+            let opaque = (*data).opaque as *mut hoedown_html_renderer_state;
+            let tests = &mut *((*opaque).opaque as *mut ::test::Collector);
+            if text.is_null() {
+                tests.register_header("", level as u32);
+            } else {
+                let text = (*text).as_bytes();
+                let text = str::from_utf8(text).unwrap();
+                tests.register_header(text, level as u32);
+            }
+        }
+    }
+
+    tests.set_position(position);
+
+    unsafe {
+        let ob = hoedown_buffer_new(DEF_OUNIT);
+        let renderer = hoedown_html_renderer_new(0, 0);
+        (*renderer).blockcode = Some(block);
+        (*renderer).header = Some(header);
+        (*((*renderer).opaque as *mut hoedown_html_renderer_state)).opaque
+                = tests as *mut _ as *mut libc::c_void;
+
+        let document = hoedown_document_new(renderer, HOEDOWN_EXTENSIONS, 16);
+        hoedown_document_render(document, ob, doc.as_ptr(),
+                                doc.len() as libc::size_t);
+        hoedown_document_free(document);
+
+        hoedown_html_renderer_free(renderer);
+        hoedown_buffer_free(ob);
+    }
+}
+
 pub fn find_testable_code(doc: &str, tests: &mut ::test::Collector, position: Span) {
     tests.set_position(position);
 

src/librustdoc/markdown.rs

@@ -25,7 +25,7 @@ use externalfiles::{ExternalHtml, LoadStringError, load_string};
 use html::render::reset_ids;
 use html::escape::Escape;
 use html::markdown;
-use html::markdown::{Markdown, MarkdownWithToc, find_testable_code};
+use html::markdown::{Markdown, MarkdownWithToc, find_testable_code, old_find_testable_code};
 use test::{TestOptions, Collector};
 
 /// Separate any lines at the start of the file that begin with `# ` or `%`.
@@ -159,6 +159,7 @@ pub fn test(input: &str, cfgs: Vec<String>, libs: SearchPaths, externs: Externs,
     let mut collector = Collector::new(input.to_string(), cfgs, libs, externs,
                                        true, opts, maybe_sysroot, None,
                                        Some(input.to_owned()));
+    old_find_testable_code(&input_str, &mut collector, DUMMY_SP);
     find_testable_code(&input_str, &mut collector, DUMMY_SP);
     test_args.insert(0, "rustdoctest".to_string());
     testing::test_main(&test_args, collector.tests);

src/librustdoc/test.rs

@@ -380,6 +380,8 @@ fn partition_source(s: &str) -> (String, String) {
 
 pub struct Collector {
     pub tests: Vec<testing::TestDescAndFn>,
+    // to be removed when hoedown will be definitely gone
+    pub old_tests: Vec<String>,
     names: Vec<String>,
     cfgs: Vec<String>,
     libs: SearchPaths,
@@ -401,6 +403,7 @@ impl Collector {
                codemap: Option<Rc<CodeMap>>, filename: Option<String>) -> Collector {
         Collector {
             tests: Vec::new(),
+            old_tests: Vec::new(),
             names: Vec::new(),
             cfgs: cfgs,
             libs: libs,
@@ -417,19 +420,36 @@ impl Collector {
         }
     }
 
-    pub fn add_test(&mut self, test: String,
-                    should_panic: bool, no_run: bool, should_ignore: bool,
-                    as_test_harness: bool, compile_fail: bool, error_codes: Vec<String>,
-                    line: usize, filename: String) {
-        let name = if self.use_headers {
+    fn generate_name(&self, line: usize, filename: &str) -> String {
+        if self.use_headers {
             if let Some(ref header) = self.current_header {
                 format!("{} - {} (line {})", filename, header, line)
             } else {
                 format!("{} - (line {})", filename, line)
             }
         } else {
             format!("{} - {} (line {})", filename, self.names.join("::"), line)
-        };
+        }
+    }
+
+    pub fn add_old_test(&mut self, line: usize, filename: String) {
+        let name = self.generate_name(line, &filename);
+        self.old_tests.push(name);
+    }
+
+    pub fn add_test(&mut self, test: String,
+                    should_panic: bool, no_run: bool, should_ignore: bool,
+                    as_test_harness: bool, compile_fail: bool, error_codes: Vec<String>,
+                    line: usize, filename: String) {
+        let name = self.generate_name(line, &filename);
+        if self.old_tests.iter().find(|&x| x == &name).is_none() {
+            let _ = writeln!(&mut io::stderr(),
+                             "WARNING: {} Code block is not currently run as a test, but will in \
+                              future versions of rustdoc. Please ensure this code block is a \
+                              runnable test, or use the `ignore` directive.",
+                             name);
+            return
+        }
         let cfgs = self.cfgs.clone();
         let libs = self.libs.clone();
         let externs = self.externs.clone();
@@ -544,6 +564,8 @@ impl<'a, 'hir> HirCollector<'a, 'hir> {
         attrs.unindent_doc_comments();
         if let Some(doc) = attrs.doc_value() {
             self.collector.cnt = 0;
+            markdown::old_find_testable_code(doc, self.collector,
+                                             attrs.span.unwrap_or(DUMMY_SP));
             markdown::find_testable_code(doc, self.collector,
                                          attrs.span.unwrap_or(DUMMY_SP));
         }

src/tools/tidy/src/main.rs

@@ -86,6 +86,7 @@ fn filter_dirs(path: &Path) -> bool {
         "src/rust-installer",
         "src/liblibc",
         "src/vendor",
+        "src/rt/hoedown",
     ];
     skip.iter().any(|p| path.ends_with(p))
 }