Upgrade to pulldown-cmark 0.11

2024-08-04 15:16:33 +02:00 · 2024-08-04 15:16:33 +02:00 · 6d0e693ed7
commit 6d0e693ed7
parent 72735cfff8
5 changed files with 101 additions and 49 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@ -456,21 +456,28 @@ dependencies = [

 [[package]]
 name = "pulldown-cmark"
-version = "0.9.6"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b"
+checksum = "8746739f11d39ce5ad5c2520a9b75285310dbfe78c541ccf832d38615765aec0"
 dependencies = [
 "bitflags 2.6.0",
 "getopts",
 "memchr",
+ "pulldown-cmark-escape",
 "unicase",
 ]

 [[package]]
-name = "pulldown-cmark-to-cmark"
-version = "11.2.0"
+name = "pulldown-cmark-escape"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6dd464f32d7631035e849fcd969a603e9bb17ceaebe8467352a7728147f34e42"
+checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae"
+
+[[package]]
+name = "pulldown-cmark-to-cmark"
+version = "15.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9c77db841443d89a57ae94f22d29c022f6d9f41b00bddbf1f4024dbaf4bdce1"
 dependencies = [
 "pulldown-cmark",
 ]
--- a/Cargo.toml
+++ b/Cargo.toml
@ -31,8 +31,8 @@ ignore = "0.4.22"
 matter = "0.1.0-alpha4"
 pathdiff = "0.2.1"
 percent-encoding = "2.3.1"
-pulldown-cmark = "0.9.3"
-pulldown-cmark-to-cmark = "11.0.2"
+pulldown-cmark = "0.11.0"
+pulldown-cmark-to-cmark = "15.0.0"
 rayon = "1.10.0"
 regex = "1.10.5"
 serde_yaml = "0.9.34"
--- a/changelog.d/252.breaking.md
+++ b/changelog.d/252.breaking.md
@ -0,0 +1,12 @@
+Upgrade [pulldown-cmark](https://crates.io/crates/pulldown-cmark) from 0.9 to 0.11
+
+pulldown-cmark is the Markdown/CommonMark parser that is used to read and convert notes (together with [pulldown-cmark-to-cmark](https://crates.io/crates/pulldown-cmark-to-cmark)).
+
+For end-users that call the obsidian-export CLI this upgrade will be mostly transparent, except that Math blocks are now properly processed without getting mangled.
+
+People who use the library directly may face more significant breaking changes if they have custom postprocessors, as pulldown-cmark's events have gone through various breaking changes.
+For more information, see:
+
+- <https://github.com/zoni/obsidian-export/pull/252>
+- <https://github.com/pulldown-cmark/pulldown-cmark/releases/tag/v0.10.0>
+- <https://github.com/zoni/obsidian-export/pull/276/files#diff-b1a35a68f14e696205874893c07fd24fdb88882b47c23cc0e0c80a30c7d53759>
--- a/src/lib.rs
+++ b/src/lib.rs
@ -19,7 +19,7 @@ use frontmatter::{frontmatter_from_str, frontmatter_to_str};
 pub use frontmatter::{Frontmatter, FrontmatterStrategy};
 use pathdiff::diff_paths;
 use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS};
-use pulldown_cmark::{CodeBlockKind, CowStr, Event, HeadingLevel, Options, Parser, Tag};
+use pulldown_cmark::{CodeBlockKind, CowStr, Event, HeadingLevel, Options, Parser, Tag, TagEnd};
 use pulldown_cmark_to_cmark::cmark_with_options;
 use rayon::prelude::*;
 use references::{ObsidianNoteReference, RefParser, RefParserState, RefType};
@ -662,20 +662,18 @@ impl<'a> Exporter<'a> {
                        // into an image reference instead. Slightly hacky, but avoids needing
                        // to keep another utility function around for this, or introducing an
                        // extra parameter on make_link_to_file.
-                        Event::Start(Tag::Link(linktype, cowstr1, cowstr2)) => {
-                            Event::Start(Tag::Image(
-                                linktype,
-                                CowStr::from(cowstr1.into_string()),
-                                CowStr::from(cowstr2.into_string()),
-                            ))
-                        }
-                        Event::End(Tag::Link(linktype, cowstr1, cowstr2)) => {
-                            Event::End(Tag::Image(
-                                linktype,
-                                CowStr::from(cowstr1.into_string()),
-                                CowStr::from(cowstr2.into_string()),
-                            ))
-                        }
+                        Event::Start(Tag::Link {
+                            link_type,
+                            dest_url,
+                            title,
+                            id,
+                        }) => Event::Start(Tag::Image {
+                            link_type,
+                            dest_url: CowStr::from(dest_url.into_string()),
+                            title: CowStr::from(title.into_string()),
+                            id: CowStr::from(id.into_string()),
+                        }),
+                        Event::End(TagEnd::Link) => Event::End(TagEnd::Image),
                        _ => event,
                    })
                    .collect()
@ -707,7 +705,7 @@ impl<'a> Exporter<'a> {
            return vec![
                Event::Start(Tag::Emphasis),
                Event::Text(CowStr::from(reference.display())),
-                Event::End(Tag::Emphasis),
+                Event::End(TagEnd::Emphasis),
            ];
        }
        let target_file = target_file.unwrap();
@ -731,16 +729,17 @@ impl<'a> Exporter<'a> {
            link.push_str(&slugify(section));
        }

-        let link_tag = Tag::Link(
-            pulldown_cmark::LinkType::Inline,
-            CowStr::from(link),
-            CowStr::from(""),
-        );
+        let link_tag = Tag::Link {
+            link_type: pulldown_cmark::LinkType::Inline,
+            dest_url: CowStr::from(link),
+            title: CowStr::from(""),
+            id: CowStr::from(""),
+        };

        vec![
-            Event::Start(link_tag.clone()),
+            Event::Start(link_tag),
            Event::Text(CowStr::from(reference.display())),
-            Event::End(link_tag.clone()),
+            Event::End(TagEnd::Link),
        ]
    }
 }
@ -841,8 +840,7 @@ fn reduce_to_section<'a>(events: MarkdownEvents<'a>, section: &str) -> MarkdownE
    for event in events {
        filtered_events.push(event.clone());
        match event {
-            // FIXME: This should propagate fragment_identifier and classes.
-            Event::Start(Tag::Heading(level, _fragment_identifier, _classes)) => {
+            Event::Start(Tag::Heading { level, .. }) => {
                last_tag_was_heading = true;
                last_level = level;
                if currently_in_target_section && level <= section_level {
@ -881,10 +879,11 @@ fn reduce_to_section<'a>(events: MarkdownEvents<'a>, section: &str) -> MarkdownE
 fn event_to_owned<'a>(event: Event<'_>) -> Event<'a> {
    match event {
        Event::Start(tag) => Event::Start(tag_to_owned(tag)),
-        Event::End(tag) => Event::End(tag_to_owned(tag)),
+        Event::End(tag) => Event::End(tag),
        Event::Text(cowstr) => Event::Text(CowStr::from(cowstr.into_string())),
        Event::Code(cowstr) => Event::Code(CowStr::from(cowstr.into_string())),
        Event::Html(cowstr) => Event::Html(CowStr::from(cowstr.into_string())),
+        Event::InlineHtml(cowstr) => Event::InlineHtml(CowStr::from(cowstr.into_string())),
        Event::FootnoteReference(cowstr) => {
            Event::FootnoteReference(CowStr::from(cowstr.into_string()))
        }
@ -892,17 +891,37 @@ fn event_to_owned<'a>(event: Event<'_>) -> Event<'a> {
        Event::HardBreak => Event::HardBreak,
        Event::Rule => Event::Rule,
        Event::TaskListMarker(checked) => Event::TaskListMarker(checked),
+        Event::InlineMath(cowstr) => Event::InlineMath(CowStr::from(cowstr.into_string())),
+        Event::DisplayMath(cowstr) => Event::DisplayMath(CowStr::from(cowstr.into_string())),
    }
 }

 fn tag_to_owned<'a>(tag: Tag<'_>) -> Tag<'a> {
    match tag {
        Tag::Paragraph => Tag::Paragraph,
-        Tag::Heading(level, _fragment_identifier, _classes) => {
-            // FIXME: This should propagate fragment_identifier and classes.
-            Tag::Heading(level, None, Vec::new())
-        }
-        Tag::BlockQuote => Tag::BlockQuote,
+        Tag::Heading {
+            level: heading_level,
+            id,
+            classes,
+            attrs,
+        } => Tag::Heading {
+            level: heading_level,
+            id: id.map(|cowstr| CowStr::from(cowstr.into_string())),
+            classes: classes
+                .into_iter()
+                .map(|cowstr| CowStr::from(cowstr.into_string()))
+                .collect(),
+            attrs: attrs
+                .into_iter()
+                .map(|(attr, value)| {
+                    (
+                        CowStr::from(attr.into_string()),
+                        value.map(|cowstr| CowStr::from(cowstr.into_string())),
+                    )
+                })
+                .collect(),
+        },
+        Tag::BlockQuote(blockquote_kind) => Tag::BlockQuote(blockquote_kind),
        Tag::CodeBlock(codeblock_kind) => Tag::CodeBlock(codeblock_kind_to_owned(codeblock_kind)),
        Tag::List(optional) => Tag::List(optional),
        Tag::Item => Tag::Item,
@ -916,16 +935,30 @@ fn tag_to_owned<'a>(tag: Tag<'_>) -> Tag<'a> {
        Tag::Emphasis => Tag::Emphasis,
        Tag::Strong => Tag::Strong,
        Tag::Strikethrough => Tag::Strikethrough,
-        Tag::Link(linktype, cowstr1, cowstr2) => Tag::Link(
-            linktype,
-            CowStr::from(cowstr1.into_string()),
-            CowStr::from(cowstr2.into_string()),
-        ),
-        Tag::Image(linktype, cowstr1, cowstr2) => Tag::Image(
-            linktype,
-            CowStr::from(cowstr1.into_string()),
-            CowStr::from(cowstr2.into_string()),
-        ),
+        Tag::Link {
+            link_type,
+            dest_url,
+            title,
+            id,
+        } => Tag::Link {
+            link_type,
+            dest_url: CowStr::from(dest_url.into_string()),
+            title: CowStr::from(title.into_string()),
+            id: CowStr::from(id.into_string()),
+        },
+        Tag::Image {
+            link_type,
+            dest_url,
+            title,
+            id,
+        } => Tag::Image {
+            link_type,
+            dest_url: CowStr::from(dest_url.into_string()),
+            title: CowStr::from(title.into_string()),
+            id: CowStr::from(id.into_string()),
+        },
+        Tag::HtmlBlock => Tag::HtmlBlock,
+        Tag::MetadataBlock(metadata_block_kind) => Tag::MetadataBlock(metadata_block_kind),
    }
 }

--- a/tests/testdata/expected/main-samples/pure-markdown-examples.md
+++ b/tests/testdata/expected/main-samples/pure-markdown-examples.md
@ -37,7 +37,7 @@
 ~~Strikethrough~~

 |Table||
-|-----|--|
+|-----|-|
 |Foo|Bar|

 [link text](link-location.md)