Upgrade to pulldown-cmark 0.11

Cargo.lock

@@ -456,21 +456,28 @@ dependencies = [
 
 [[package]]
 name = "pulldown-cmark"
-version = "0.9.6"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b"
+checksum = "8746739f11d39ce5ad5c2520a9b75285310dbfe78c541ccf832d38615765aec0"
 dependencies = [
  "bitflags 2.6.0",
  "getopts",
  "memchr",
+ "pulldown-cmark-escape",
  "unicase",
 ]
 
 [[package]]
-name = "pulldown-cmark-to-cmark"
+name = "pulldown-cmark-escape"
-version = "11.2.0"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6dd464f32d7631035e849fcd969a603e9bb17ceaebe8467352a7728147f34e42"
+checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae"
+
+[[package]]
+name = "pulldown-cmark-to-cmark"
+version = "15.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9c77db841443d89a57ae94f22d29c022f6d9f41b00bddbf1f4024dbaf4bdce1"
 dependencies = [
  "pulldown-cmark",
 ]

Cargo.toml

@@ -31,8 +31,8 @@ ignore = "0.4.22"
 matter = "0.1.0-alpha4"
 pathdiff = "0.2.1"
 percent-encoding = "2.3.1"
-pulldown-cmark = "0.9.3"
+pulldown-cmark = "0.11.0"
-pulldown-cmark-to-cmark = "11.0.2"
+pulldown-cmark-to-cmark = "15.0.0"
 rayon = "1.10.0"
 regex = "1.10.5"
 serde_yaml = "0.9.34"

changelog.d/252.breaking.md

@@ -0,0 +1,12 @@
+Upgrade [pulldown-cmark](https://crates.io/crates/pulldown-cmark) from 0.9 to 0.11
+
+pulldown-cmark is the Markdown/CommonMark parser that is used to read and convert notes (together with [pulldown-cmark-to-cmark](https://crates.io/crates/pulldown-cmark-to-cmark)).
+
+For end-users that call the obsidian-export CLI this upgrade will be mostly transparent, except that Math blocks are now properly processed without getting mangled.
+
+People who use the library directly may face more significant breaking changes if they have custom postprocessors, as pulldown-cmark's events have gone through various breaking changes.
+For more information, see:
+
+- <https://github.com/zoni/obsidian-export/pull/252>
+- <https://github.com/pulldown-cmark/pulldown-cmark/releases/tag/v0.10.0>
+- <https://github.com/zoni/obsidian-export/pull/276/files#diff-b1a35a68f14e696205874893c07fd24fdb88882b47c23cc0e0c80a30c7d53759>

src/lib.rs

@@ -19,7 +19,7 @@ use frontmatter::{frontmatter_from_str, frontmatter_to_str};
 pub use frontmatter::{Frontmatter, FrontmatterStrategy};
 use pathdiff::diff_paths;
 use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS};
-use pulldown_cmark::{CodeBlockKind, CowStr, Event, HeadingLevel, Options, Parser, Tag};
+use pulldown_cmark::{CodeBlockKind, CowStr, Event, HeadingLevel, Options, Parser, Tag, TagEnd};
 use pulldown_cmark_to_cmark::cmark_with_options;
 use rayon::prelude::*;
 use references::{ObsidianNoteReference, RefParser, RefParserState, RefType};
@@ -662,20 +662,18 @@ impl<'a> Exporter<'a> {
                         // into an image reference instead. Slightly hacky, but avoids needing
                         // to keep another utility function around for this, or introducing an
                         // extra parameter on make_link_to_file.
-                        Event::Start(Tag::Link(linktype, cowstr1, cowstr2)) => {
+                        Event::Start(Tag::Link {
-                            Event::Start(Tag::Image(
+                            link_type,
-                                linktype,
+                            dest_url,
-                                CowStr::from(cowstr1.into_string()),
+                            title,
-                                CowStr::from(cowstr2.into_string()),
+                            id,
-                            ))
+                        }) => Event::Start(Tag::Image {
-                        }
+                            link_type,
-                        Event::End(Tag::Link(linktype, cowstr1, cowstr2)) => {
+                            dest_url: CowStr::from(dest_url.into_string()),
-                            Event::End(Tag::Image(
+                            title: CowStr::from(title.into_string()),
-                                linktype,
+                            id: CowStr::from(id.into_string()),
-                                CowStr::from(cowstr1.into_string()),
+                        }),
-                                CowStr::from(cowstr2.into_string()),
+                        Event::End(TagEnd::Link) => Event::End(TagEnd::Image),
-                            ))
-                        }
                         _ => event,
                     })
                     .collect()
@@ -707,7 +705,7 @@ impl<'a> Exporter<'a> {
             return vec![
                 Event::Start(Tag::Emphasis),
                 Event::Text(CowStr::from(reference.display())),
-                Event::End(Tag::Emphasis),
+                Event::End(TagEnd::Emphasis),
             ];
         }
         let target_file = target_file.unwrap();
@@ -731,16 +729,17 @@ impl<'a> Exporter<'a> {
             link.push_str(&slugify(section));
         }
 
-        let link_tag = Tag::Link(
+        let link_tag = Tag::Link {
-            pulldown_cmark::LinkType::Inline,
+            link_type: pulldown_cmark::LinkType::Inline,
-            CowStr::from(link),
+            dest_url: CowStr::from(link),
-            CowStr::from(""),
+            title: CowStr::from(""),
-        );
+            id: CowStr::from(""),
+        };
 
         vec![
-            Event::Start(link_tag.clone()),
+            Event::Start(link_tag),
             Event::Text(CowStr::from(reference.display())),
-            Event::End(link_tag.clone()),
+            Event::End(TagEnd::Link),
         ]
     }
 }
@@ -841,8 +840,7 @@ fn reduce_to_section<'a>(events: MarkdownEvents<'a>, section: &str) -> MarkdownE
     for event in events {
         filtered_events.push(event.clone());
         match event {
-            // FIXME: This should propagate fragment_identifier and classes.
+            Event::Start(Tag::Heading { level, .. }) => {
-            Event::Start(Tag::Heading(level, _fragment_identifier, _classes)) => {
                 last_tag_was_heading = true;
                 last_level = level;
                 if currently_in_target_section && level <= section_level {
@@ -881,10 +879,11 @@ fn reduce_to_section<'a>(events: MarkdownEvents<'a>, section: &str) -> MarkdownE
 fn event_to_owned<'a>(event: Event<'_>) -> Event<'a> {
     match event {
         Event::Start(tag) => Event::Start(tag_to_owned(tag)),
-        Event::End(tag) => Event::End(tag_to_owned(tag)),
+        Event::End(tag) => Event::End(tag),
         Event::Text(cowstr) => Event::Text(CowStr::from(cowstr.into_string())),
         Event::Code(cowstr) => Event::Code(CowStr::from(cowstr.into_string())),
         Event::Html(cowstr) => Event::Html(CowStr::from(cowstr.into_string())),
+        Event::InlineHtml(cowstr) => Event::InlineHtml(CowStr::from(cowstr.into_string())),
         Event::FootnoteReference(cowstr) => {
             Event::FootnoteReference(CowStr::from(cowstr.into_string()))
         }
@@ -892,17 +891,37 @@ fn event_to_owned<'a>(event: Event<'_>) -> Event<'a> {
         Event::HardBreak => Event::HardBreak,
         Event::Rule => Event::Rule,
         Event::TaskListMarker(checked) => Event::TaskListMarker(checked),
+        Event::InlineMath(cowstr) => Event::InlineMath(CowStr::from(cowstr.into_string())),
+        Event::DisplayMath(cowstr) => Event::DisplayMath(CowStr::from(cowstr.into_string())),
     }
 }
 
 fn tag_to_owned<'a>(tag: Tag<'_>) -> Tag<'a> {
     match tag {
         Tag::Paragraph => Tag::Paragraph,
-        Tag::Heading(level, _fragment_identifier, _classes) => {
+        Tag::Heading {
-            // FIXME: This should propagate fragment_identifier and classes.
+            level: heading_level,
-            Tag::Heading(level, None, Vec::new())
+            id,
-        }
+            classes,
-        Tag::BlockQuote => Tag::BlockQuote,
+            attrs,
+        } => Tag::Heading {
+            level: heading_level,
+            id: id.map(|cowstr| CowStr::from(cowstr.into_string())),
+            classes: classes
+                .into_iter()
+                .map(|cowstr| CowStr::from(cowstr.into_string()))
+                .collect(),
+            attrs: attrs
+                .into_iter()
+                .map(|(attr, value)| {
+                    (
+                        CowStr::from(attr.into_string()),
+                        value.map(|cowstr| CowStr::from(cowstr.into_string())),
+                    )
+                })
+                .collect(),
+        },
+        Tag::BlockQuote(blockquote_kind) => Tag::BlockQuote(blockquote_kind),
         Tag::CodeBlock(codeblock_kind) => Tag::CodeBlock(codeblock_kind_to_owned(codeblock_kind)),
         Tag::List(optional) => Tag::List(optional),
         Tag::Item => Tag::Item,
@@ -916,16 +935,30 @@ fn tag_to_owned<'a>(tag: Tag<'_>) -> Tag<'a> {
         Tag::Emphasis => Tag::Emphasis,
         Tag::Strong => Tag::Strong,
         Tag::Strikethrough => Tag::Strikethrough,
-        Tag::Link(linktype, cowstr1, cowstr2) => Tag::Link(
+        Tag::Link {
-            linktype,
+            link_type,
-            CowStr::from(cowstr1.into_string()),
+            dest_url,
-            CowStr::from(cowstr2.into_string()),
+            title,
-        ),
+            id,
-        Tag::Image(linktype, cowstr1, cowstr2) => Tag::Image(
+        } => Tag::Link {
-            linktype,
+            link_type,
-            CowStr::from(cowstr1.into_string()),
+            dest_url: CowStr::from(dest_url.into_string()),
-            CowStr::from(cowstr2.into_string()),
+            title: CowStr::from(title.into_string()),
-        ),
+            id: CowStr::from(id.into_string()),
+        },
+        Tag::Image {
+            link_type,
+            dest_url,
+            title,
+            id,
+        } => Tag::Image {
+            link_type,
+            dest_url: CowStr::from(dest_url.into_string()),
+            title: CowStr::from(title.into_string()),
+            id: CowStr::from(id.into_string()),
+        },
+        Tag::HtmlBlock => Tag::HtmlBlock,
+        Tag::MetadataBlock(metadata_block_kind) => Tag::MetadataBlock(metadata_block_kind),
     }
 }
 

tests/testdata/expected/main-samples/pure-markdown-examples.md

@@ -37,7 +37,7 @@
 ~~Strikethrough~~
 
 |Table||
-|-----|--|
+|-----|-|
 |Foo|Bar|
 
 [link text](link-location.md)