Implement frontmatter based filtering (#163)

This allows limiting the notes that will be exported using `--skip-tags` and `--only-tags`

---------

Co-authored-by: Martin Heuschober <martin.heuschober@posteo.net>
Co-authored-by: Nick Groenen <nick@groenen.me>
Co-authored-by: Martin Heuschober <martin_heuschober@trimble.com>

README.md

@@ -138,9 +138,15 @@ To completely remove any frontmatter from exported notes, use `--frontmatter=nev
 
 ## Ignoring files
 
-By default, hidden files, patterns listed in `.export-ignore` as well as any files ignored by git (if your vault is part of a git repository) will be excluded from exports.
+The following files are not exported by default:
+
+* hidden files (can be adjusted with `--hidden`)
+* files mattching a pattern listed in `.export-ignore` (can be adjusted with `--ignore-file`)
+* any files that are ignored by git (can be adjusted with `--no-git`)
+* any files having `private: true` in their frontmatter (the keyword `private` can be changed with `--ignore-frontmatter-keyword`)
+* using `--skip-tags foo --skip-tags bar` will skip any files that have the tags `foo` or `bar` in their frontmatter
+* using `--only-tags foo --only-tags bar` will skip any files that **don't** have the tags `foo` or `bar` in their frontmatter
 
-These options may be adjusted with `--hidden`, `--ignore-file` and `--no-git` if desired.
 (See `--help` for more information).
 
 Notes linking to ignored notes will be unlinked (they'll only include the link text).

docs/usage-advanced.md

@@ -12,9 +12,15 @@ To completely remove any frontmatter from exported notes, use `--frontmatter=nev
 
 ## Ignoring files
 
-By default, hidden files, patterns listed in `.export-ignore` as well as any files ignored by git (if your vault is part of a git repository) will be excluded from exports.
+The following files are not exported by default:
+
+* hidden files (can be adjusted with `--hidden`)
+* files mattching a pattern listed in `.export-ignore` (can be adjusted with `--ignore-file`)
+* any files that are ignored by git (can be adjusted with `--no-git`)
+* any files having `private: true` in their frontmatter (the keyword `private` can be changed with `--ignore-frontmatter-keyword`)
+* using `--skip-tags foo --skip-tags bar` will skip any files that have the tags `foo` or `bar` in their frontmatter
+* using `--only-tags foo --only-tags bar` will skip any files that **don't** have the tags `foo` or `bar` in their frontmatter
 
-These options may be adjusted with `--hidden`, `--ignore-file` and `--no-git` if desired.
 (See `--help` for more information).
 
 Notes linking to ignored notes will be unlinked (they'll only include the link text).

src/main.rs

@@ -1,7 +1,7 @@
 use eyre::{eyre, Result};
 use gumdrop::Options;
-use obsidian_export::postprocessors::softbreaks_to_hardbreaks;
+use obsidian_export::{postprocessors::*, ExportError};
-use obsidian_export::{ExportError, Exporter, FrontmatterStrategy, WalkOptions};
+use obsidian_export::{Exporter, FrontmatterStrategy, WalkOptions};
 use std::{env, path::PathBuf};
 
 const VERSION: &str = env!("CARGO_PKG_VERSION");
@@ -39,6 +39,12 @@ struct Opts {
     )]
     ignore_file: String,
 
+    #[options(no_short, help = "Exclude files with this tag from the export")]
+    skip_tags: Vec<String>,
+
+    #[options(no_short, help = "Export only files with this tag")]
+    only_tags: Vec<String>,
+
     #[options(no_short, help = "Export hidden files", default = "false")]
     hidden: bool,
 
@@ -94,6 +100,9 @@ fn main() {
         exporter.add_postprocessor(&softbreaks_to_hardbreaks);
     }
 
+    let tags_postprocessor = filter_by_tags(args.skip_tags, args.only_tags);
+    exporter.add_postprocessor(&tags_postprocessor);
+
     if let Some(path) = args.start_at {
         exporter.start_at(path);
     }

src/postprocessors.rs

@@ -2,6 +2,7 @@
 
 use super::{Context, MarkdownEvents, PostprocessorResult};
 use pulldown_cmark::Event;
+use serde_yaml::Value;
 
 /// This postprocessor converts all soft line breaks to hard line breaks. Enabling this mimics
 /// Obsidian's _'Strict line breaks'_ setting.
@@ -16,3 +17,90 @@ pub fn softbreaks_to_hardbreaks(
     }
     PostprocessorResult::Continue
 }
+
+pub fn filter_by_tags(
+    skip_tags: Vec<String>,
+    only_tags: Vec<String>,
+) -> impl Fn(&mut Context, &mut MarkdownEvents) -> PostprocessorResult {
+    move |context: &mut Context, _events: &mut MarkdownEvents| -> PostprocessorResult {
+        match context.frontmatter.get("tags") {
+            None => filter_by_tags_(&[], &skip_tags, &only_tags),
+            Some(Value::Sequence(tags)) => filter_by_tags_(tags, &skip_tags, &only_tags),
+            _ => PostprocessorResult::Continue,
+        }
+    }
+}
+
+fn filter_by_tags_(
+    tags: &[Value],
+    skip_tags: &[String],
+    only_tags: &[String],
+) -> PostprocessorResult {
+    let skip = skip_tags
+        .iter()
+        .any(|tag| tags.contains(&Value::String(tag.to_string())));
+    let include = only_tags.is_empty()
+        || only_tags
+            .iter()
+            .any(|tag| tags.contains(&Value::String(tag.to_string())));
+
+    if skip || !include {
+        PostprocessorResult::StopAndSkipNote
+    } else {
+        PostprocessorResult::Continue
+    }
+}
+
+#[test]
+fn test_filter_tags() {
+    let tags = vec![
+        Value::String("skip".to_string()),
+        Value::String("publish".to_string()),
+    ];
+    let empty_tags = vec![];
+    assert_eq!(
+        filter_by_tags_(&empty_tags, &[], &[]),
+        PostprocessorResult::Continue,
+        "When no exclusion & inclusion are specified, files without tags are included"
+    );
+    assert_eq!(
+        filter_by_tags_(&tags, &[], &[]),
+        PostprocessorResult::Continue,
+        "When no exclusion & inclusion are specified, files with tags are included"
+    );
+    assert_eq!(
+        filter_by_tags_(&tags, &["exclude".to_string()], &[]),
+        PostprocessorResult::Continue,
+        "When exclusion tags don't match files with tags are included"
+    );
+    assert_eq!(
+        filter_by_tags_(&empty_tags, &["exclude".to_string()], &[]),
+        PostprocessorResult::Continue,
+        "When exclusion tags don't match files without tags are included"
+    );
+    assert_eq!(
+        filter_by_tags_(&tags, &[], &["publish".to_string()]),
+        PostprocessorResult::Continue,
+        "When exclusion tags don't match files with tags are included"
+    );
+    assert_eq!(
+        filter_by_tags_(&empty_tags, &[], &["include".to_string()]),
+        PostprocessorResult::StopAndSkipNote,
+        "When inclusion tags are specified files without tags are excluded"
+    );
+    assert_eq!(
+        filter_by_tags_(&tags, &[], &["include".to_string()]),
+        PostprocessorResult::StopAndSkipNote,
+        "When exclusion tags don't match files with tags are exluded"
+    );
+    assert_eq!(
+        filter_by_tags_(&tags, &["skip".to_string()], &["skip".to_string()]),
+        PostprocessorResult::StopAndSkipNote,
+        "When both inclusion and exclusion tags are the same exclusion wins"
+    );
+    assert_eq!(
+        filter_by_tags_(&tags, &["skip".to_string()], &["publish".to_string()]),
+        PostprocessorResult::StopAndSkipNote,
+        "When both inclusion and exclusion tags match exclusion wins"
+    );
+}

tests/export_test.rs

@@ -37,7 +37,7 @@ fn test_main_variants_with_default_options() {
                 entry.path().display()
             )
         });
-        let actual = read_to_string(tmp_dir.path().clone().join(PathBuf::from(&filename)))
+        let actual = read_to_string(tmp_dir.path().join(PathBuf::from(&filename)))
             .unwrap_or_else(|_| panic!("failed to read {} from temporary exportdir", filename));
 
         assert_eq!(
@@ -62,7 +62,6 @@ fn test_frontmatter_never() {
     let actual = read_to_string(
         tmp_dir
             .path()
-            .clone()
             .join(PathBuf::from("note-with-frontmatter.md")),
     )
     .unwrap();
@@ -85,7 +84,6 @@ fn test_frontmatter_always() {
     let actual = read_to_string(
         tmp_dir
             .path()
-            .clone()
             .join(PathBuf::from("note-without-frontmatter.md")),
     )
     .unwrap();
@@ -96,7 +94,6 @@ fn test_frontmatter_always() {
     let actual = read_to_string(
         tmp_dir
             .path()
-            .clone()
             .join(PathBuf::from("note-with-frontmatter.md")),
     )
     .unwrap();
@@ -114,10 +111,7 @@ fn test_exclude() {
     .run()
     .expect("exporter returned error");
 
-    let excluded_note = tmp_dir
+    let excluded_note = tmp_dir.path().join(PathBuf::from("excluded-note.md"));
-        .path()
-        .clone()
-        .join(PathBuf::from("excluded-note.md"));
     assert!(
         !excluded_note.exists(),
         "exluded-note.md was found in tmpdir, but should be absent due to .export-ignore rules"
@@ -136,14 +130,14 @@ fn test_single_file_to_dir() {
 
     assert_eq!(
         read_to_string("tests/testdata/expected/single-file/note.md").unwrap(),
-        read_to_string(tmp_dir.path().clone().join(PathBuf::from("note.md"))).unwrap(),
+        read_to_string(tmp_dir.path().join(PathBuf::from("note.md"))).unwrap(),
     );
 }
 
 #[test]
 fn test_single_file_to_file() {
     let tmp_dir = TempDir::new().expect("failed to make tempdir");
-    let dest = tmp_dir.path().clone().join(PathBuf::from("export.md"));
+    let dest = tmp_dir.path().join(PathBuf::from("export.md"));
 
     Exporter::new(
         PathBuf::from("tests/testdata/input/single-file/note.md"),
@@ -178,7 +172,7 @@ fn test_start_at_subdir() {
 
     assert_eq!(
         expected,
-        read_to_string(tmp_dir.path().clone().join(PathBuf::from("Note B.md"))).unwrap(),
+        read_to_string(tmp_dir.path().join(PathBuf::from("Note B.md"))).unwrap(),
     );
 }
 
@@ -204,14 +198,14 @@ fn test_start_at_file_within_subdir_destination_is_dir() {
 
     assert_eq!(
         expected,
-        read_to_string(tmp_dir.path().clone().join(PathBuf::from("Note B.md"))).unwrap(),
+        read_to_string(tmp_dir.path().join(PathBuf::from("Note B.md"))).unwrap(),
     );
 }
 
 #[test]
 fn test_start_at_file_within_subdir_destination_is_file() {
     let tmp_dir = TempDir::new().expect("failed to make tempdir");
-    let dest = tmp_dir.path().clone().join(PathBuf::from("note.md"));
+    let dest = tmp_dir.path().join(PathBuf::from("note.md"));
     let mut exporter = Exporter::new(
         PathBuf::from("tests/testdata/input/start-at/"),
         dest.clone(),
@@ -360,7 +354,7 @@ fn test_no_recursive_embeds() {
 
     assert_eq!(
         read_to_string("tests/testdata/expected/infinite-recursion/Note A.md").unwrap(),
-        read_to_string(tmp_dir.path().clone().join(PathBuf::from("Note A.md"))).unwrap(),
+        read_to_string(tmp_dir.path().join(PathBuf::from("Note A.md"))).unwrap(),
     );
 }
 
@@ -392,7 +386,7 @@ fn test_non_ascii_filenames() {
                 entry.path().display()
             )
         });
-        let actual = read_to_string(tmp_dir.path().clone().join(PathBuf::from(&filename)))
+        let actual = read_to_string(tmp_dir.path().join(PathBuf::from(&filename)))
             .unwrap_or_else(|_| panic!("failed to read {} from temporary exportdir", filename));
 
         assert_eq!(
@@ -422,6 +416,6 @@ fn test_same_filename_different_directories() {
             .unwrap()
     };
 
-    let actual = read_to_string(tmp_dir.path().clone().join(PathBuf::from("Note.md"))).unwrap();
+    let actual = read_to_string(tmp_dir.path().join(PathBuf::from("Note.md"))).unwrap();
     assert_eq!(expected, actual);
 }

tests/postprocessors_test.rs

@@ -1,4 +1,4 @@
-use obsidian_export::postprocessors::softbreaks_to_hardbreaks;
+use obsidian_export::postprocessors::{filter_by_tags, softbreaks_to_hardbreaks};
 use obsidian_export::{Context, Exporter, MarkdownEvents, PostprocessorResult};
 use pretty_assertions::assert_eq;
 use pulldown_cmark::{CowStr, Event};
@@ -8,6 +8,7 @@ use std::fs::{read_to_string, remove_file};
 use std::path::PathBuf;
 use std::sync::Mutex;
 use tempfile::TempDir;
+use walkdir::WalkDir;
 
 /// This postprocessor replaces any instance of "foo" with "bar" in the note body.
 fn foo_to_bar(_ctx: &mut Context, events: &mut MarkdownEvents) -> PostprocessorResult {
@@ -44,7 +45,7 @@ fn test_postprocessors() {
     exporter.run().unwrap();
 
     let expected = read_to_string("tests/testdata/expected/postprocessors/Note.md").unwrap();
-    let actual = read_to_string(tmp_dir.path().clone().join(PathBuf::from("Note.md"))).unwrap();
+    let actual = read_to_string(tmp_dir.path().join(PathBuf::from("Note.md"))).unwrap();
     assert_eq!(expected, actual);
 }
 
@@ -66,7 +67,7 @@ fn test_postprocessor_stophere() {
 #[test]
 fn test_postprocessor_stop_and_skip() {
     let tmp_dir = TempDir::new().expect("failed to make tempdir");
-    let note_path = tmp_dir.path().clone().join(PathBuf::from("Note.md"));
+    let note_path = tmp_dir.path().join(PathBuf::from("Note.md"));
 
     let mut exporter = Exporter::new(
         PathBuf::from("tests/testdata/input/postprocessors"),
@@ -86,7 +87,7 @@ fn test_postprocessor_stop_and_skip() {
 #[test]
 fn test_postprocessor_change_destination() {
     let tmp_dir = TempDir::new().expect("failed to make tempdir");
-    let original_note_path = tmp_dir.path().clone().join(PathBuf::from("Note.md"));
+    let original_note_path = tmp_dir.path().join(PathBuf::from("Note.md"));
     let mut exporter = Exporter::new(
         PathBuf::from("tests/testdata/input/postprocessors"),
         tmp_dir.path().to_path_buf(),
@@ -102,7 +103,7 @@ fn test_postprocessor_change_destination() {
     });
     exporter.run().unwrap();
 
-    let new_note_path = tmp_dir.path().clone().join(PathBuf::from("MovedNote.md"));
+    let new_note_path = tmp_dir.path().join(PathBuf::from("MovedNote.md"));
     assert!(!original_note_path.exists());
     assert!(new_note_path.exists());
 }
@@ -131,7 +132,7 @@ fn test_postprocessor_stateful_callback() {
 
     exporter.run().unwrap();
 
-    let expected = tmp_dir.path().clone();
+    let expected = tmp_dir.path();
 
     let parents = parents.lock().unwrap();
     println!("{:?}", parents);
@@ -158,7 +159,7 @@ fn test_embed_postprocessors() {
     let expected =
         read_to_string("tests/testdata/expected/postprocessors/Note_embed_postprocess_only.md")
             .unwrap();
-    let actual = read_to_string(tmp_dir.path().clone().join(PathBuf::from("Note.md"))).unwrap();
+    let actual = read_to_string(tmp_dir.path().join(PathBuf::from("Note.md"))).unwrap();
     assert_eq!(expected, actual);
 }
 
@@ -178,7 +179,7 @@ fn test_embed_postprocessors_stop_and_skip() {
     let expected =
         read_to_string("tests/testdata/expected/postprocessors/Note_embed_stop_and_skip.md")
             .unwrap();
-    let actual = read_to_string(tmp_dir.path().clone().join(PathBuf::from("Note.md"))).unwrap();
+    let actual = read_to_string(tmp_dir.path().join(PathBuf::from("Note.md"))).unwrap();
     assert_eq!(expected, actual);
 }
 
@@ -244,12 +245,48 @@ fn test_softbreaks_to_hardbreaks() {
 
     let expected =
         read_to_string("tests/testdata/expected/postprocessors/hard_linebreaks.md").unwrap();
-    let actual = read_to_string(
+    let actual = read_to_string(tmp_dir.path().join(PathBuf::from("hard_linebreaks.md"))).unwrap();
-        tmp_dir
-            .path()
-            .clone()
-            .join(PathBuf::from("hard_linebreaks.md")),
-    )
-    .unwrap();
     assert_eq!(expected, actual);
 }
+
+#[test]
+fn test_filter_by_tags() {
+    let tmp_dir = TempDir::new().expect("failed to make tempdir");
+    let mut exporter = Exporter::new(
+        PathBuf::from("tests/testdata/input/filter-by-tags"),
+        tmp_dir.path().to_path_buf(),
+    );
+    let filter_by_tags = filter_by_tags(
+        vec!["private".to_string(), "no-export".to_string()],
+        vec!["export".to_string()],
+    );
+    exporter.add_postprocessor(&filter_by_tags);
+    exporter.run().unwrap();
+
+    let walker = WalkDir::new("tests/testdata/expected/filter-by-tags/")
+        // Without sorting here, different test runs may trigger the first assertion failure in
+        // unpredictable order.
+        .sort_by(|a, b| a.file_name().cmp(b.file_name()))
+        .into_iter();
+    for entry in walker {
+        let entry = entry.unwrap();
+        if entry.metadata().unwrap().is_dir() {
+            continue;
+        };
+        let filename = entry.file_name().to_string_lossy().into_owned();
+        let expected = read_to_string(entry.path()).unwrap_or_else(|_| {
+            panic!(
+                "failed to read {} from testdata/expected/filter-by-tags",
+                entry.path().display()
+            )
+        });
+        let actual = read_to_string(tmp_dir.path().join(PathBuf::from(&filename)))
+            .unwrap_or_else(|_| panic!("failed to read {} from temporary exportdir", filename));
+
+        assert_eq!(
+            expected, actual,
+            "{} does not have expected content",
+            filename
+        );
+    }
+}

tests/testdata/expected/filter-by-tags/export-me.md

@@ -0,0 +1,7 @@
+---
+tags:
+- export
+- me
+---
+
+A public note

tests/testdata/expected/filter-by-tags/export.md

@@ -0,0 +1,6 @@
+---
+tags:
+- export
+---
+
+A public note

tests/testdata/input/filter-by-tags/export-me.md

@@ -0,0 +1,5 @@
+---
+tags: [export, me]
+---
+
+A public note

tests/testdata/input/filter-by-tags/export-no-export.md

@@ -0,0 +1,5 @@
+---
+tags: [export, no-export, private]
+---
+
+A private note

tests/testdata/input/filter-by-tags/export.md

@@ -0,0 +1,5 @@
+---
+tags: [export]
+---
+
+A public note

tests/testdata/input/filter-by-tags/no-frontmatter.md

@@ -0,0 +1 @@
+A note without frontmatter should be exported.

tests/testdata/input/filter-by-tags/no-no-export.md

@@ -0,0 +1,5 @@
+---
+tags: [no, no-export]
+---
+
+A private note

tests/testdata/input/filter-by-tags/no-tags.md

@@ -0,0 +1,5 @@
+---
+title: foo
+---
+
+A public note.

tests/testdata/input/filter-by-tags/private.md

@@ -0,0 +1,5 @@
+---
+tags: [private]
+---
+
+A private note.