From c2de7761480aa5a9d2b8167c8ab643011cbe6d93 Mon Sep 17 00:00:00 2001 From: Nick Groenen Date: Sat, 28 Nov 2020 13:14:50 +0100 Subject: [PATCH] Public release --- .github/dependabot.yml | 12 + .github/workflows/release.yml | 125 ++++ .github/workflows/test.yml | 80 +++ .gitignore | 9 + .pre-commit-config.yaml | 14 + Cargo.lock | 639 ++++++++++++++++++ Cargo.toml | 43 ++ LICENSE-APACHE | 201 ++++++ LICENSE-MIT | 25 + README.md | 100 +++ book/README.md | 3 + book/book-src/Installation.md | 24 + book/book-src/License.md | 3 + book/book-src/README.md | 100 +++ book/book-src/SUMMARY.md | 6 + book/book-src/Usage.md | 18 + book/book-src/index.md | 12 + book/book-src/installation.md | 17 + book/book-src/license.md | 3 + book/book-src/usage.md | 62 ++ book/book.toml | 6 + book/obsidian-src/.obsidian/config | 1 + book/obsidian-src/.obsidian/workspaces.json | 1 + book/obsidian-src/README.md | 4 + book/obsidian-src/SUMMARY.md | 6 + book/obsidian-src/index.md | 16 + book/obsidian-src/installation.md | 18 + book/obsidian-src/license.md | 6 + book/obsidian-src/usage.md | 65 ++ src/lib.rs | 555 +++++++++++++++ src/main.rs | 48 ++ src/walker.rs | 58 ++ tests/export_test.rs | 335 +++++++++ .../testdata/expected/main-samples/embeds.md | 29 + .../main-samples/note-with-frontmatter.md | 5 + .../main-samples/note-without-frontmatter.md | 1 + .../main-samples/obsidian-wikilinks.md | 14 + .../main-samples/partial-embed-at-EOF.md | 1 + .../main-samples/pure-markdown-examples.md | 48 ++ tests/testdata/expected/non-ascii/Embed.md | 3 + tests/testdata/expected/non-ascii/Link.md | 1 + .../expected/non-ascii/Note with 'quotes'.md | 1 + .../Note.md | 7 + .../dir1/Note.md | 1 + .../dir2/Note.md | 1 + tests/testdata/expected/single-file/note.md | 3 + .../testdata/input/infinite-recursion/note.md | 1 + .../input/main-samples/.export-ignore | 1 + tests/testdata/input/main-samples/embeds.md | 28 + .../input/main-samples/excluded-note.md | 1 + tests/testdata/input/main-samples/foo.md | 1 + .../main-samples/note-with-frontmatter.md | 5 + .../main-samples/note-without-frontmatter.md | 1 + tests/testdata/input/main-samples/note.pdf | Bin 0 -> 10435 bytes .../input/main-samples/obsidian-wikilinks.md | 14 + .../main-samples/partial-embed-at-EOF.md | 1 + .../main-samples/pure-markdown-examples.md | 46 ++ tests/testdata/input/main-samples/white.png | Bin 0 -> 90 bytes tests/testdata/input/non-ascii/Embed.md | 3 + tests/testdata/input/non-ascii/Link.md | 1 + .../input/non-ascii/Note with 'quotes'.md | 1 + .../Note.md | 7 + .../dir1/Note.md | 1 + .../dir2/Note.md | 1 + tests/testdata/input/single-file/note.md | 3 + .../testdata/input/single-file/other-note.md | 1 + 66 files changed, 2847 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/test.yml create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 LICENSE-APACHE create mode 100644 LICENSE-MIT create mode 100644 README.md create mode 100644 book/README.md create mode 100644 book/book-src/Installation.md create mode 100644 book/book-src/License.md create mode 100644 book/book-src/README.md create mode 100644 book/book-src/SUMMARY.md create mode 100644 book/book-src/Usage.md create mode 100644 book/book-src/index.md create mode 100644 book/book-src/installation.md create mode 100644 book/book-src/license.md create mode 100644 book/book-src/usage.md create mode 100644 book/book.toml create mode 100644 book/obsidian-src/.obsidian/config create mode 100644 book/obsidian-src/.obsidian/workspaces.json create mode 100644 book/obsidian-src/README.md create mode 100644 book/obsidian-src/SUMMARY.md create mode 100644 book/obsidian-src/index.md create mode 100644 book/obsidian-src/installation.md create mode 100644 book/obsidian-src/license.md create mode 100644 book/obsidian-src/usage.md create mode 100644 src/lib.rs create mode 100644 src/main.rs create mode 100644 src/walker.rs create mode 100644 tests/export_test.rs create mode 100644 tests/testdata/expected/main-samples/embeds.md create mode 100644 tests/testdata/expected/main-samples/note-with-frontmatter.md create mode 100644 tests/testdata/expected/main-samples/note-without-frontmatter.md create mode 100644 tests/testdata/expected/main-samples/obsidian-wikilinks.md create mode 100644 tests/testdata/expected/main-samples/partial-embed-at-EOF.md create mode 100644 tests/testdata/expected/main-samples/pure-markdown-examples.md create mode 100644 tests/testdata/expected/non-ascii/Embed.md create mode 100644 tests/testdata/expected/non-ascii/Link.md create mode 100644 tests/testdata/expected/non-ascii/Note with 'quotes'.md create mode 100644 tests/testdata/expected/same-filename-different-directories/Note.md create mode 100644 tests/testdata/expected/same-filename-different-directories/dir1/Note.md create mode 100644 tests/testdata/expected/same-filename-different-directories/dir2/Note.md create mode 100644 tests/testdata/expected/single-file/note.md create mode 100644 tests/testdata/input/infinite-recursion/note.md create mode 100644 tests/testdata/input/main-samples/.export-ignore create mode 100644 tests/testdata/input/main-samples/embeds.md create mode 100644 tests/testdata/input/main-samples/excluded-note.md create mode 100644 tests/testdata/input/main-samples/foo.md create mode 100644 tests/testdata/input/main-samples/note-with-frontmatter.md create mode 100644 tests/testdata/input/main-samples/note-without-frontmatter.md create mode 100644 tests/testdata/input/main-samples/note.pdf create mode 100644 tests/testdata/input/main-samples/obsidian-wikilinks.md create mode 100644 tests/testdata/input/main-samples/partial-embed-at-EOF.md create mode 100644 tests/testdata/input/main-samples/pure-markdown-examples.md create mode 100644 tests/testdata/input/main-samples/white.png create mode 100644 tests/testdata/input/non-ascii/Embed.md create mode 100644 tests/testdata/input/non-ascii/Link.md create mode 100644 tests/testdata/input/non-ascii/Note with 'quotes'.md create mode 100644 tests/testdata/input/same-filename-different-directories/Note.md create mode 100644 tests/testdata/input/same-filename-different-directories/dir1/Note.md create mode 100644 tests/testdata/input/same-filename-different-directories/dir2/Note.md create mode 100644 tests/testdata/input/single-file/note.md create mode 100644 tests/testdata/input/single-file/other-note.md diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..f2b43eb --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + - package-ecosystem: cargo + directory: "/" + schedule: + interval: weekly + open-pull-requests-limit: 15 + + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: monthly diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..45bce97 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,125 @@ +on: + push: + tags: + - "v*" + +name: Create release + +jobs: + create-release: + name: Create release + runs-on: ubuntu-latest + outputs: + upload_url: "${{ steps.create_release.outputs.upload_url }}" + steps: + - id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: Release ${{ github.ref }} + draft: true + prerelease: false + + build-linux: + name: Linux binary + needs: create-release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + + - uses: actions-rs/cargo@v1 + with: + command: build + args: --release --locked + + - run: strip target/release/obsidian-export + + - uses: actions/upload-artifact@v2 + with: + name: Linux binary + path: target/release/obsidian-export + retention-days: 7 + + - uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create-release.outputs.upload_url }} + asset_path: target/release/obsidian-export + asset_name: obsidian-export_Linux-x86_64 + asset_content_type: application/octet-stream + + build-windows: + name: Windows binary + needs: create-release + runs-on: windows-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + + - uses: actions-rs/cargo@v1 + with: + command: build + args: --release --locked + + - run: strip target/release/obsidian-export.exe + + - uses: actions/upload-artifact@v2 + with: + name: Windows binary + path: target/release/obsidian-export.exe + retention-days: 7 + + - uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create-release.outputs.upload_url }} + asset_path: target/release/obsidian-export.exe + asset_name: obsidian-export_Windows-x64_64 + asset_content_type: application/octet-stream + + build-macos: + name: Mac OS binary + needs: create-release + runs-on: macos-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + + - uses: actions-rs/cargo@v1 + with: + command: build + args: --release --locked + + - run: strip target/release/obsidian-export + + - uses: actions/upload-artifact@v2 + with: + name: MacOS binary + path: target/release/obsidian-export + retention-days: 7 + + - uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create-release.outputs.upload_url }} + asset_path: target/release/obsidian-export + asset_name: obsidian-export_MacOS-x86_64 + asset_content_type: application/octet-stream diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..a67dc43 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,80 @@ +on: [push] + +name: CI tests + +jobs: + linting: + name: Run lints + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + + - run: rustup component add rustfmt + - uses: actions-rs/cargo@v1 + with: + command: fmt + args: --all -- --check + + - uses: actions-rs/cargo@v1 + with: + command: check + + - run: rustup component add clippy + - uses: actions-rs/cargo@v1 + with: + command: clippy + args: -- -D warnings + + test-linux: + name: Test on Linux + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - uses: actions-rs/cargo@v1 + with: + command: test + + test-windows: + name: Test on Windows + runs-on: windows-latest + steps: + - run: git config --system core.autocrlf false && git config --system core.eol lf + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - uses: actions-rs/cargo@v1 + with: + command: test + + coverage: + name: Code coverage + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - uses: actions-rs/tarpaulin@v0.1 + with: + version: "latest" + args: "--ignore-tests" + out-type: "Html" + - uses: actions/upload-artifact@v2 + with: + name: tarpaulin-report + path: tarpaulin-report.html diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5c8a7a5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +/target + +# Generated mdBook HTML +/book/book + +/book/obsidian-src/.obsidian/workspace + +# Tarpaulin (https://github.com/xd009642/tarpaulin) HTML coverage report +tarpaulin-report.html diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..910714c --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,14 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: 9136088a246768144165fcc3ecc3d31bb686920a # frozen: v3.3.0 + hooks: + - id: check-yaml + - repo: https://github.com/doublify/pre-commit-rust + rev: eeee35a89e69d5772bdee97db1a6a898467b686e # frozen: v1.0 + hooks: + - id: fmt + - id: cargo-check + - id: clippy + args: ["--", "-D", "warnings"] diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..0dd5b29 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,639 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "aho-corasick" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +dependencies = [ + "memchr", +] + +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +dependencies = [ + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "bstr" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473fc6b38233f9af7baa94fb5852dca389e3d95b8e21c8e3719301462c5d9faf" +dependencies = [ + "memchr", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "const_fn" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c478836e029dcef17fb47c89023448c64f781a046e0300e257ad8225ae59afab" + +[[package]] +name = "crossbeam-channel" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d" +dependencies = [ + "cfg-if 1.0.0", + "const_fn", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "lazy_static", +] + +[[package]] +name = "ctor" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fbaabec2c953050352311293be5c6aba8e141ba19d6811862b232d6fd020484" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "difference" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" + +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "eyre" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f29abf4740a4778632fe27a4f681ef5b7a6f659aeba3330ac66f48e20cfa3b7" +dependencies = [ + "indenter", + "once_cell", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "getopts" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "wasi", +] + +[[package]] +name = "globset" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c152169ef1e421390738366d2f796655fec62621dabbd0fd476f905934061e4a" +dependencies = [ + "aho-corasick", + "bstr", + "fnv", + "log", + "regex", +] + +[[package]] +name = "gumdrop" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46571f5d540478cf70d2a42dd0d6d8e9f4b9cc7531544b93311e657b86568a0b" +dependencies = [ + "gumdrop_derive", +] + +[[package]] +name = "gumdrop_derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915ef07c710d84733522461de2a734d4d62a3fd39a4d4f404c2f385ef8618d05" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "hermit-abi" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" +dependencies = [ + "libc", +] + +[[package]] +name = "ignore" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b287fb45c60bb826a0dc68ff08742b9d88a2fea13d6e0c286b3172065aaf878c" +dependencies = [ + "crossbeam-utils", + "globset", + "lazy_static", + "log", + "memchr", + "regex", + "same-file", + "thread_local", + "walkdir", + "winapi-util", +] + +[[package]] +name = "indenter" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0bd112d44d9d870a6819eb505d04dd92b5e4d94bb8c304924a0872ae7016fb5" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" + +[[package]] +name = "log" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +dependencies = [ + "cfg-if 0.1.10", +] + +[[package]] +name = "matter" +version = "0.1.0-alpha4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc16e839c57e0ad77957c42d39baab3692a1c6fa47692066470cddc24a5b0cd0" +dependencies = [ + "lazy_static", + "regex", +] + +[[package]] +name = "memchr" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" + +[[package]] +name = "memoffset" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "obsidian-export" +version = "0.1.0" +dependencies = [ + "eyre", + "gumdrop", + "ignore", + "lazy_static", + "matter", + "pathdiff", + "percent-encoding", + "pretty_assertions", + "pulldown-cmark", + "pulldown-cmark-to-cmark", + "rayon", + "regex", + "snafu", + "tempfile", + "walkdir", +] + +[[package]] +name = "once_cell" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0" + +[[package]] +name = "output_vt100" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53cdc5b785b7a58c5aad8216b3dfa114df64b0b06ae6e1501cef91df2fbdf8f9" +dependencies = [ + "winapi", +] + +[[package]] +name = "pathdiff" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877630b3de15c0b64cc52f659345724fbf6bdad9bd9566699fc53688f3c34a34" + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "pretty_assertions" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f81e1644e1b54f5a68959a29aa86cde704219254669da328ecfdf6a1f09d427" +dependencies = [ + "ansi_term", + "ctor", + "difference", + "output_vt100", +] + +[[package]] +name = "proc-macro2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "pulldown-cmark" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffade02495f22453cd593159ea2f59827aae7f53fa8323f756799b670881dcf8" +dependencies = [ + "bitflags", + "getopts", + "memchr", + "unicase", +] + +[[package]] +name = "pulldown-cmark-to-cmark" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8f2b9878102358ec65434fdd1a9a161f8648bb2f531acc9260e4d094c96de23" +dependencies = [ + "pulldown-cmark", +] + +[[package]] +name = "quote" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom", + "libc", + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rayon" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "lazy_static", + "num_cpus", +] + +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "regex" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-syntax" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "snafu" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eab12d3c261b2308b0d80c26fffb58d17eba81a4be97890101f416b478c79ca7" +dependencies = [ + "doc-comment", + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1508efa03c362e23817f96cde18abed596a25219a8b2c66e8db33c03543d315b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "syn" +version = "1.0.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2af957a63d6bd42255c359c93d9bfdb97076bd3b820897ce55ffbfbf107f44" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tempfile" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "rand", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "version_check" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" + +[[package]] +name = "walkdir" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..3a162a8 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "obsidian-export" +version = "0.1.0" +authors = ["Nick Groenen "] +edition = "2018" +license = "MIT OR Apache-2.0" +readme = "README.md" +repository = "https://github.com/zoni/obsidian-export" +documentation = "https://docs.rs/obsidian-export" +description = """ +Rust library and associated CLI program to export an Obsidian vault to regular Markdown. +""" +categories = ["command-line-utilities", "text-processing"] +keywords = ["markdown", "obsidian"] + +[lib] +name = "obsidian_export" +path = "src/lib.rs" + +[[bin]] +name = "obsidian-export" +path = "src/main.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +eyre = "0.6.3" +gumdrop = "0.8.0" +ignore = "0.4.17" +lazy_static = "1.4.0" +matter = "0.1.0-alpha4" +pathdiff = "0.2.0" +percent-encoding = "2.1.0" +pulldown-cmark = "0.8.0" +pulldown-cmark-to-cmark = "6.0.0" +rayon = "1.5.0" +regex = "1.4.2" +snafu = "0.6.10" + +[dev-dependencies] +pretty_assertions = "0.6.1" +tempfile = "3.1.0" +walkdir = "2.3.1" diff --git a/LICENSE-APACHE b/LICENSE-APACHE new file mode 100644 index 0000000..16fe87b --- /dev/null +++ b/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/LICENSE-MIT b/LICENSE-MIT new file mode 100644 index 0000000..269c89d --- /dev/null +++ b/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2020 Nick Groenen + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..acee440 --- /dev/null +++ b/README.md @@ -0,0 +1,100 @@ +# Obsidian Export + +*Rust library and associated CLI program to export an [Obsidian](https://obsidian.md/) vault to regular Markdown (specifically: [CommonMark](https://commonmark.org/))* + +* Recursively export Obsidian Markdown files to CommonMark. +* Supports `[[note]]`-style references as well as `![[note]]` file includes. + * `[[note#heading]]` linking/embedding not yet supported, but planned. +* Support for [gitignore](https://git-scm.com/docs/gitignore)-style exclude patterns (default: `.export-ignore`). +* Automatically excludes files that are ignored by Git when the vault is located in a Git repository. + +Please note obsidian-export is not officially endorsed by the Obsidian team. +It supports most but not all of Obsidian's Markdown flavor. + + +## Installation + + > + > **Note**: + > *Obsidian-export* has been developed on Linux. + > Windows and Mac OS are covered as part of the continuous integration tests run on GitHub, but these have not been tested by the author. + > Experience reports from users on these operating systems would be welcomed. + +Binary releases for x86-64 processors are provided for Windows, Linux and Mac operating systems on a best-effort basis. +These may be downloaded from: [https://github.com/zoni/obsidian-export/releases](https://github.com/zoni/obsidian-export/releases) + +Alternatively, *obsidian-export* may be compiled from source using [Cargo](https://doc.rust-lang.org/cargo/), the official package manager for Rust, by using the following steps: + +1. Install the Rust toolchain: [https://www.rust-lang.org/tools/install](https://www.rust-lang.org/tools/install) +1. Run: `cargo install https://github.com/zoni/obsidian-export.git --locked` + +The same `cargo install` command can later be used to upgrade to a newer release as well. + + +## Usage + +The main interface of *obsidian-export* is the `obsidian-export` CLI command. +In it's most basic form, `obsidian-export` takes just two mandatory arguments, a source and a destination: + +````sh +obsidian-export my-obsidian-vault /tmp/export +```` + +This will export all of the files from `my-obsidian-vault` to `/tmp/export`, except for those listed in `.export-ignore` or `.gitignore`. + +It is also possible to export individual files: + +````sh +# Export as some-note.md to /tmp/export/ +obsidian-export my-obsidian-vault/some-note.md /tmp/export/ +# Export as exported-note.md in /tmp/ +obsidian-export my-obsidian-vault/some-note.md /tmp/exported-note.md +```` + +### Character encodings + +At present, UTF-8 character encoding is assumed for all note text as well as filenames. +All text and file handling performs [lossy conversion to Unicode strings](https://doc.rust-lang.org/std/string/struct.String.html#method.from_utf8_lossy). + +Use of non-UTF8 encodings may lead to issues like incorrect text replacement and failure to find linked notes. +While this may change in the future, there are no plans to change this behavior in the short term. + +### Frontmatter + +By default, frontmatter is copied over "as-is". + +Some static site generators are picky about frontmatter and require it to be present. +Some get tripped up when Markdown files don't have frontmatter but start with a list item or horizontal rule. +In these cases, `--frontmatter=always` can be used to insert an empty frontmatter entry. + +To completely remove any frontmatter from exported notes, use `--frontmatter=never`. + +### Ignoring files + +By default, hidden files, patterns listed in `.export-ignore` as well as any files ignored by git (if your vault is part of a git repository) will be excluded from exports. +These options will become configurable in the next release. + +Notes linking to ignored notes will be unlinked (they'll only include the link text). Embeds of ignored notes will be skipped entirely. + +#### Ignorefile syntax + +The syntax for `.export-ignore` files is identical to that of [gitignore](https://git-scm.com/docs/gitignore) files. +Here's an example: + +```` +# Ignore the directory private that is located at the top of the export tree +/private +# Ignore any file or directory called `test` +test +# Ignore any PDF file +*.pdf +# ..but include special.pdf +!special.pdf +```` + +For more comprehensive documentation and examples, see the [gitignore](https://git-scm.com/docs/gitignore) manpage. + + +## License + +Obsidian-export is dual-licensed under the [Apache 2.0](https://github.com/zoni/obsidian-export/blob/master/LICENSE-APACHE) and the [MIT](https://github.com/zoni/obsidian-export/blob/master/LICENSE-MIT) licenses. diff --git a/book/README.md b/book/README.md new file mode 100644 index 0000000..06cb0b6 --- /dev/null +++ b/book/README.md @@ -0,0 +1,3 @@ +This is an example of using obsidian-export to build documentation with the source stored as an Obsidian vault. + +The project level README.md is generated using this setup. diff --git a/book/book-src/Installation.md b/book/book-src/Installation.md new file mode 100644 index 0000000..b46b7b9 --- /dev/null +++ b/book/book-src/Installation.md @@ -0,0 +1,24 @@ +## Installation + +I don't currently provide binary releases, though I may create these if there is sufficient demand. +Until then, users can install *obsidian-export* from source using [Cargo](https://doc.rust-lang.org/cargo/): + +1. Install the Rust toolchain: [https://www.rust-lang.org/tools/install](https://www.rust-lang.org/tools/install) +1. Run: `cargo install https://github.com/zoni/obsidian-export.git --locked` + +The same `cargo install` command can be used to upgrade to a newer version as well. + +### Supported Operating Systems + +Obsidian-export has only been tested on GNU/Linux, but should run on any modern Unix-like system. + +Windows has not been tested and is unsupported at this time. +Experience reports from Windows users would be welcome however, and Windows support may be considered if the current UTF-8 filename assumption (see below) can hold true on Windows. + +### Character encodings + +At present, UTF-8 character encoding is assumed for all note text as well as filenames. +All text and file handling performs [lossy conversion to Unicode strings](https://doc.rust-lang.org/std/string/struct.String.html#method.from_utf8_lossy). + +Use of non-UTF8 encodings may lead to issues like incorrect text replacement and failure to find linked notes. +While this may change in the future, there are no plans to change this behavior on the short term. diff --git a/book/book-src/License.md b/book/book-src/License.md new file mode 100644 index 0000000..5078806 --- /dev/null +++ b/book/book-src/License.md @@ -0,0 +1,3 @@ +## License + +Obsidian-export is dual-licensed under the [Apache 2.0](https://github.com/zoni/obsidian-export/blob/master/LICENSE-APACHE) and the [MIT](https://github.com/zoni/obsidian-export/blob/master/LICENSE-MIT) licenses. diff --git a/book/book-src/README.md b/book/book-src/README.md new file mode 100644 index 0000000..acee440 --- /dev/null +++ b/book/book-src/README.md @@ -0,0 +1,100 @@ +# Obsidian Export + +*Rust library and associated CLI program to export an [Obsidian](https://obsidian.md/) vault to regular Markdown (specifically: [CommonMark](https://commonmark.org/))* + +* Recursively export Obsidian Markdown files to CommonMark. +* Supports `[[note]]`-style references as well as `![[note]]` file includes. + * `[[note#heading]]` linking/embedding not yet supported, but planned. +* Support for [gitignore](https://git-scm.com/docs/gitignore)-style exclude patterns (default: `.export-ignore`). +* Automatically excludes files that are ignored by Git when the vault is located in a Git repository. + +Please note obsidian-export is not officially endorsed by the Obsidian team. +It supports most but not all of Obsidian's Markdown flavor. + + +## Installation + + > + > **Note**: + > *Obsidian-export* has been developed on Linux. + > Windows and Mac OS are covered as part of the continuous integration tests run on GitHub, but these have not been tested by the author. + > Experience reports from users on these operating systems would be welcomed. + +Binary releases for x86-64 processors are provided for Windows, Linux and Mac operating systems on a best-effort basis. +These may be downloaded from: [https://github.com/zoni/obsidian-export/releases](https://github.com/zoni/obsidian-export/releases) + +Alternatively, *obsidian-export* may be compiled from source using [Cargo](https://doc.rust-lang.org/cargo/), the official package manager for Rust, by using the following steps: + +1. Install the Rust toolchain: [https://www.rust-lang.org/tools/install](https://www.rust-lang.org/tools/install) +1. Run: `cargo install https://github.com/zoni/obsidian-export.git --locked` + +The same `cargo install` command can later be used to upgrade to a newer release as well. + + +## Usage + +The main interface of *obsidian-export* is the `obsidian-export` CLI command. +In it's most basic form, `obsidian-export` takes just two mandatory arguments, a source and a destination: + +````sh +obsidian-export my-obsidian-vault /tmp/export +```` + +This will export all of the files from `my-obsidian-vault` to `/tmp/export`, except for those listed in `.export-ignore` or `.gitignore`. + +It is also possible to export individual files: + +````sh +# Export as some-note.md to /tmp/export/ +obsidian-export my-obsidian-vault/some-note.md /tmp/export/ +# Export as exported-note.md in /tmp/ +obsidian-export my-obsidian-vault/some-note.md /tmp/exported-note.md +```` + +### Character encodings + +At present, UTF-8 character encoding is assumed for all note text as well as filenames. +All text and file handling performs [lossy conversion to Unicode strings](https://doc.rust-lang.org/std/string/struct.String.html#method.from_utf8_lossy). + +Use of non-UTF8 encodings may lead to issues like incorrect text replacement and failure to find linked notes. +While this may change in the future, there are no plans to change this behavior in the short term. + +### Frontmatter + +By default, frontmatter is copied over "as-is". + +Some static site generators are picky about frontmatter and require it to be present. +Some get tripped up when Markdown files don't have frontmatter but start with a list item or horizontal rule. +In these cases, `--frontmatter=always` can be used to insert an empty frontmatter entry. + +To completely remove any frontmatter from exported notes, use `--frontmatter=never`. + +### Ignoring files + +By default, hidden files, patterns listed in `.export-ignore` as well as any files ignored by git (if your vault is part of a git repository) will be excluded from exports. +These options will become configurable in the next release. + +Notes linking to ignored notes will be unlinked (they'll only include the link text). Embeds of ignored notes will be skipped entirely. + +#### Ignorefile syntax + +The syntax for `.export-ignore` files is identical to that of [gitignore](https://git-scm.com/docs/gitignore) files. +Here's an example: + +```` +# Ignore the directory private that is located at the top of the export tree +/private +# Ignore any file or directory called `test` +test +# Ignore any PDF file +*.pdf +# ..but include special.pdf +!special.pdf +```` + +For more comprehensive documentation and examples, see the [gitignore](https://git-scm.com/docs/gitignore) manpage. + + +## License + +Obsidian-export is dual-licensed under the [Apache 2.0](https://github.com/zoni/obsidian-export/blob/master/LICENSE-APACHE) and the [MIT](https://github.com/zoni/obsidian-export/blob/master/LICENSE-MIT) licenses. diff --git a/book/book-src/SUMMARY.md b/book/book-src/SUMMARY.md new file mode 100644 index 0000000..6368ef3 --- /dev/null +++ b/book/book-src/SUMMARY.md @@ -0,0 +1,6 @@ +# Summary + +* [Introduction](index.md) +* [Installation](installation.md) +* [Usage](usage.md) +* [License](license.md) diff --git a/book/book-src/Usage.md b/book/book-src/Usage.md new file mode 100644 index 0000000..0c598c5 --- /dev/null +++ b/book/book-src/Usage.md @@ -0,0 +1,18 @@ +## Usage + +The main interface of obsidian-export is the similarly-named `obsidian-export` CLI command. +In it's most basic form, `obsidian-export` takes just two mandatory arguments, source and destination: + +obsidian-export ~/Knowledgebase /tmp/export + +This will export all of the files from `~/Knowledgebase` to `/tmp/export`, except for those listed in `.export-ignore` or `.gitignore`. + +### Frontmatter + +By default, frontmatter is copied over "as-is". + +Some static site generators are picky about frontmatter and require it to be present. +Some get tripped up when Markdown files don't have frontmatter but start with a list item or horizontal rule. +In these cases, `--frontmatter=always` can be used to insert an empty frontmatter entry. + +To completely remove any frontmatter from exported notes, use `--frontmatter=never`. diff --git a/book/book-src/index.md b/book/book-src/index.md new file mode 100644 index 0000000..a7ad4c5 --- /dev/null +++ b/book/book-src/index.md @@ -0,0 +1,12 @@ +# Obsidian Export + +*Rust library and associated CLI program to export an [Obsidian](https://obsidian.md/) vault to regular Markdown (specifically: [CommonMark](https://commonmark.org/))* + +* Recursively export Obsidian Markdown files to CommonMark. +* Supports `[[note]]`-style references as well as `![[note]]` file includes. + * `[[note#heading]]` linking/embedding not yet supported, but planned. +* Support for [gitignore](https://git-scm.com/docs/gitignore)-style exclude patterns (default: `.export-ignore`). +* Automatically excludes files that are ignored by Git when the vault is located in a Git repository. + +Please note obsidian-export is not officially endorsed by the Obsidian team. +It supports most but not all of Obsidian's Markdown flavor. diff --git a/book/book-src/installation.md b/book/book-src/installation.md new file mode 100644 index 0000000..159bdd9 --- /dev/null +++ b/book/book-src/installation.md @@ -0,0 +1,17 @@ +## Installation + + > + > **Note**: + > *Obsidian-export* has been developed on Linux. + > Windows and Mac OS are covered as part of the continuous integration tests run on GitHub, but these have not been tested by the author. + > Experience reports from users on these operating systems would be welcomed. + +Binary releases for x86-64 processors are provided for Windows, Linux and Mac operating systems on a best-effort basis. +These may be downloaded from: [https://github.com/zoni/obsidian-export/releases](https://github.com/zoni/obsidian-export/releases) + +Alternatively, *obsidian-export* may be compiled from source using [Cargo](https://doc.rust-lang.org/cargo/), the official package manager for Rust, by using the following steps: + +1. Install the Rust toolchain: [https://www.rust-lang.org/tools/install](https://www.rust-lang.org/tools/install) +1. Run: `cargo install https://github.com/zoni/obsidian-export.git --locked` + +The same `cargo install` command can later be used to upgrade to a newer release as well. diff --git a/book/book-src/license.md b/book/book-src/license.md new file mode 100644 index 0000000..5078806 --- /dev/null +++ b/book/book-src/license.md @@ -0,0 +1,3 @@ +## License + +Obsidian-export is dual-licensed under the [Apache 2.0](https://github.com/zoni/obsidian-export/blob/master/LICENSE-APACHE) and the [MIT](https://github.com/zoni/obsidian-export/blob/master/LICENSE-MIT) licenses. diff --git a/book/book-src/usage.md b/book/book-src/usage.md new file mode 100644 index 0000000..3ec49c6 --- /dev/null +++ b/book/book-src/usage.md @@ -0,0 +1,62 @@ +## Usage + +The main interface of *obsidian-export* is the `obsidian-export` CLI command. +In it's most basic form, `obsidian-export` takes just two mandatory arguments, a source and a destination: + +````sh +obsidian-export my-obsidian-vault /tmp/export +```` + +This will export all of the files from `my-obsidian-vault` to `/tmp/export`, except for those listed in `.export-ignore` or `.gitignore`. + +It is also possible to export individual files: + +````sh +# Export as some-note.md to /tmp/export/ +obsidian-export my-obsidian-vault/some-note.md /tmp/export/ +# Export as exported-note.md in /tmp/ +obsidian-export my-obsidian-vault/some-note.md /tmp/exported-note.md +```` + +### Character encodings + +At present, UTF-8 character encoding is assumed for all note text as well as filenames. +All text and file handling performs [lossy conversion to Unicode strings](https://doc.rust-lang.org/std/string/struct.String.html#method.from_utf8_lossy). + +Use of non-UTF8 encodings may lead to issues like incorrect text replacement and failure to find linked notes. +While this may change in the future, there are no plans to change this behavior in the short term. + +### Frontmatter + +By default, frontmatter is copied over "as-is". + +Some static site generators are picky about frontmatter and require it to be present. +Some get tripped up when Markdown files don't have frontmatter but start with a list item or horizontal rule. +In these cases, `--frontmatter=always` can be used to insert an empty frontmatter entry. + +To completely remove any frontmatter from exported notes, use `--frontmatter=never`. + +### Ignoring files + +By default, hidden files, patterns listed in `.export-ignore` as well as any files ignored by git (if your vault is part of a git repository) will be excluded from exports. +These options will become configurable in the next release. + +Notes linking to ignored notes will be unlinked (they'll only include the link text). Embeds of ignored notes will be skipped entirely. + +#### Ignorefile syntax + +The syntax for `.export-ignore` files is identical to that of [gitignore](https://git-scm.com/docs/gitignore) files. +Here's an example: + +```` +# Ignore the directory private that is located at the top of the export tree +/private +# Ignore any file or directory called `test` +test +# Ignore any PDF file +*.pdf +# ..but include special.pdf +!special.pdf +```` + +For more comprehensive documentation and examples, see the [gitignore](https://git-scm.com/docs/gitignore) manpage. diff --git a/book/book.toml b/book/book.toml new file mode 100644 index 0000000..536caf2 --- /dev/null +++ b/book/book.toml @@ -0,0 +1,6 @@ +[book] +authors = ["Nick Groenen"] +language = "en" +multilingual = false +src = "book-src" +title = "Obsidian Export" diff --git a/book/obsidian-src/.obsidian/config b/book/obsidian-src/.obsidian/config new file mode 100644 index 0000000..dc8a0e8 --- /dev/null +++ b/book/obsidian-src/.obsidian/config @@ -0,0 +1 @@ +{"theme":"moonstone","pluginEnabledStatus":{"Open in default app":true,"file-explorer":true,"global-search":true,"switcher":true,"graph":true,"backlink":true,"command-palette":true,"markdown-importer":false,"word-count":true,"tag-pane":false,"daily-notes":false,"slides":false,"open-with-default-app":true,"random-note":false,"page-preview":true,"zk-prefixer":false,"starred":false,"outline":true,"templates":false,"workspaces":false},"isSidebarCollapsed":false,"isRightSidedockCollapsed":true,"lastOpenSidebarTab":"File explorer","useTab":false,"showLineNumber":true,"foldHeading":true,"foldIndent":true,"vimMode":true,"newFileLocation":"current","hotkeys":{"switcher:open":[{"modifiers":["Mod"],"key":" "}],"app:go-back":[{"modifiers":["Mod"],"key":"o"}],"app:go-forward":[{"modifiers":["Mod"],"key":"i"}],"editor:toggle-bold":[{"modifiers":["Mod"],"key":"8"}],"editor:toggle-italics":[{"modifiers":["Mod"],"key":"-"}],"editor:toggle-highlight":[{"modifiers":["Mod"],"key":"="}],"editor:delete-paragraph":[],"editor:focus-top":[{"modifiers":["Alt"],"key":"k"}],"editor:focus-bottom":[{"modifiers":["Alt"],"key":"j"}],"editor:focus-left":[{"modifiers":["Alt"],"key":"h"}],"editor:focus-right":[{"modifiers":["Alt"],"key":"l"}],"workspace:split-horizontal":[{"modifiers":["Alt"],"key":"s"}],"workspace:split-vertical":[{"modifiers":["Alt"],"key":"v"}],"workspace:toggle-pin":[{"modifiers":["Alt"],"key":"t"}],"graph:open":[],"backlink:open-backlinks":[{"modifiers":["Alt"],"key":"b"}],"workspace:close":[{"modifiers":["Alt"],"key":"w"}],"editor:swap-line-up":[{"modifiers":["Mod","Shift"],"key":"K"}],"editor:swap-line-down":[{"modifiers":["Mod","Shift"],"key":"J"}],"outline:open":[{"modifiers":["Alt"],"key":"o"}],"app:toggle-left-sidebar":[{"modifiers":["Alt"],"key":","}],"app:toggle-right-sidebar":[{"modifiers":["Alt"],"key":"."}],"graph:open-local":[{"modifiers":["Alt"],"key":"g"}]},"lastOpenRightSidedockTab":"Backlinks","sideDockWidth":{"left":300,"right":301},"fileSortOrder":"alphabetical","promptDelete":false,"readableLineLength":true,"alwaysUpdateLinks":true,"spellcheck":true,"strictLineBreaks":true,"spellcheckDictionary":[],"autoPairMarkdown":false,"autoPairBrackets":false,"showFrontmatter":true,"enabledPlugins":["todoist-sync-plugin"],"defaultViewMode":"preview","obsidianCss":false} \ No newline at end of file diff --git a/book/obsidian-src/.obsidian/workspaces.json b/book/obsidian-src/.obsidian/workspaces.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/book/obsidian-src/.obsidian/workspaces.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/book/obsidian-src/README.md b/book/obsidian-src/README.md new file mode 100644 index 0000000..a115b65 --- /dev/null +++ b/book/obsidian-src/README.md @@ -0,0 +1,4 @@ +![[index]] +![[installation]] +![[usage]] +![[license]] diff --git a/book/obsidian-src/SUMMARY.md b/book/obsidian-src/SUMMARY.md new file mode 100644 index 0000000..11fab07 --- /dev/null +++ b/book/obsidian-src/SUMMARY.md @@ -0,0 +1,6 @@ +# Summary + +- [[index|Introduction]] +- [[Installation]] +- [[Usage]] +- [[License]] diff --git a/book/obsidian-src/index.md b/book/obsidian-src/index.md new file mode 100644 index 0000000..93ab865 --- /dev/null +++ b/book/obsidian-src/index.md @@ -0,0 +1,16 @@ +# Obsidian Export + +_Rust library and associated CLI program to export an [Obsidian] vault to regular Markdown (specifically: [CommonMark])_ + +- Recursively export Obsidian Markdown files to CommonMark. +- Supports `[[note]]`-style references as well as `![[note]]` file includes. + - `[[note#heading]]` linking/embedding not yet supported, but planned. +- Support for [gitignore]-style exclude patterns (default: `.export-ignore`). +- Automatically excludes files that are ignored by Git when the vault is located in a Git repository. + +Please note obsidian-export is not officially endorsed by the Obsidian team. +It supports most but not all of Obsidian's Markdown flavor. + +[Obsidian]: https://obsidian.md/ +[CommonMark]: https://commonmark.org/ +[gitignore]: https://git-scm.com/docs/gitignore diff --git a/book/obsidian-src/installation.md b/book/obsidian-src/installation.md new file mode 100644 index 0000000..804ae23 --- /dev/null +++ b/book/obsidian-src/installation.md @@ -0,0 +1,18 @@ +## Installation + +> **Note**: +> _Obsidian-export_ has been developed on Linux. +> Windows and Mac OS are covered as part of the continuous integration tests run on GitHub, but these have not been tested by the author. +> Experience reports from users on these operating systems would be welcomed. + +Binary releases for x86-64 processors are provided for Windows, Linux and Mac operating systems on a best-effort basis. +These may be downloaded from: + +Alternatively, _obsidian-export_ may be compiled from source using [Cargo], the official package manager for Rust, by using the following steps: + +1. Install the Rust toolchain: +2. Run: `cargo install https://github.com/zoni/obsidian-export.git --locked` + +The same `cargo install` command can later be used to upgrade to a newer release as well. + +[Cargo]: https://doc.rust-lang.org/cargo/ diff --git a/book/obsidian-src/license.md b/book/obsidian-src/license.md new file mode 100644 index 0000000..9f2f923 --- /dev/null +++ b/book/obsidian-src/license.md @@ -0,0 +1,6 @@ +## License + +Obsidian-export is dual-licensed under the [Apache 2.0] and the [MIT] licenses. + +[Apache 2.0]: https://github.com/zoni/obsidian-export/blob/master/LICENSE-APACHE +[MIT]: https://github.com/zoni/obsidian-export/blob/master/LICENSE-MIT diff --git a/book/obsidian-src/usage.md b/book/obsidian-src/usage.md new file mode 100644 index 0000000..e4bc8ce --- /dev/null +++ b/book/obsidian-src/usage.md @@ -0,0 +1,65 @@ +## Usage + +The main interface of _obsidian-export_ is the `obsidian-export` CLI command. +In it's most basic form, `obsidian-export` takes just two mandatory arguments, a source and a destination: + +```sh +obsidian-export my-obsidian-vault /tmp/export +``` + +This will export all of the files from `my-obsidian-vault` to `/tmp/export`, except for those listed in `.export-ignore` or `.gitignore`. + +It is also possible to export individual files: + +```sh +# Export as some-note.md to /tmp/export/ +obsidian-export my-obsidian-vault/some-note.md /tmp/export/ +# Export as exported-note.md in /tmp/ +obsidian-export my-obsidian-vault/some-note.md /tmp/exported-note.md +``` + +### Character encodings + +At present, UTF-8 character encoding is assumed for all note text as well as filenames. +All text and file handling performs [lossy conversion to Unicode strings][from_utf8_lossy]. + +Use of non-UTF8 encodings may lead to issues like incorrect text replacement and failure to find linked notes. +While this may change in the future, there are no plans to change this behavior in the short term. + +### Frontmatter + +By default, frontmatter is copied over "as-is". + +Some static site generators are picky about frontmatter and require it to be present. +Some get tripped up when Markdown files don't have frontmatter but start with a list item or horizontal rule. +In these cases, `--frontmatter=always` can be used to insert an empty frontmatter entry. + +To completely remove any frontmatter from exported notes, use `--frontmatter=never`. + +### Ignoring files + +By default, hidden files, patterns listed in `.export-ignore` as well as any files ignored by git (if your vault is part of a git repository) will be excluded from exports. +These options will become configurable in the next release. + +Notes linking to ignored notes will be unlinked (they'll only include the link text). Embeds of ignored notes will be skipped entirely. + +#### Ignorefile syntax + +The syntax for `.export-ignore` files is identical to that of [gitignore] files. +Here's an example: + +``` +# Ignore the directory private that is located at the top of the export tree +/private +# Ignore any file or directory called `test` +test +# Ignore any PDF file +*.pdf +# ..but include special.pdf +!special.pdf +``` + +For more comprehensive documentation and examples, see the [gitignore] manpage. + +[from_utf8_lossy]: https://doc.rust-lang.org/std/string/struct.String.html#method.from_utf8_lossy +[gitignore]: https://git-scm.com/docs/gitignore diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..d36edf8 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,555 @@ +#[macro_use] +extern crate lazy_static; + +mod walker; + +use pathdiff::diff_paths; +use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS}; +use pulldown_cmark::{CodeBlockKind, CowStr, Event, Options, Parser, Tag}; +use pulldown_cmark_to_cmark::{self, cmark_with_options}; +use rayon::prelude::*; +use regex::Regex; +use snafu::{ResultExt, Snafu}; +use std::ffi::OsString; +use std::fs::{self, File}; +use std::io::prelude::*; +use std::io::ErrorKind; +use std::path::{Path, PathBuf}; +use std::str; +use walker::{vault_contents, WalkOptions}; + +type Result = std::result::Result; +type MarkdownTree<'a> = Vec>; + +lazy_static! { + static ref OBSIDIAN_NOTE_LINK_RE: Regex = + Regex::new(r"^(?P[^#|]+)(#(?P.+?))??(\|(?P