From cea6834fef54da59fc1692d1db0221b93793238f Mon Sep 17 00:00:00 2001
From: Jacky Zhao <j.zhao2k19@gmail.com>
Date: Wed, 9 Aug 2023 00:26:33 -0700
Subject: [PATCH] profiling, better concurrency heuristics

---
 .gitignore                         | 1 +
 content/advanced/making plugins.md | 2 +-
 package.json                       | 3 ++-
 quartz/processors/parse.ts         | 9 +++++----
 4 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/.gitignore b/.gitignore
index 5177d6632..fd96fec90 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,7 @@
 .gitignore
 node_modules
 public
+prof
 tsconfig.tsbuildinfo
 .obsidian
 .quartz-cache
diff --git a/content/advanced/making plugins.md b/content/advanced/making plugins.md
index a73069e6d..d2db67e41 100644
--- a/content/advanced/making plugins.md	
+++ b/content/advanced/making plugins.md	
@@ -2,7 +2,7 @@
 title: Making your own plugins
 ---
 
-This part of the documentation will assume you have some basic coding knowledge and will include code snippets that describe the interface of what Quartz plugins should look like.
+This part of the documentation will assume you have some basic coding knowledge in TypeScript and will include code snippets that describe the interface of what Quartz plugins should look like.
 
 ![[quartz-transform-pipeline.png]]
 
diff --git a/package.json b/package.json
index 0d5e7a03c..29b46f206 100644
--- a/package.json
+++ b/package.json
@@ -14,7 +14,8 @@
   "scripts": {
     "check": "tsc --noEmit && npx prettier . --check",
     "format": "npx prettier . --write",
-    "test": "tsx ./quartz/path.test.ts"
+    "test": "tsx ./quartz/path.test.ts",
+    "profile": "0x -D prof ./quartz/bootstrap-cli.mjs build --concurrency=1"
   },
   "keywords": [
     "site generator",
diff --git a/quartz/processors/parse.ts b/quartz/processors/parse.ts
index 69df69b3d..299d59afe 100644
--- a/quartz/processors/parse.ts
+++ b/quartz/processors/parse.ts
@@ -9,7 +9,6 @@ import { PerfTimer } from "../perf"
 import { read } from "to-vfile"
 import { FilePath, QUARTZ, slugifyFilePath } from "../path"
 import path from "path"
-import os from "os"
 import workerpool, { Promise as WorkerPromise } from "workerpool"
 import { QuartzLogger } from "../log"
 import { trace } from "../trace"
@@ -82,6 +81,7 @@ export function createFileParser(ctx: BuildCtx, fps: FilePath[]) {
     const res: ProcessedContent[] = []
     for (const fp of fps) {
       try {
+        const perf = new PerfTimer()
         const file = await read(fp)
 
         // strip leading and trailing whitespace
@@ -101,7 +101,7 @@ export function createFileParser(ctx: BuildCtx, fps: FilePath[]) {
         res.push([newAst, file])
 
         if (argv.verbose) {
-          console.log(`[process] ${fp} -> ${file.data.slug}`)
+          console.log(`[process] ${fp} -> ${file.data.slug} (${perf.timeSince()})`)
         }
       } catch (err) {
         trace(`\nFailed to process \`${fp}\``, err as Error)
@@ -112,14 +112,15 @@ export function createFileParser(ctx: BuildCtx, fps: FilePath[]) {
   }
 }
 
+const clamp = (num: number, min: number, max: number) => Math.min(Math.max(Math.round(num), min), max);
 export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<ProcessedContent[]> {
   const { argv } = ctx
   const perf = new PerfTimer()
   const log = new QuartzLogger(argv.verbose)
 
+  // rough heuristics: 128 gives enough time for v8 to JIT and optimize parsing code paths
   const CHUNK_SIZE = 128
-  let concurrency =
-    ctx.argv.concurrency ?? (fps.length < CHUNK_SIZE ? 1 : os.availableParallelism())
+  const concurrency = ctx.argv.concurrency ?? clamp(fps.length / CHUNK_SIZE, 1, 4)
 
   let res: ProcessedContent[] = []
   log.start(`Parsing input files using ${concurrency} threads`)