Refactored instant loading (#5032)

material/base.html

@@ -240,7 +240,7 @@
       </script>
     {% endblock %}
     {% block scripts %}
-      <script src="{{ 'assets/javascripts/bundle.ce72ebac.min.js' | url }}"></script>
+      <script src="{{ 'assets/javascripts/bundle.b78d2936.min.js' | url }}"></script>
       {% for path in config.extra_javascript %}
         <script src="{{ path | url }}"></script>
       {% endfor %}

src/assets/javascripts/bundle.ts

@@ -148,7 +148,8 @@ setupClipboardJS({ alert$ })
 
 /* Set up instant loading, if enabled */
 if (feature("navigation.instant"))
-  setupInstantLoading({ document$, location$, viewport$ })
+  setupInstantLoading({ location$, viewport$ })
+    .subscribe(document$)
 
 /* Set up version selector */
 if (config.version?.provider === "mike")

src/assets/javascripts/integrations/instant/index.ts

@@ -22,54 +22,40 @@
 
 import {
   EMPTY,
-  NEVER,
   Observable,
   Subject,
   bufferCount,
   catchError,
-  concatMap,
+  concat,
   debounceTime,
-  distinctUntilChanged,
   distinctUntilKeyChanged,
-  filter,
+  endWith,
   fromEvent,
+  ignoreElements,
   map,
-  merge,
   of,
-  sample,
   share,
   skip,
-  skipUntil,
-  switchMap
+  startWith,
+  switchMap,
+  take,
+  withLatestFrom
 } from "rxjs"
 
 import { configuration, feature } from "~/_"
 import {
   Viewport,
-  ViewportOffset,
   getElements,
+  getLocation,
   getOptionalElement,
   request,
   setLocation,
   setLocationHash
 } from "~/browser"
 import { getComponentElement } from "~/components"
-import { h } from "~/utilities"
 
 import { fetchSitemap } from "../sitemap"
 
-/* ----------------------------------------------------------------------------
- * Types
- * ------------------------------------------------------------------------- */
-
-/**
- * History state
- */
-export interface HistoryState {
-  url: URL                             /* State URL */
-  offset?: ViewportOffset              /* State viewport offset */
-}
-
 /* ----------------------------------------------------------------------------
  * Helper types
  * ------------------------------------------------------------------------- */
@@ -78,7 +64,6 @@ export interface HistoryState {
  * Setup options
  */
 interface SetupOptions {
-  document$: Subject<Document>         /* Document subject */
   location$: Subject<URL>              /* Location subject */
   viewport$: Observable<Viewport>      /* Viewport observable */
 }
@@ -88,152 +73,163 @@ interface SetupOptions {
  * ------------------------------------------------------------------------- */
 
 /**
- * Set up instant loading
+ * Set up instant navigation
  *
- * When fetching, theoretically, we could use `responseType: "document"`, but
- * since all MkDocs links are relative, we need to make sure that the current
- * location matches the document we just loaded. Otherwise any relative links
- * in the document could use the old location.
- *
- * This is the reason why we need to synchronize history events and the process
- * of fetching the document for navigation changes (except `popstate` events):
- *
- * 1. Fetch document via `XMLHTTPRequest`
- * 2. Set new location via `history.pushState`
- * 3. Parse and emit fetched document
- *
- * For `popstate` events, we must not use `history.pushState`, or the forward
- * history will be irreversibly overwritten. In case the request fails, the
- * location change is dispatched regularly.
+ * This is a heavily orchestrated operation - see inline comments to learn how
+ * this works with Material for MkDocs, and how you can hook into it.
  *
  * @param options - Options
+ *
+ * @returns Document observable
  */
 export function setupInstantLoading(
-  { document$, location$, viewport$ }: SetupOptions
-): void {
+  { location$, viewport$ }: SetupOptions
+): Observable<Document> {
   const config = configuration()
   if (location.protocol === "file:")
-    return
+    return EMPTY
 
-  /* Disable automatic scroll restoration */
-  if ("scrollRestoration" in history) {
-    history.scrollRestoration = "manual"
-
-    /* Hack: ensure that reloads restore viewport offset */
-    fromEvent(window, "beforeunload")
-      .subscribe(() => {
-        history.scrollRestoration = "auto"
-      })
-  }
-
-  /* Hack: ensure absolute favicon link to omit 404s when switching */
-  const favicon = getOptionalElement<HTMLLinkElement>("link[rel=icon]")
-  if (typeof favicon !== "undefined")
-    favicon.href = favicon.href
-
-  /* Intercept internal navigation */
-  const push$ = fetchSitemap()
+  // Load sitemap immediately, so we have it available when the user initiates
+  // the first instant navigation request, and canonicalize URLs to the current
+  // base URL. The base URL will remain stable in between loads, as it's only
+  // read at the first initialization of the application.
+  const sitemap$ = fetchSitemap()
     .pipe(
-      map(paths => paths.map(path => `${new URL(path, config.base)}`)),
-      switchMap(urls => fromEvent<MouseEvent>(document.body, "click")
-        .pipe(
-          filter(ev => !ev.metaKey && !ev.ctrlKey),
-          switchMap(ev => {
-            if (ev.target instanceof Element) {
-              const el = ev.target.closest("a")
-              if (el && !el.target) {
-                const url = new URL(el.href)
-
-                /* Canonicalize URL */
-                url.search = ""
-                url.hash = ""
-
-                /* Check if URL should be intercepted */
-                if (
-                  url.pathname !== location.pathname &&
-                  urls.includes(url.toString())
-                ) {
-                  ev.preventDefault()
-                  return of({
-                    url: new URL(el.href)
-                  })
-                }
-              }
-            }
-            return NEVER
-          })
-        )
-      ),
-      share<HistoryState>()
+      map(paths => paths.map(path => `${new URL(path, config.base)}`))
     )
 
-  /* Intercept history back and forward */
-  const pop$ = fromEvent<PopStateEvent>(window, "popstate")
+  // Intercept inter-site navigation - to keep the number of event listeners
+  // low we use the fact that uncaptured events bubble up to the body. This also
+  // has the nice property that we don't need to detach and then again attach
+  // event listeners when instant navigation occurs.
+  const instant$ = fromEvent<MouseEvent>(document.body, "click")
     .pipe(
-      filter(ev => ev.state !== null),
-      map(ev => ({
-        url: new URL(location.href),
-        offset: ev.state
-      })),
-      share<HistoryState>()
-    )
+      withLatestFrom(sitemap$),
+      switchMap(([ev, sitemap]) => {
+        if (!(ev.target instanceof Element))
+          return EMPTY
 
-  /* Emit location change */
-  merge(push$, pop$)
-    .pipe(
-      distinctUntilChanged((a, b) => a.url.href === b.url.href),
-      map(({ url }) => url)
-    )
-      .subscribe(location$)
+        // Skip, as target is not within a link - clicks on non-link elements
+        // are also captured, which we need to exclude from processing.
+        const el = ev.target.closest("a")
+        if (el === null)
+          return EMPTY
 
-  /* Fetch document via `XMLHTTPRequest` */
-  const response$ = location$
-    .pipe(
-      distinctUntilKeyChanged("pathname"),
-      switchMap(url => request(url.href)
-        .pipe(
-          catchError(() => {
-            setLocation(url)
-            return NEVER
-          })
-        )
-      ),
+        // Skip, as link opens in new window - we now know we have captured a
+        // click on a link, but the link either has a `target` property defined,
+        // or the user pressed the `meta` or `ctrl` key to open it in a new
+        // window. Thus, we need to filter those events, too.
+        if (el.target || ev.metaKey || ev.ctrlKey)
+          return EMPTY
+
+        // Next, we must check if the URL is relevant for us, i.e., if it's an
+        // internal link to a page that is managed by MkDocs. Only then we can
+        // be sure that the structure of the page to be loaded adheres to the
+        // current document structure and can subsequently be injected into it
+        // without doing a full reload. For this reason, we must canonicalize
+        // the URL by removing all search parameters and hash fragments.
+        const url = new URL(el.href)
+        url.search = url.hash = ""
+
+        // Skip, if URL is not included in the sitemap - this could be the case
+        // when linking between versions or languages, or to another page that
+        // the author included as part of the build, but that is not managed by
+        // MkDocs. In that case we must not continue with instant navigation.
+        if (!sitemap.includes(`${url}`))
+          return EMPTY
+
+        // We now know that we have a link to an internal page, so we prevent
+        // the browser from navigation and emit the URL for instant navigation.
+        // Note that this also includes anchor links, which means we need to
+        // implement anchor positioning ourselves.
+        ev.preventDefault()
+        return of(new URL(el.href))
+      }),
       share()
     )
 
-  /* Set new location via `history.pushState` */
-  push$
-    .pipe(
-      sample(response$)
-    )
-      .subscribe(({ url }) => {
-        history.pushState({}, "", `${url}`)
-      })
+  // Before fetching for the first time, resolve the absolute favicon position,
+  // as the browser will try to fetch the icon immediately.
+  instant$.pipe(take(1))
+    .subscribe(() => {
+      const favicon = getOptionalElement<HTMLLinkElement>("link[rel=icon]")
+      if (typeof favicon !== "undefined")
+        favicon.href = favicon.href
+    })
 
-  /* Parse and emit fetched document */
+  // Enable scroll restoration before window unloads - this is essential to
+  // ensure that full reloads (F5) restore the viewport offset correctly. If
+  // only popstate events wouldn't reset the scroll position prior to their
+  // emission, we could just reset this in popstate. Meh.
+  fromEvent(window, "beforeunload")
+    .subscribe(() => {
+      history.scrollRestoration = "auto"
+    })
+
+  // When an instant navigation event occurs, disable scroll restoration, since
+  // we must normalize and synchronize the behavior across all browsers. For
+  // instance, when the user clicks the back or forward button, the browser
+  // would immediately jump to the position of the previous document.
+  instant$.pipe(withLatestFrom(viewport$))
+    .subscribe(([url, { offset }]) => {
+      history.scrollRestoration = "manual"
+
+      // While it would be better UX to defer the history state change until the
+      // document was fully fetched and parsed, we must schedule it here, since
+      // popstate events are emitted when history state changes happen. Moreover
+      // we need to back up the current viewport offset, so we can restore it
+      // when popstate events occur, e.g., when the browser's back and forward
+      // buttons are used for navigation.
+      history.replaceState(offset, "")
+      history.pushState(null, "", url)
+    })
+
+  // Emit URL that should be fetched via instant loading on location subject,
+  // which was passed into this function. The idea is that instant loading can
+  // be intercepted by other parts of the application, which can synchronously
+  // back up or restore state before instant loading happens.
+  instant$.subscribe(location$)
+
+  // Fetch document - when fetching, we could use `responseType: document`, but
+  // since all MkDocs links are relative, we need to make sure that the current
+  // location matches the document we just loaded. Otherwise any relative links
+  // in the document might use the old location. If the request fails for some
+  // reason, we fall back to regular navigation and set the location explicitly,
+  // which will force-load the page. Furthermore, we must pre-warm the buffer
+  // for the duplicate check, or the first click on an anchor link will also
+  // trigger an instant loading event, which doesn't make sense.
+  const response$ = location$
+    .pipe(
+      startWith(getLocation()),
+      distinctUntilKeyChanged("pathname"),
+      skip(1),
+      switchMap(url => request(url)
+        .pipe(
+          catchError(() => {
+            setLocation(url)
+            return EMPTY
+          })
+        )
+      )
+    )
+
+  // Initialize the DOM parser, parse the returned HTML, and replace selected
+  // meta tags and components before handing control down to the application.
   const dom = new DOMParser()
-  response$
+  const document$ = response$
     .pipe(
       switchMap(res => res.text()),
-      map(res => dom.parseFromString(res, "text/html"))
-    )
-      .subscribe(document$)
-
-  /* Replace meta tags and components */
-  document$
-    .pipe(
-      skip(1)
-    )
-      .subscribe(replacement => {
+      switchMap(res => {
+        const document = dom.parseFromString(res, "text/html")
         for (const selector of [
 
-          /* Meta tags */
+          // Meta tags
           "title",
           "link[rel=canonical]",
           "meta[name=author]",
           "meta[name=description]",
 
-          /* Components */
+          // Components
           "[data-md-component=announce]",
           "[data-md-component=container]",
           "[data-md-component=header-topic]",
@@ -245,7 +241,7 @@ export function setupInstantLoading(
             : []
         ]) {
           const source = getOptionalElement(selector)
-          const target = getOptionalElement(selector, replacement)
+          const target = getOptionalElement(selector, document)
           if (
             typeof source !== "undefined" &&
             typeof target !== "undefined"
@@ -253,68 +249,95 @@ export function setupInstantLoading(
             source.replaceWith(target)
           }
         }
+
+        // After meta tags and components were replaced, re-evaluate scripts
+        // that were provided by the author as part of Markdown files.
+        const container = getComponentElement("container")
+        return concat(getElements("script", container))
+          .pipe(
+            switchMap(el => {
+              const script = document.createElement("script")
+              if (el.src) {
+                for (const name of el.getAttributeNames())
+                  script.setAttribute(name, el.getAttribute(name)!)
+                el.replaceWith(script)
+
+                // Complete when script is loaded
+                return new Observable(observer => {
+                  script.onload = () => observer.complete()
+                })
+
+              // Complete immediately
+              } else {
+                script.textContent = el.textContent
+                el.replaceWith(script)
+                return EMPTY
+              }
+            }),
+            ignoreElements(),
+            endWith(document)
+          )
+      }),
+      share()
+    )
+
+  // Intercept popstate events, e.g. when using the browser's back and forward
+  // buttons, and emit new location for fetching and parsing.
+  const popstate$ = fromEvent<PopStateEvent>(window, "popstate")
+  popstate$.pipe(map(() => getLocation()))
+    .subscribe(location$)
+
+  // Intercept clicks on anchor links, and scroll document into position. As
+  // we disabled scroll restoration, we need to do this manually here.
+  location$
+    .pipe(
+      startWith(getLocation()),
+      bufferCount(2, 1),
+      switchMap(([prev, next]) => (
+        prev.pathname === next.pathname &&
+        prev.hash     !== next.hash
+      )
+        ? of(next)
+        : EMPTY
+      )
+    )
+      .subscribe(url => {
+        if (history.state !== null || !url.hash) {
+          window.scrollTo(0, history.state?.y ?? 0)
+        } else {
+          history.scrollRestoration = "auto"
+          setLocationHash(url.hash)
+          history.scrollRestoration = "manual"
+        }
       })
 
-  /* Re-evaluate scripts */
+  // After parsing the document, check if the current history entry has a state.
+  // This may happen when users press the back or forward button to visit a page
+  // that was already seen. If there's no state, it means a new page was visited
+  // and we should scroll to the top, unless an anchor is given.
+  document$.pipe(withLatestFrom(location$))
+    .subscribe(([, url]) => {
+      if (history.state !== null || !url.hash) {
+        window.scrollTo(0, history.state?.y ?? 0)
+      } else {
+        setLocationHash(url.hash)
+      }
+    })
+
+  // If the current history is not empty, register an event listener updating
+  // the current history state whenever the scroll position changes. This must
+  // be debounced and cannot be done in popstate, as popstate has already
+  // removed the entry from the history.
   document$
     .pipe(
-      skip(1),
-      map(() => getComponentElement("container")),
-      switchMap(el => getElements("script", el)),
-      concatMap(el => {
-        const script = h("script")
-        if (el.src) {
-          for (const name of el.getAttributeNames())
-            script.setAttribute(name, el.getAttribute(name)!)
-          el.replaceWith(script)
-
-          /* Complete when script is loaded */
-          return new Observable(observer => {
-            script.onload = () => observer.complete()
-          })
-
-        /* Complete immediately */
-        } else {
-          script.textContent = el.textContent
-          el.replaceWith(script)
-          return EMPTY
-        }
-      })
-    )
-      .subscribe()
-
-  /* Emit history state change */
-  merge(push$, pop$)
-    .pipe(
-      sample(document$)
-    )
-      .subscribe(({ url, offset }) => {
-        if (url.hash && !offset) {
-          setLocationHash(url.hash)
-        } else {
-          window.scrollTo(0, offset?.y || 0)
-        }
-      })
-
-  /* Debounce update of viewport offset */
-  viewport$
-    .pipe(
-      skipUntil(push$),
-      debounceTime(250),
-      distinctUntilKeyChanged("offset")
+      switchMap(() => viewport$),
+      distinctUntilKeyChanged("offset"),
+      debounceTime(100)
     )
       .subscribe(({ offset }) => {
         history.replaceState(offset, "")
       })
 
-  /* Set viewport offset from history */
-  merge(push$, pop$)
-    .pipe(
-      bufferCount(2, 1),
-      filter(([a, b]) => a.url.pathname === b.url.pathname),
-      map(([, state]) => state)
-    )
-      .subscribe(({ offset }) => {
-        window.scrollTo(0, offset?.y || 0)
-      })
+  // Return document
+  return document$
 }