Improved search experience in many languages

material/assets/manifest.json

@@ -1,10 +1,10 @@
 {
-  "assets/javascripts/bundle.js": "assets/javascripts/bundle.565e6875.min.js",
+  "assets/javascripts/bundle.js": "assets/javascripts/bundle.dd72b84b.min.js",
-  "assets/javascripts/bundle.js.map": "assets/javascripts/bundle.565e6875.min.js.map",
+  "assets/javascripts/bundle.js.map": "assets/javascripts/bundle.dd72b84b.min.js.map",
   "assets/javascripts/vendor.js": "assets/javascripts/vendor.de50e36d.min.js",
   "assets/javascripts/vendor.js.map": "assets/javascripts/vendor.de50e36d.min.js.map",
-  "assets/javascripts/worker/search.js": "assets/javascripts/worker/search.7a2ea5ba.min.js",
+  "assets/javascripts/worker/search.js": "assets/javascripts/worker/search.9b3611bd.min.js",
-  "assets/javascripts/worker/search.js.map": "assets/javascripts/worker/search.7a2ea5ba.min.js.map",
+  "assets/javascripts/worker/search.js.map": "assets/javascripts/worker/search.9b3611bd.min.js.map",
   "assets/stylesheets/main.css": "assets/stylesheets/main.545621a7.min.css",
   "assets/stylesheets/main.css.map": "assets/stylesheets/main.545621a7.min.css.map",
   "assets/stylesheets/palette.css": "assets/stylesheets/palette.36d1b78f.min.css",

material/base.html

@@ -183,7 +183,7 @@
     </div>
     {% block scripts %}
       <script src="{{ 'assets/javascripts/vendor.de50e36d.min.js' | url }}"></script>
-      <script src="{{ 'assets/javascripts/bundle.565e6875.min.js' | url }}"></script>
+      <script src="{{ 'assets/javascripts/bundle.dd72b84b.min.js' | url }}"></script>
       {%- set translations = {} -%}
       {%- for key in [
         "clipboard.copy",
@@ -207,7 +207,7 @@
           base: "{{ base_url }}",
           features: {{ config.theme.features | tojson }},
           search: Object.assign({
-            worker: "{{ 'assets/javascripts/worker/search.7a2ea5ba.min.js' | url }}"
+            worker: "{{ 'assets/javascripts/worker/search.9b3611bd.min.js' | url }}"
           }, typeof search !== "undefined" && search)
         })
       </script>

material/partials/language/ar.html

@@ -11,8 +11,7 @@
   "footer.next": "التالية",
   "meta.comments": "التعليقات",
   "meta.source": "المصدر",
-  "search.config.lang": "ar",
+  "search.config.pipeline": " ",
-  "search.config.pipeline": "stemmer",
   "search.placeholder": "بحث",
   "search.result.placeholder": "اكتب لبدء البحث",
   "search.result.none": "لا توجد نتائج",

material/partials/language/cs.html

@@ -10,7 +10,6 @@
   "footer.next": "Další",
   "meta.comments": "Komentáře",
   "meta.source": "Zdroj",
-  "search.config.lang": "ro",
   "search.placeholder": "Hledat",
   "search.result.placeholder": "Pište co se má vyhledat",
   "search.result.none": "Nenalezeny žádné dokumenty",

material/partials/language/et.html

@@ -10,7 +10,6 @@
   "footer.next": "Järgmine",
   "meta.comments": "Kommentaarid",
   "meta.source": "Lähtekood",
-  "search.config.pipeline": "",
   "search.placeholder": "Otsi",
   "search.result.placeholder": "Otsimiseks kirjuta siia",
   "search.result.none": "Otsingule ei leitud ühtegi vastet",

material/partials/language/gr.html

@@ -10,6 +10,7 @@
   "footer.next": "Προηγούμενη",
   "meta.comments": "Σχόλια",
   "meta.source": "Πηγή",
+  "search.config.pipeline": " ",
   "search.placeholder": "Αναζήτηση",
   "search.result.placeholder": "Πληκτρολογήστε για να αρχίσει η αναζήτηση",
   "search.result.none": "Δε βρέθηκαν αντίστοιχα αρχεία",

material/partials/language/he.html

@@ -11,8 +11,7 @@
   "footer.next": "הַבָּא",
   "meta.comments": "הערות",
   "meta.source": "מָקוֹר",
-  "search.config.lang": "ar",
+  "search.config.pipeline": " ",
-  "search.config.pipeline": "stemmer",
   "search.placeholder": "לחפש",
   "search.result.placeholder": "הקלד כדי להתחיל לחפש",
   "search.result.none": "אין מסמכים תואמים",

material/partials/language/hi.html

@@ -10,8 +10,7 @@
   "footer.next": "आगामी",
   "meta.comments": "टिप्पणियाँ",
   "meta.source": "स्रोत",
-  "search.config.lang": "ar"
+  "search.config.pipeline": " ",
-  "search.config.pipeline": "stemmer",
   "search.placeholder": "खोज",
   "search.result.placeholder": "खोज शुरू करने के लिए टाइप करें",
   "search.result.none": "कोई मिलान डॉक्यूमेंट नहीं",

material/partials/language/hr.html

@@ -10,7 +10,6 @@
   "footer.next": "Sljedeće",
   "meta.comments": "Komentari",
   "meta.source": "Izvor",
-  "search.config.pipeline": "",
   "search.placeholder": "Pretraživanje",
   "search.result.placeholder": "Unesite pojam pretraživanja",
   "search.result.none": "Ništa nije pronađeno",

material/partials/language/ja.html

@@ -11,6 +11,7 @@
   "meta.comments": "コメント",
   "meta.source": "ソース",
   "search.config.lang": "ja",
+  "search.config.pipeline": "trimmer, stemmer",
   "search.config.separator": "[\s\-　、。，．]+",
   "search.placeholder": "検索",
   "search.result.placeholder": "検索キーワードを入力してください",

material/partials/language/kr.html

@@ -10,7 +10,6 @@
   "footer.next": "다음",
   "meta.comments": "댓글",
   "meta.source": "출처",
-  "search.config.lang": "ja",
   "search.config.pipeline": " ",
   "search.placeholder": "검색",
   "search.result.placeholder": "검색어를 입력하세요",

material/partials/language/my.html

@@ -13,8 +13,7 @@
   "meta.comments": "မှတ်ချက်များ",
   "meta.source": "ရင်းမြစ်",
   "nav.title": "လမ်းညွှန်",
-  "search.config.lang": "ar",
+  "search.config.pipeline": " ",
-  "search.config.pipeline": "stemmer",
   "search.placeholder": "ရှာရန်",
   "search.reset": "ရှင်းလင်း",
   "search.result.placeholder": "ရှာဖွေခြင်းစရန် စာရိုက်ပါ",

material/partials/language/ru.html

@@ -11,7 +11,6 @@
   "meta.comments": "Комментарии",
   "meta.source": "Исходный код",
   "search.config.lang": "ru",
-  "search.config.pipeline": "stemmer",
   "search.placeholder": "Поиск",
   "search.result.placeholder": "Начните печатать для поиска",
   "search.result.none": "Совпадений не найдено",

material/partials/language/sh.html

@@ -10,7 +10,6 @@
   "footer.next": "Sledeće",
   "meta.comments": "Komentari",
   "meta.source": "Izvor",
-  "search.config.lang": "ro",
   "search.placeholder": "Pretraga",
   "search.result.placeholder": "Unesite pojam pretrage",
   "search.result.none": "Ništa nije pronađeno",

material/partials/language/sr.html

@@ -11,7 +11,6 @@
   "meta.comments": "Коментари",
   "meta.source": "Извор",
   "search.config.lang": "ru",
-  "search.config.pipeline": "stemmer",
   "search.placeholder": "Претрага",
   "search.result.placeholder": "Унесите појам претраге",
   "search.result.none": "Нису пронађени документи",

material/partials/language/th.html

@@ -10,6 +10,7 @@
   "footer.next": "ต่อไป",
   "meta.comments": "ความคิดเห็น",
   "meta.source": "แหล่งที่มา",
+  "search.config.pipeline": " ",
   "search.placeholder": "ค้นหา",
   "search.result.placeholder": "พิมพ์เพื่อเริ่มค้นหา",
   "search.result.none": "ไม่พบเอกสารที่ตรงกัน",

material/partials/language/vi.html

@@ -10,6 +10,7 @@
   "footer.next": "Sau",
   "meta.comments": "Bình luận",
   "meta.source": "Mã nguồn",
+  "search.config.lang": "vi",
   "search.placeholder": "Tìm kiếm",
   "search.result.placeholder": "Nhập để bắt đầu tìm kiếm",
   "search.result.none": "Không tìm thấy tài liệu liên quan",

material/partials/language/zh-Hant.html

@@ -11,6 +11,7 @@
   "meta.comments": "評論",
   "meta.source": "來源",
   "search.config.lang": "ja",
+  "search.config.pipeline": "trimmer, stemmer",
   "search.config.separator": "[\，\。]+",
   "search.placeholder": "搜尋",
   "search.result.placeholder": "鍵入以開始檢索",

material/partials/language/zh-TW.html

@@ -11,6 +11,7 @@
   "meta.comments": "留言",
   "meta.source": "來源",
   "search.config.lang": "ja",
+  "search.config.pipeline": "trimmer, stemmer",
   "search.config.separator": "[\s\-　、。，．？；]+",
   "search.placeholder": "搜尋",
   "search.result.placeholder": "打字進行搜尋",

material/partials/language/zh.html

@@ -11,6 +11,7 @@
   "meta.comments": "评论",
   "meta.source": "来源",
   "search.config.lang": "ja",
+  "search.config.pipeline": "trimmer, stemmer",
   "search.config.separator": "[\，\。]+",
   "search.placeholder": "搜索",
   "search.result.placeholder": "键入以开始搜索",

src/assets/javascripts/integrations/search/_/index.ts

@@ -58,9 +58,9 @@ export interface SearchIndexDocument {
  * Search index pipeline function
  */
 export type SearchIndexPipelineFn =
-  | "stemmer"                          /* Stemmer */
-  | "stopWordFilter"                   /* Stop word filter */
   | "trimmer"                          /* Trimmer */
+  | "stopWordFilter"                   /* Stop word filter */
+  | "stemmer"                          /* Stemmer */
 
 /**
  * Search index pipeline
@@ -92,6 +92,25 @@ export interface SearchResult {
   sections: SectionDocument[]          /* Section documents */
 }
 
+/* ----------------------------------------------------------------------------
+ * Functions
+ * ------------------------------------------------------------------------- */
+
+/**
+ * Compute the difference of two lists of strings
+ *
+ * @param a - 1st list of strings
+ * @param b - 2nd list of strings
+ *
+ * @return Difference
+ */
+function difference(a: string[], b: string[]): string[] {
+  const [x, y] = [new Set(a), new Set(b)]
+  return [
+    ...new Set([...x].filter(value => !y.has(value)))
+  ]
+}
+
 /* ----------------------------------------------------------------------------
  * Class
  * ------------------------------------------------------------------------- */
@@ -147,15 +166,20 @@ export class Search {
           this.use((lunr as any).multiLanguage(...config.lang))
         }
 
-        /* Set up pipeline according to configuration */
+        /* Compute functions to be removed from the pipeline */
-        this.pipeline.reset()
+        const fns = difference([
-        for (const fn of pipeline!)
+          "trimmer", "stopWordFilter", "stemmer"
-          this.pipeline.add(lunr[fn])
+        ], pipeline!)
 
-        /* Set up search pipeline according to configuration */
+        /* Remove functions from the pipeline for every language */
-        this.searchPipeline.reset()
+        for (const lang of config.lang.map(language => (
-        for (const fn of pipeline!)
+          language === "en" ? lunr : (lunr as any)[language]
-          this.searchPipeline.add(lunr[fn])
+        ))) {
+          for (const fn of fns) {
+            this.pipeline.remove(lang[fn])
+            this.searchPipeline.remove(lang[fn])
+          }
+        }
 
         /* Set up fields and reference */
         this.field("title", { boost: 1000 })

src/assets/javascripts/integrations/search/transform/index.ts

@@ -51,5 +51,5 @@ export function defaultTransform(value: string): string {
   return value
     .replace(/(?:^|\s+)[*+-:^~]+(?=\s+|$)/g, "")
     .trim()
-    .replace(/\s+|\b$/g, "* ")
+    .replace(/\s+|(?![^\x00-\x7F]|^)$|\b$/g, "* ")
 }

src/partials/language/ar.html

@@ -31,8 +31,7 @@
   "footer.next": "التالية",
   "meta.comments": "التعليقات",
   "meta.source": "المصدر",
-  "search.config.lang": "ar",
+  "search.config.pipeline": " ",
-  "search.config.pipeline": "stemmer",
   "search.placeholder": "بحث",
   "search.result.placeholder": "اكتب لبدء البحث",
   "search.result.none": "لا توجد نتائج",

src/partials/language/cs.html

@@ -30,7 +30,6 @@
   "footer.next": "Další",
   "meta.comments": "Komentáře",
   "meta.source": "Zdroj",
-  "search.config.lang": "ro",
   "search.placeholder": "Hledat",
   "search.result.placeholder": "Pište co se má vyhledat",
   "search.result.none": "Nenalezeny žádné dokumenty",

src/partials/language/et.html

@@ -30,7 +30,6 @@
   "footer.next": "Järgmine",
   "meta.comments": "Kommentaarid",
   "meta.source": "Lähtekood",
-  "search.config.pipeline": "",
   "search.placeholder": "Otsi",
   "search.result.placeholder": "Otsimiseks kirjuta siia",
   "search.result.none": "Otsingule ei leitud ühtegi vastet",

src/partials/language/gr.html

@@ -30,6 +30,7 @@
   "footer.next": "Προηγούμενη",
   "meta.comments": "Σχόλια",
   "meta.source": "Πηγή",
+  "search.config.pipeline": " ",
   "search.placeholder": "Αναζήτηση",
   "search.result.placeholder": "Πληκτρολογήστε για να αρχίσει η αναζήτηση",
   "search.result.none": "Δε βρέθηκαν αντίστοιχα αρχεία",

src/partials/language/he.html

@@ -31,8 +31,7 @@
   "footer.next": "הַבָּא",
   "meta.comments": "הערות",
   "meta.source": "מָקוֹר",
-  "search.config.lang": "ar",
+  "search.config.pipeline": " ",
-  "search.config.pipeline": "stemmer",
   "search.placeholder": "לחפש",
   "search.result.placeholder": "הקלד כדי להתחיל לחפש",
   "search.result.none": "אין מסמכים תואמים",

src/partials/language/hi.html

@@ -30,8 +30,7 @@
   "footer.next": "आगामी",
   "meta.comments": "टिप्पणियाँ",
   "meta.source": "स्रोत",
-  "search.config.lang": "ar"
+  "search.config.pipeline": " ",
-  "search.config.pipeline": "stemmer",
   "search.placeholder": "खोज",
   "search.result.placeholder": "खोज शुरू करने के लिए टाइप करें",
   "search.result.none": "कोई मिलान डॉक्यूमेंट नहीं",

src/partials/language/hr.html

@@ -30,7 +30,6 @@
   "footer.next": "Sljedeće",
   "meta.comments": "Komentari",
   "meta.source": "Izvor",
-  "search.config.pipeline": "",
   "search.placeholder": "Pretraživanje",
   "search.result.placeholder": "Unesite pojam pretraživanja",
   "search.result.none": "Ništa nije pronađeno",

src/partials/language/ja.html

@@ -31,6 +31,7 @@
   "meta.comments": "コメント",
   "meta.source": "ソース",
   "search.config.lang": "ja",
+  "search.config.pipeline": "trimmer, stemmer",
   "search.config.separator": "[\s\-　、。，．]+",
   "search.placeholder": "検索",
   "search.result.placeholder": "検索キーワードを入力してください",

src/partials/language/kr.html

@@ -30,7 +30,6 @@
   "footer.next": "다음",
   "meta.comments": "댓글",
   "meta.source": "출처",
-  "search.config.lang": "ja",
   "search.config.pipeline": " ",
   "search.placeholder": "검색",
   "search.result.placeholder": "검색어를 입력하세요",

src/partials/language/my.html

@@ -33,8 +33,7 @@
   "meta.comments": "မှတ်ချက်များ",
   "meta.source": "ရင်းမြစ်",
   "nav.title": "လမ်းညွှန်",
-  "search.config.lang": "ar",
+  "search.config.pipeline": " ",
-  "search.config.pipeline": "stemmer",
   "search.placeholder": "ရှာရန်",
   "search.reset": "ရှင်းလင်း",
   "search.result.placeholder": "ရှာဖွေခြင်းစရန် စာရိုက်ပါ",

src/partials/language/ru.html

@@ -31,7 +31,6 @@
   "meta.comments": "Комментарии",
   "meta.source": "Исходный код",
   "search.config.lang": "ru",
-  "search.config.pipeline": "stemmer",
   "search.placeholder": "Поиск",
   "search.result.placeholder": "Начните печатать для поиска",
   "search.result.none": "Совпадений не найдено",

src/partials/language/sh.html

@@ -30,7 +30,6 @@
   "footer.next": "Sledeće",
   "meta.comments": "Komentari",
   "meta.source": "Izvor",
-  "search.config.lang": "ro",
   "search.placeholder": "Pretraga",
   "search.result.placeholder": "Unesite pojam pretrage",
   "search.result.none": "Ništa nije pronađeno",

src/partials/language/sr.html

@@ -31,7 +31,6 @@
   "meta.comments": "Коментари",
   "meta.source": "Извор",
   "search.config.lang": "ru",
-  "search.config.pipeline": "stemmer",
   "search.placeholder": "Претрага",
   "search.result.placeholder": "Унесите појам претраге",
   "search.result.none": "Нису пронађени документи",

src/partials/language/th.html

@@ -30,6 +30,7 @@
   "footer.next": "ต่อไป",
   "meta.comments": "ความคิดเห็น",
   "meta.source": "แหล่งที่มา",
+  "search.config.pipeline": " ",
   "search.placeholder": "ค้นหา",
   "search.result.placeholder": "พิมพ์เพื่อเริ่มค้นหา",
   "search.result.none": "ไม่พบเอกสารที่ตรงกัน",

src/partials/language/vi.html

@@ -30,6 +30,7 @@
   "footer.next": "Sau",
   "meta.comments": "Bình luận",
   "meta.source": "Mã nguồn",
+  "search.config.lang": "vi",
   "search.placeholder": "Tìm kiếm",
   "search.result.placeholder": "Nhập để bắt đầu tìm kiếm",
   "search.result.none": "Không tìm thấy tài liệu liên quan",

src/partials/language/zh-Hant.html

@@ -31,6 +31,7 @@
   "meta.comments": "評論",
   "meta.source": "來源",
   "search.config.lang": "ja",
+  "search.config.pipeline": "trimmer, stemmer",
   "search.config.separator": "[\，\。]+",
   "search.placeholder": "搜尋",
   "search.result.placeholder": "鍵入以開始檢索",

src/partials/language/zh-TW.html

@@ -31,6 +31,7 @@
   "meta.comments": "留言",
   "meta.source": "來源",
   "search.config.lang": "ja",
+  "search.config.pipeline": "trimmer, stemmer",
   "search.config.separator": "[\s\-　、。，．？；]+",
   "search.placeholder": "搜尋",
   "search.result.placeholder": "打字進行搜尋",

src/partials/language/zh.html

@@ -31,6 +31,7 @@
   "meta.comments": "评论",
   "meta.source": "来源",
   "search.config.lang": "ja",
+  "search.config.pipeline": "trimmer, stemmer",
   "search.config.separator": "[\，\。]+",
   "search.placeholder": "搜索",
   "search.result.placeholder": "键入以开始搜索",