Merge pull request #147 from squidfunk/chore/setup-gemini-test-environment

Setup visual regression testing

.eslintignore

@@ -18,7 +18,11 @@
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 # IN THE SOFTWARE.
 
-# Build files
+# Files generated by build
 /build
 /material
 /site
+
+# Files generated by visual tests
+/gemini-report
+/tests/visual/data

.githooks/post-merge/npm-update.sh

@@ -25,6 +25,6 @@ CHANGED="$(git diff-tree -r --name-only --no-commit-id ORIG_HEAD HEAD)"
 
 # Perform install and prune of NPM dependencies if package.json changed
 if $(echo "$CHANGED" | grep --quiet package.json); then
-  echo "Hook[post-merge]: Updating dependencies..."
+  echo "Hook[post-merge]: Updating dependencies"
   npm install && npm prune
 fi

.githooks/pre-commit/branch.sh

@@ -22,7 +22,7 @@
 
 # Determine current branch
 BRANCH=$(git rev-parse --abbrev-ref HEAD)
-echo "Hook[pre-commit]: Checking branch..."
+echo "Hook[pre-commit]: Checking branch"
 
 # If we're on master, abort commit
 if [[ "$BRANCH" == "master" ]]; then

.githooks/pre-commit/lint.sh

@@ -46,6 +46,6 @@ FILES=$(git diff --cached --name-only --diff-filter=ACMR | \
 
 # Run the check and print indicator
 if [ "$FILES" ]; then
-  echo "Hook[pre-commit]: Running linter..."
+  echo "Hook[pre-commit]: Running linter"
   npm run lint --silent || exit 1
 fi

.gitignore

@@ -23,14 +23,19 @@
 
 # NPM-related
 /node_modules
-/npm-debug.log
+/npm-debug.log*
 
-# Build files
+# Files generated by build
 /build
 /manifest.json
 /MANIFEST
 /site
 
+# Files generated by visual tests
+/gemini-report
+/tests/visual/baseline/local
+/tests/visual/data
+
 # Distribution files
 /dist
 /mkdocs_material.egg-info

.travis.yml

@@ -27,6 +27,29 @@ node_js:
   - 6
   - 7
 
+# Build visual tests separately
+matrix:
+  include:
+    - node_js: 5
+      addons:
+        artifacts:
+          paths:
+            - gemini-report
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - gcc-4.8
+            - g++-4.8
+      env:
+        - CXX=g++-4.8
+      install: yarn install
+      script: yarn run test:visual:run
+
+# Limit clone depth to 5, to speed up build
+git:
+  depth: 5
+
 # Cache dependencies
 cache:
   pip: true
@@ -34,9 +57,14 @@ cache:
   directories:
     - node_modules
 
+# Install yarn as Travis doesn't support it out of the box
+before_install: npm install -g yarn
+
+# Do not install optional dependencies by default
+install: yarn install --ignore-optional
+
 # Install dependencies
-before_script:
-  - pip install --user -r requirements.txt
+before_script: pip install --user -r requirements.txt
 
 # Perform build and tests
 script: yarn run build

Gulpfile.babel.js

@@ -35,14 +35,17 @@ const config = {
     src: "src/assets",                 /* Source directory for assets */
     build: "material/assets"           /* Target directory for assets */
   },
-  lib: "lib",                          /* Libraries */
+  lib: "lib",                          /* Libraries and tasks */
+  tests: {
+    visual: "tests/visual"             /* Base directory for visual tests */
+  },
   views: {
     src: "src",                        /* Source directory for views */
     build: "material"                  /* Target directory for views */
   }
 }
 
-const args = yargs
+let args = yargs
   .default("clean",      false)        /* Clean before build */
   .default("karma",      true)         /* Karma watchdog */
   .default("lint",       true)         /* Lint sources */
@@ -52,6 +55,12 @@ const args = yargs
   .default("sourcemaps", false)        /* Create sourcemaps */
   .argv
 
+/* Only use the last value seen, so overrides are possible */
+args = Object.keys(args).reduce((result, arg) => {
+  result[arg] = [].concat(args[arg]).pop()
+  return result
+}, {})
+
 /* ----------------------------------------------------------------------------
  * Overrides and helpers
  * ------------------------------------------------------------------------- */
@@ -92,7 +101,10 @@ gulp.src = (...glob) => {
  * Helper function to load a task
  */
 const load = task => {
-  return require(`./${config.lib}/tasks/${task}`)(gulp, config, args)
+  return done => {
+    return require(`./${config.lib}/tasks/${task}`)
+      .call(gulp, gulp, config, args)(done)
+  }
 }
 
 /* ----------------------------------------------------------------------------
@@ -228,12 +240,11 @@ gulp.task("assets:clean", [
  * Minify views
  */
 gulp.task("views:build", (args.revision ? [
-  "assets:images:build",
-  "assets:stylesheets:build",
-  "assets:javascripts:build"
+  "assets:build"
 ] : []).concat(args.clean ? [
   "views:clean"
-] : []), load("views/build"))
+] : []),
+  load("views/build"))
 
 /*
  * Clean views
@@ -267,14 +278,45 @@ gulp.task("mkdocs:serve",
   load("mkdocs/serve"))
 
 /* ----------------------------------------------------------------------------
- * Tests
+ * Visual tests
  * ------------------------------------------------------------------------- */
 
 /*
- * Start karma test runner
+ * Generate visual tests
  */
-gulp.task("tests:unit:watch",
-  load("tests/unit/watch"))
+gulp.task("tests:visual:generate", [
+].concat(args.clean ? [
+  "tests:visual:clean",
+  "assets:build",
+  "views:build"
+] : []),
+  load("tests/visual/generate"))
+
+/*
+ * Run visual tests
+ */
+gulp.task("tests:visual:run", [
+  "tests:visual:generate"
+], load("tests/visual/run"))
+
+/*
+ * Update reference images for visual tests
+ */
+gulp.task("tests:visual:update",
+  load("tests/visual/update"))
+
+/*
+ * Clean files generated by visual tests
+ */
+gulp.task("tests:visual:clean",
+  load("tests/visual/clean"))
+
+/*
+ * Open a SauceConnect session for manual testing
+ */
+gulp.task("tests:visual:session", [
+  "tests:visual:generate"
+], load("tests/visual/session"))
 
 /* ----------------------------------------------------------------------------
  * Interface
@@ -286,9 +328,9 @@ gulp.task("tests:unit:watch",
 gulp.task("build", [
   "assets:build",
   "views:build"
-].concat(args.mkdocs
-  ? "mkdocs:build"
-  : []))
+].concat(args.mkdocs ? [
+  "mkdocs:build"
+] : []))
 
 /*
  * Clean assets and documentation

lib/.eslintrc

@@ -0,0 +1,6 @@
+{
+  "rules": {
+    "no-invalid-this": 0,
+    "max-params": 0
+  }
+}

lib/providers/jsx.js

@@ -21,7 +21,7 @@
  */
 
 /* ----------------------------------------------------------------------------
- * Definition
+ * Module
  * ------------------------------------------------------------------------- */
 
 export default /* JSX */ {

lib/servers/ecstatic.js

@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2016-2017 Martin Donath <martin.donath@squidfunk.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+import ecstatic from "ecstatic"
+import * as http from "http"
+
+/* ----------------------------------------------------------------------------
+ * Locals
+ * ------------------------------------------------------------------------- */
+
+/* Static file server */
+let server = null
+
+/* ----------------------------------------------------------------------------
+ * Functions
+ * ------------------------------------------------------------------------- */
+
+/**
+ * Start static file server
+ *
+ * @param {string} directory - Directory to serve
+ * @param {number} port - Port to listen on
+ * @param {Function} done - Resolve callback
+ */
+export const start = (directory, port, done) => {
+  server = http.createServer(ecstatic({
+    root: directory
+  }))
+
+  /* Listen and register signal handlers */
+  server.listen(port, "127.0.0.1", done)
+  for (const signal of ["SIGTERM", "SIGINT", "exit"])
+    process.on(signal, stop)
+}
+
+/**
+ * Stop static file server
+ *
+ * @param {Function} done - Resolve callback
+ */
+export const stop = done => {
+  if (server) {
+    server.close(done)
+    server = null
+  }
+}

lib/servers/sauce-connect.js

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2016-2017 Martin Donath <martin.donath@squidfunk.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+import launcher from "sauce-connect-launcher"
+
+/* ----------------------------------------------------------------------------
+ * Locals
+ * ------------------------------------------------------------------------- */
+
+/* SauceConnect process */
+let server = null
+
+/* ----------------------------------------------------------------------------
+ * Functions
+ * ------------------------------------------------------------------------- */
+
+/**
+ * Open SauceConnect tunnel
+ *
+ * @param {string} id - Unique identifier
+ * @param {string} username - SauceConnect username
+ * @param {string} accesskey - SauceConnect accesskey
+ * @param {Function} done - Resolve callback
+ */
+export const start = (id, username, accesskey, done) => {
+  launcher({
+    username,
+    accessKey: accesskey,
+    tunnelIdentifier: id
+  }, (err, proc) => {
+    if (err)
+      throw new Error(err)
+    server = proc
+    done()
+  })
+
+  /* Register signal handlers */
+  for (const signal of ["SIGTERM", "SIGINT", "exit"])
+    process.on(signal, stop)
+}
+
+/**
+ * Close SauceConnect tunnel
+ *
+ * @param {Function} done - Resolve callback
+ */
+export const stop = done => {
+  if (server) {
+    server.close(done)
+    server = null
+  }
+}

lib/servers/selenium.js

@@ -33,19 +33,33 @@ let server = null
  * Definition
  * ------------------------------------------------------------------------- */
 
+/**
+ * Start Selenium
+ *
+ * @param {Function} done - Resolve callback
+ */
 export const start = done => {
   selenium.start({}, (err, proc) => {
+
+    /* Register signal handlers */
+    for (const signal of ["SIGTERM", "SIGINT", "exit"])
+      process.on(signal, stop)
     if (err) {
 
       /* Install selenium, if not present */
       if (/^Missing(.*)chromedriver$/.test(err.message)) {
-        selenium.install(done)
-
-        /* Start selenium again */
-        selenium.start({}, (err_, proc_) => {
-          server = proc_
+        new Promise(resolve => {
+          selenium.install({}, resolve)
         })
 
+          /* Start selenium again */
+          .then(() => {
+            selenium.start({}, (err_, proc_) => {
+              server = proc_
+              done()
+            })
+          })
+
       /* Otherwise, throw error */
       } else {
         throw err
@@ -53,20 +67,21 @@ export const start = done => {
     }
 
     /* Remember process handle */
-    server = server || proc
+    server = proc
     done()
   })
 }
 
-export const stop = () => {
-  if (server)
+/**
+ * Stop Selenium
+ *
+ * @param {Function} done - Resolve callback
+ */
+export const stop = done => {
+  if (server) {
+    if (typeof done === "function")
+      server.on("exit", done)
     server.kill()
+    server = null
+  }
 }
-
-/* ----------------------------------------------------------------------------
- * Signal handler
- * ------------------------------------------------------------------------- */
-
-/* Register signal handler for all relevant events */
-for (const signal of ["SIGTERM", "SIGINT", "exit"])
-  process.on(signal, stop)

lib/tasks/.eslintrc

@@ -1,5 +0,0 @@
-{
-  "rules": {
-    "no-invalid-this": 0
-  }
-}

lib/tasks/assets/javascripts/build/application.js

@@ -101,7 +101,7 @@ export default (gulp, config, args) => {
           },
 
           /* Sourcemap support */
-          devtool: args.sourcemaps ? "source-map" : ""
+          devtool: args.sourcemaps ? "inline-source-map" : ""
         }))
 
       /* Revisioning */

lib/tasks/assets/javascripts/lint.js

@@ -23,6 +23,7 @@
 import path from "path"
 import through from "through2"
 import util from "gulp-util"
+
 import { CLIEngine } from "eslint"
 
 /* ----------------------------------------------------------------------------

lib/tasks/assets/stylesheets/build.js

@@ -25,6 +25,7 @@ import gulpif from "gulp-if"
 import mincss from "gulp-cssnano"
 import mqpacker from "css-mqpacker"
 import postcss from "gulp-postcss"
+import pseudoclasses from "postcss-pseudo-classes"
 import rev from "gulp-rev"
 import sass from "gulp-sass"
 import sourcemaps from "gulp-sourcemaps"
@@ -54,7 +55,11 @@ export default (gulp, config, args) => {
         postcss([
           autoprefixer(),
           mqpacker
-        ]))
+        ].concat(!args.optimize ? [
+          pseudoclasses({
+            "restrictTo": ["hover", "focus"]
+          })
+        ] : [])))
 
       /* Minify sources */
       .pipe(gulpif(args.optimize, mincss()))
@@ -63,7 +68,7 @@ export default (gulp, config, args) => {
       .pipe(gulpif(args.revision, rev()))
       .pipe(gulpif(args.revision,
         version({ manifest: gulp.src("manifest.json") })))
-      .pipe(gulpif(args.sourcemaps, sourcemaps.write(".")))
+      .pipe(gulpif(args.sourcemaps, sourcemaps.write()))
       .pipe(gulp.dest(`${config.assets.build}/stylesheets`))
       .pipe(gulpif(args.revision,
         rev.manifest("manifest.json", {

lib/tasks/mkdocs/serve.js

@@ -39,7 +39,7 @@ export default () => {
       server.kill()
 
     /* Spawn MkDocs server */
-    server = child.spawn("mkdocs", ["serve", "-a", "0.0.0.0:8000"], {
+    server = child.spawn("mkdocs", ["serve", "--dev-addr", "0.0.0.0:8000"], {
       stdio: "inherit"
     })
   }

lib/tasks/tests/visual/clean.js

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2016-2017 Martin Donath <martin.donath@squidfunk.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+import clean from "del"
+import vinyl from "vinyl-paths"
+
+/* ----------------------------------------------------------------------------
+ * Task: clean files generated by visual tests
+ * ------------------------------------------------------------------------- */
+
+export default (gulp, config) => {
+  return () => {
+    return gulp.src([
+      `${config.tests.visual}/data`,
+      "./gemini-report"
+    ])
+      .pipe(vinyl(clean))
+  }
+}

lib/tasks/tests/visual/generate.js

@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2016-2017 Martin Donath <martin.donath@squidfunk.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+import child from "child_process"
+import path from "path"
+import through from "through2"
+import util from "gulp-util"
+
+/* ----------------------------------------------------------------------------
+ * Task: generate visual tests
+ * ------------------------------------------------------------------------- */
+
+export default (gulp, config) => {
+  const theme = path.resolve(process.cwd(), config.views.build)
+  return () => {
+    return gulp.src(`${config.tests.visual}/suites/**/mkdocs.yml`)
+      .pipe(
+        through.obj(function(file, enc, done) {
+          if (file.isNull() || file.isStream())
+            return done()
+
+          /* Resolve test name and destination */
+          const name = path.relative(`${config.tests.visual}/suites`,
+            path.dirname(file.path))
+          const site = path.resolve(process.cwd(),
+            `${config.tests.visual}/data`, name, "_")
+
+          /* Generate test fixtures with freshly built theme */
+          const proc = child.spawnSync("mkdocs", [
+            "build", "--site-dir", site, "--theme-dir", theme
+          ], {
+            cwd: path.dirname(file.path)
+          })
+
+            /* Emit error, if any */
+          if (proc.status)
+            this.emit("error", new util.PluginError("mkdocs",
+                `Terminated with errors: ${proc.stderr.toString()}`))
+
+          /* Terminate */
+          done()
+        }))
+  }
+}

lib/tasks/tests/visual/run.js

@@ -0,0 +1,125 @@
+/*
+ * Copyright (c) 2016-2017 Martin Donath <martin.donath@squidfunk.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+import moniker from "moniker"
+import path from "path"
+import * as ecstatic from "~/lib/servers/ecstatic"
+import * as sauce from "~/lib/servers/sauce-connect"
+import * as selenium from "~/lib/servers/selenium"
+
+import Gemini from "gemini"
+
+/* ----------------------------------------------------------------------------
+ * Task: run visual tests
+ * ------------------------------------------------------------------------- */
+
+export default (gulp, config, args) => {
+  const id = process.env.TRAVIS
+    ? `Travis #${process.env.TRAVIS_BUILD_NUMBER}`
+    : `Local #${moniker.choose()}`
+  return done => {
+
+    /* Start static file server */
+    let error = false
+    new Promise(resolve => {
+      ecstatic.start(`${config.tests.visual}/data`, 8000, resolve)
+
+    /* Create and start test runner */
+    }).then(() => {
+      return new Promise((resolve, reject) => {
+
+        /* Start SauceConnect tunnel */
+        if (process.env.CI || process.env.SAUCE) {
+          if (!process.env.SAUCE_USERNAME ||
+              !process.env.SAUCE_ACCESS_KEY)
+            throw new Error(
+              "SauceConnect: please provide SAUCE_USERNAME " +
+              "and SAUCE_ACCESS_KEY")
+
+          /* Start tunnel, if credentials are given */
+          sauce.start(
+            id,
+            process.env.SAUCE_USERNAME,
+            process.env.SAUCE_ACCESS_KEY,
+            err => {
+              return err ? reject(err) : resolve(sauce)
+            })
+
+        /* Start Selenium */
+        } else {
+          selenium.start(() => resolve(selenium))
+        }
+      })
+
+        /* Setup and run Gemini */
+        .then(runner => {
+          const gemini = require(
+            path.join(process.cwd(), `${config.tests.visual}/config`,
+              process.env.CI || process.env.SAUCE
+                ? "gemini.sauce-connect.json"
+                : "gemini.selenium.json"))
+
+          /* Add dynamic configuration to capabilities */
+          for (const key of Object.keys(gemini.browsers)) {
+            const caps = gemini.browsers[key].desiredCapabilities
+            caps.tunnelIdentifier = id
+            caps.public = "private"
+            caps.name = id
+
+            /* Adjust configuration for Travis CI */
+            if (process.env.CI && process.env.TRAVIS)
+              caps.public = "public"
+          }
+
+          /* Start Gemini and return runner upon finish */
+          return new Gemini(gemini).test(`${config.tests.visual}/suites`, {
+            reporters: ["flat", "html"],
+            browsers: args.browsers ? [].concat(args.browsers) : null
+          })
+
+            /* Return runner for graceful stop */
+            .then(status => {
+              error = status.failed + status.errored > 0
+              return runner
+            })
+        })
+
+        /* Stop test runner */
+        .then(runner => {
+          return new Promise(resolve => {
+            runner.stop(resolve)
+          })
+        })
+
+    /* Stop static file server */
+    })
+      .then(() => {
+        ecstatic.stop(() => {
+          return error
+            ? done(new Error("Gemini terminated with errors"))
+            : done()
+        })
+      }, err => {
+        return done(new Error(err))
+      })
+  }
+}

lib/tasks/tests/visual/session.js

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2016-2017 Martin Donath <martin.donath@squidfunk.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+import moniker from "moniker"
+import * as ecstatic from "~/lib/servers/ecstatic"
+import * as sauce from "~/lib/servers/sauce-connect"
+
+/* ----------------------------------------------------------------------------
+ * Task: run visual tests
+ * ------------------------------------------------------------------------- */
+
+export default (gulp, config) => {
+  return done => {
+
+    /* Start static file server */
+    new Promise(resolve => {
+      ecstatic.start(`${config.tests.visual}/data`, 8000, resolve)
+
+    /* Open SauceConnect tunnel */
+    }).then(() => {
+      return new Promise((resolve, reject) => {
+        if (!process.env.SAUCE_USERNAME ||
+            !process.env.SAUCE_ACCESS_KEY)
+          throw new Error(
+            "SauceConnect: please provide SAUCE_USERNAME " +
+            "and SAUCE_ACCESS_KEY")
+
+        /* Open tunnel */
+        sauce.start(
+          `Local #${moniker.choose()}`,
+          process.env.SAUCE_USERNAME,
+          process.env.SAUCE_ACCESS_KEY,
+          err => {
+            return err ? reject(err) : resolve(sauce)
+          })
+      })
+
+        /* Close tunnel on CTRL-C */
+        .then(() => {
+          return new Promise(resolve => {
+            process.on("SIGINT", () => {
+              sauce.stop(resolve)
+            })
+          })
+        })
+
+    /* Stop static file server */
+    })
+      .then(() => {
+        ecstatic.stop(done)
+      }, err => {
+        return done(err)
+      })
+  }
+}

lib/tasks/tests/visual/update.js

@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2016-2017 Martin Donath <martin.donath@squidfunk.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+import fs from "fs"
+import path from "path"
+import through from "through2"
+
+/* ----------------------------------------------------------------------------
+ * Task: update reference images for visual tests
+ * ------------------------------------------------------------------------- */
+
+export default (gulp, config) => {
+  return () => {
+    const base = path.join(
+      process.cwd(), `${config.tests.visual}/config`)
+
+    /* Read Gemini configs and map browsers to screenshot directories */
+    const mapping = fs.readdirSync(base)
+      .reduce((result, filename) => {
+        return Object.assign(result, (gemini => {
+          return Object.keys(gemini.browsers)
+            .reduce((browsers, name) => {
+              browsers[name] = gemini.screenshotsDir
+              return browsers
+            }, {})
+        })(require(path.join(base, filename))))
+      }, {})
+
+    /* Prepare filenames */
+    const dest = path.join(process.cwd(), `${config.tests.visual}/baseline`)
+    return gulp.src("gemini-report/images/**/*~current.png")
+      .pipe(
+        through.obj(function(file, enc, done) {
+          if (file.isNull() || file.isStream())
+            return done()
+
+          /* Remove the state from the filename */
+          file.path = file.path.replace("~current", "")
+
+          /* Retrieve the folder for the environment of the baseline */
+          const folder = path.relative(dest,
+            mapping[path.basename(file.path, ".png")])
+          file.path = file.path.replace("images", `images/${folder}`)
+
+          /* Push file to next stage */
+          this.push(file)
+          done()
+        }))
+
+      /* Update reference images */
+      .pipe(gulp.dest(dest))
+  }
+}

material/base.html

@@ -19,7 +19,7 @@
       {% else %}
         <link rel="shortcut icon" href="{{ base_url }}/assets/images/favicon.ico">
       {% endif %}
-      <meta name="generator" content="mkdocs+mkdocs-material#1.0.3">
+      <meta name="generator" content="mkdocs-{{ mkdocs_version }}, mkdocs-material-1.0.3">
     {% endblock %}
     {% block htmltitle %}
       {% if page.title %}
@@ -31,7 +31,13 @@
       {% endif %}
     {% endblock %}
     {% block libs %}
-      <script src="{{ base_url }}/assets/javascripts/modernizr-facb31f4a3.js"></script>
+      <script src="{{ base_url }}/assets/javascripts/modernizr-5b0c41c2b5.js"></script>
+    {% endblock %}
+    {% block styles %}
+      <link rel="stylesheet" href="{{ base_url }}/assets/stylesheets/application-ee6a3f36b0.css">
+      {% if config.extra.palette %}
+        <link rel="stylesheet" href="{{ base_url }}/assets/stylesheets/application-02ce7adcc2.palette.css">
+      {% endif %}
     {% endblock %}
     {% block fonts %}
       {% if config.extra.font != "none" %}
@@ -44,15 +50,9 @@
       {% endif %}
       <link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons">
     {% endblock %}
-    {% block styles %}
-      <link rel="stylesheet" href="{{ base_url }}/assets/stylesheets/application-bc099a55ca.css">
-      {% if config.extra.palette %}
-        <link rel="stylesheet" href="{{ base_url }}/assets/stylesheets/application-02ce7adcc2.palette.css">
-      {% endif %}
-      {% for path in extra_css %}
-        <link rel="stylesheet" href="{{ path }}">
-      {% endfor %}
-    {% endblock %}
+    {% for path in extra_css %}
+      <link rel="stylesheet" href="{{ path }}">
+    {% endfor %}
     {% block extrahead %}{% endblock %}
   </head>
   {% set palette = config.extra.get("palette", {}) %}

mkdocs.yml

@@ -31,7 +31,7 @@ repo_url: https://github.com/squidfunk/mkdocs-material
 # Copyright
 copyright: 'Copyright © 2016 - 2017 Martin Donath'
 
-# Documentation and theme
+# Theme directory
 theme_dir: material
 
 # Options

package.json

@@ -27,7 +27,9 @@
     "clean": "scripts/clean",
     "lint": "scripts/lint",
     "start": "scripts/start",
-    "test": "scripts/test"
+    "test:visual:run": "scripts/test/visual/run",
+    "test:visual:update": "scripts/test/visual/update",
+    "test:visual:session": "scripts/test/visual/session"
   },
   "dependencies": {},
   "devDependencies": {
@@ -45,6 +47,7 @@
     "css-mqpacker": "^5.0.1",
     "custom-event-polyfill": "^0.3.0",
     "del": "^2.2.2",
+    "ecstatic": "^2.1.0",
     "eslint": "^3.14.0",
     "eslint-plugin-mocha": "^4.8.0",
     "fastclick": "^1.0.6",
@@ -83,8 +86,9 @@
     "material-shadows": "^3.0.1",
     "mocha": "^3.2.0",
     "modularscale-sass": "^2.1.1",
+    "moniker": "^0.1.2",
     "node-notifier": "^5.0.0",
-    "selenium-standalone": "^6.0.0",
+    "postcss-pseudo-classes": "^0.1.0",
     "stylelint": "^7.8.0",
     "stylelint-config-standard": "^16.0.0",
     "stylelint-order": "^0.2.2",
@@ -96,6 +100,11 @@
     "whatwg-fetch": "^2.0.1",
     "yargs": "^6.6.0"
   },
+  "optionalDependencies": {
+    "gemini": "^4.14.3",
+    "sauce-connect-launcher": "^1.2.0",
+    "selenium-standalone": "^6.0.0"
+  },
   "engines": {
     "node": ">= 4.5.0"
   },

requirements.txt

@@ -20,4 +20,4 @@
 
 mkdocs>=0.16
 pygments
-pymdown-extensions
+pymdown-extensions>=1.2

scripts/build

@@ -28,4 +28,4 @@ if [[ ! -d `npm bin` ]]; then
 fi
 
 # Run command
-`npm bin`/gulp build --clean --optimize --revision
+`npm bin`/gulp build --clean --optimize --revision "$@"

scripts/clean

@@ -28,4 +28,4 @@ if [[ ! -d `npm bin` ]]; then
 fi
 
 # Run command
-`npm bin`/gulp clean
+`npm bin`/gulp clean "$@"

scripts/lint

@@ -28,7 +28,7 @@ if [[ ! -d `npm bin` ]]; then
 fi
 
 # Run ESLint
-`npm bin`/eslint .
+`npm bin`/eslint --max-warnings 0 .
 ESLINT=$?
 
 # Run Stylelint

scripts/start

@@ -28,4 +28,4 @@ if [[ ! -d `npm bin` ]]; then
 fi
 
 # Run command
-`npm bin`/gulp watch --no-lint
+`npm bin`/gulp watch --no-lint "$@"

scripts/test/visual/run

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Copyright (c) 2016-2017 Martin Donath <martin.donath@squidfunk.com>
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+# Check if "npm install" was executed
+if [[ ! -d `npm bin` ]]; then
+  echo "\"node_modules\" not found:"
+  echo "npm install"
+  exit 1
+fi
+
+# Run command
+`npm bin`/gulp tests:visual:run --clean --no-optimize "$@"

scripts/test/visual/session

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Copyright (c) 2016-2017 Martin Donath <martin.donath@squidfunk.com>
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+# Check if "npm install" was executed
+if [[ ! -d `npm bin` ]]; then
+  echo "\"node_modules\" not found:"
+  echo "npm install"
+  exit 1
+fi
+
+# Run command
+`npm bin`/gulp tests:visual:session "$@"

scripts/test/visual/update

@@ -28,4 +28,4 @@ if [[ ! -d `npm bin` ]]; then
 fi
 
 # Run command
-`npm bin`/gulp test
+`npm bin`/gulp tests:visual:update "$@"

src/assets/stylesheets/base/_typeset.scss

@@ -34,14 +34,9 @@ body {
 body,
 input {
   color: $md-color-black;
-  // font-family: "Roboto", "Helvetica Neue", Helvetica, Arial, sans-serif;
   font-feature-settings: "kern", "onum", "liga";
+  font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
   font-weight: 400;
-
-  // Use system fonts, if browser doesn't support webfonts
-  .no-fontface & {
-    font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
-  }
 }
 
 // Proportionally spaced fonts
@@ -49,14 +44,9 @@ pre,
 code,
 kbd {
   color: $md-color-black;
-  // font-family: "Roboto Mono", "Courier New", Courier, monospace;
   font-feature-settings: "kern", "onum", "liga";
+  font-family: "Courier New", Courier, monospace;
   font-weight: 400;
-
-  // Use system fonts, if browser doesn't support webfonts
-  .no-fontface & {
-    font-family: "Courier New", Courier, monospace;
-  }
 }
 
 // ----------------------------------------------------------------------------

src/assets/stylesheets/layout/_footer.scss

@@ -50,8 +50,8 @@
     padding-bottom: 0.8rem;
     transition: opacity 0.25s;
 
-    // [mobile landscape +]: Set proportional width
-    @include break-from-device(mobile landscape) {
+    // [tablet +]: Set proportional width
+    @include break-from-device(tablet) {
       width: 50%;
     }
 
@@ -68,8 +68,8 @@
       // Title
       .md-footer-nav__title {
 
-        // [mobile portrait -]: Hide title for previous page
-        @include break-to-device(mobile portrait) {
+        // [mobile -]: Hide title for previous page
+        @include break-to-device(mobile) {
           display: none;
         }
       }

src/assets/stylesheets/layout/_nav.scss

@@ -84,7 +84,7 @@
     }
   }
 
-  // Icon buttons
+  // Button with logo
   &__button {
     @extend %md-icon, %md-icon__button;
 
@@ -140,7 +140,8 @@
       color: $md-color-primary;
     }
 
-    // Hovered item
+    // Focused or hovered item
+    &:focus,
     &:hover {
       color: $md-color-accent;
     }

src/base.html

@@ -56,7 +56,8 @@
       {% endif %}
 
       <!-- Generator banner -->
-      <meta name="generator" content="mkdocs+$theme-name$#$theme-version$" />
+      <meta name="generator"
+        content="mkdocs-{{ mkdocs_version }}, $theme-name$-$theme-version$" />
     {% endblock %}
 
     <!-- Block: site title -->
@@ -75,6 +76,20 @@
       <script src="{{ base_url }}/assets/javascripts/modernizr.js"></script>
     {% endblock %}
 
+    <!-- Block: stylesheets -->
+    {% block styles %}
+
+      <!-- Theme-related stylesheets -->
+      <link rel="stylesheet" type="text/css"
+          href="{{ base_url }}/assets/stylesheets/application.css" />
+
+      <!-- Extra color palette -->
+      {% if config.extra.palette %}
+        <link rel="stylesheet" type="text/css"
+            href="{{ base_url }}/assets/stylesheets/application.palette.css" />
+      {% endif %}
+    {% endblock %}
+
     <!-- Block: webfonts -->
     {% block fonts %}
       {% if config.extra.font != "none" %}
@@ -101,24 +116,10 @@
           href="https://fonts.googleapis.com/icon?family=Material+Icons" />
     {% endblock %}
 
-    <!-- Block: stylesheets -->
-    {% block styles %}
-
-      <!-- Theme-related stylesheets -->
-      <link rel="stylesheet" type="text/css"
-          href="{{ base_url }}/assets/stylesheets/application.css" />
-
-      <!-- Extra color palette -->
-      {% if config.extra.palette %}
-        <link rel="stylesheet" type="text/css"
-            href="{{ base_url }}/assets/stylesheets/application.palette.css" />
-      {% endif %}
-
-      <!-- Custom stylesheets -->
-      {% for path in extra_css %}
-        <link rel="stylesheet" type="text/css" href="{{ path }}" />
-      {% endfor %}
-    {% endblock %}
+    <!-- Custom stylesheets -->
+    {% for path in extra_css %}
+      <link rel="stylesheet" type="text/css" href="{{ path }}" />
+    {% endfor %}
 
     <!-- Block: custom front matter -->
     {% block extrahead %}{% endblock %}

tests/visual/.eslintrc

@@ -0,0 +1,8 @@
+{
+  "globals": {
+    "gemini": true
+  },
+  "rules": {
+    "no-loop-func": 0
+  }
+}