diff --git a/.circleci/config.yml b/.circleci/config.yml
index bf4404ce95f..e3ac14d862c 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -5,9 +5,10 @@ gendoc: &gendoc
     scripts/gendoc.py
 
 genswagger: &genswagger
-  name: Generate the swagger
+  name: Validate sources and generate swagger json
   command: |
     source /env/bin/activate
+    scripts/check-swagger-sources.py
     scripts/dump-swagger.py
 
 buildswaggerui: &buildswaggerui
@@ -27,10 +28,7 @@ checkexamples: &checkexamples
   name: Check Event Examples
   command: |
     source /env/bin/activate
-    cd event-schemas
-    ./check_examples.py
-    cd ../api
-    ./check_examples.py
+    scripts/check-event-schema-examples.py
 
 genmatrixassets: &genmatrixassets
   name: Generate/Verify matrix.org assets
@@ -41,9 +39,9 @@ genmatrixassets: &genmatrixassets
 validateapi: &validateapi
   name: Validate OpenAPI specifications
   command: |
-    cd api
+    cd scripts
     npm install
-    node validator.js -s "client-server"
+    node validator.js -s "../data/api/client-server"
 
 buildspeculator: &buildspeculator
   name: Build Speculator
diff --git a/event-schemas/check_examples.py b/scripts/check-event-schema-examples.py
similarity index 90%
rename from event-schemas/check_examples.py
rename to scripts/check-event-schema-examples.py
index 31daa478315..50f9de7becc 100755
--- a/event-schemas/check_examples.py
+++ b/scripts/check-event-schema-examples.py
@@ -128,7 +128,14 @@ def check_example_dir(exampledir, schemadir):
 
 
 if __name__ == '__main__':
+    # Get the directory that this script is residing in
+    script_directory = os.path.dirname(os.path.realpath(__file__))
+
+    # Resolve the directories to check, relative to the script path
+    examples_directory = os.path.join(script_directory, "../event-schemas/examples")
+    schema_directory = os.path.join(script_directory, "../event-schemas/schema")
+
     try:
-        check_example_dir("examples", "schema")
+        check_example_dir(examples_directory, schema_directory)
     except:
         sys.exit(1)
diff --git a/api/check_examples.py b/scripts/check-swagger-sources.py
similarity index 69%
rename from api/check_examples.py
rename to scripts/check-swagger-sources.py
index 94f3495e3c6..df99b7c004d 100755
--- a/api/check_examples.py
+++ b/scripts/check-swagger-sources.py
@@ -108,13 +108,36 @@ def check_swagger_file(filepath):
 
 
 def resolve_references(path, schema):
+    """Recurse through a given schema until we find a $ref key. Upon doing so,
+    check that the referenced file exists, then load it up and check all of the
+    references in that file. Continue on until we've hit all dead ends.
+
+    $ref values are deleted from schemas as they are validated, to prevent
+    duplicate work.
+    """
     if isinstance(schema, dict):
         # do $ref first
         if '$ref' in schema:
-            value = schema['$ref']
-            path = os.path.abspath(os.path.join(os.path.dirname(path), value))
-            ref = load_file("file://" + path)
-            result = resolve_references(path, ref)
+            # Pull the referenced filepath from the schema
+            referenced_file = schema['$ref']
+
+            # Referenced filepaths are relative, so take the current path's
+            # directory and append the relative, referenced path to it.
+            inner_path = os.path.join(os.path.dirname(path), referenced_file)
+
+            # Then convert the path (which may contiain '../') into a
+            # normalised, absolute path
+            inner_path = os.path.abspath(inner_path)
+
+            # Load the referenced file
+            ref = load_file("file://" + inner_path)
+
+            # Check that the references in *this* file are valid
+            result = resolve_references(inner_path, ref)
+
+            # They were valid, and so were the sub-references. Delete
+            # the reference here to ensure we don't pass over it again
+            # when checking other files
             del schema['$ref']
         else:
             result = {}
@@ -143,15 +166,22 @@ def load_file(path):
 
 
 if __name__ == '__main__':
-    paths = sys.argv[1:]
-    if not paths:
-        paths = []
-        for (root, dirs, files) in os.walk(os.curdir):
-            for filename in files:
-                if filename.endswith(".yaml"):
-                    paths.append(os.path.join(root, filename))
-    for path in paths:
-        try:
-            check_swagger_file(path)
-        except Exception as e:
-            raise ValueError("Error checking file %r" % (path,), e)
+    # Get the directory that this script is residing in
+    script_directory = os.path.dirname(os.path.realpath(__file__))
+
+    # Resolve the directory containing the swagger sources,
+    # relative to the script path
+    source_files_directory = os.path.realpath(os.path.join(script_directory, "../data"))
+
+    # Walk the source path directory, looking for YAML files to check
+    for (root, dirs, files) in os.walk(source_files_directory):
+        for filename in files:
+            if not filename.endswith(".yaml"):
+                continue
+
+            path = os.path.join(root, filename)
+
+            try:
+                check_swagger_file(path)
+            except Exception as e:
+                raise ValueError("Error checking file %s" % (path,), e)
diff --git a/scripts/package.json b/scripts/package.json
new file mode 100644
index 00000000000..84b9dd7b550
--- /dev/null
+++ b/scripts/package.json
@@ -0,0 +1,15 @@
+{
+  "name": "swagger-cli-validator",
+  "version": "0.0.1",
+  "description": "",
+  "main": "validator.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "nopt": "^3.0.2",
+    "swagger-parser": "^3.2.1"
+  }
+}
diff --git a/scripts/test-and-build.sh b/scripts/test-and-build.sh
index f45e2da612e..76f49e2e8d1 100755
--- a/scripts/test-and-build.sh
+++ b/scripts/test-and-build.sh
@@ -11,12 +11,16 @@ virtualenv -p python3 env
 python --version
 pip --version
 
+# Install python dependencies
 pip install -r scripts/requirements.txt
 
+# Install node dependencies
+npm install --prefix=scripts
+
 # do sanity checks on the examples and swagger
-(cd event-schemas/ && ./check_examples.py)
-(cd api && ./check_examples.py)
-(cd api && npm install && node validator.js -s "client-server")
+scripts/check-event-schema-examples.py
+scripts/check-swagger-sources.py
+node scripts/validator.js --schema "data/api/client-server"
 
 : ${GOPATH:=${WORKSPACE}/.gopath}
 mkdir -p "${GOPATH}"
diff --git a/scripts/validator.js b/scripts/validator.js
new file mode 100644
index 00000000000..c20dd993073
--- /dev/null
+++ b/scripts/validator.js
@@ -0,0 +1,86 @@
+"use strict";
+var fs = require("fs");
+var nopt = require("nopt");
+var parser = require("swagger-parser");
+var path = require("path");
+
+var opts = nopt({
+    "help": Boolean,
+    "schema": path
+}, {
+    "h": "--help",
+    "s": "--schema"
+});
+
+if (opts.help) {
+    console.log(
+        "Use swagger-parser to validate against Swagger 2.0\n"+
+        "Usage:\n"+
+        "  node validator.js -s <schema_file_or_folder>"
+    );
+    process.exit(0);
+}
+if (!opts.schema) {
+    console.error("No [s]chema specified.");
+    process.exit(1);
+}
+
+
+var errFn = function(err, api) {
+    if (!err) {
+        return;
+    }
+    console.error(err);
+    process.exit(1);
+};
+
+/**
+ * @brief Produce a handler for parser.validate().
+ * Recommended usage: `parser.validate(filename, makeHandler(filename));`
+ * or `parser.validate(schema, makeHandler());`.
+ * @param scope - usually a filename, this will be used to denote
+ *                an (in)valid schema in console output; "Schema" if undefined
+ * @returns {function} the handler that can be passed to parser.validate
+ */
+function makeHandler(scope) {
+    if (!scope)
+        scope = "Schema";
+    return function(err, api, metadata) {
+        if (err) {
+            console.error("%s is not valid.", scope || "Schema");
+            errFn(err, api, metadata); // Won't return
+        }
+
+        Object.keys(api.paths).forEach(function (endpoint) {
+            var operationsMap = api.paths[endpoint];
+            Object.keys(operationsMap).forEach(function (verb) {
+                if (!operationsMap[verb]["operationId"]) {
+                    console.error("%s is not valid", scope);
+                    errFn("operationId is missing in " + endpoint + ", verb " + verb, api);
+                }
+            })
+        });
+
+        console.log("%s is valid.", scope);
+    }
+}
+
+var isDir = fs.lstatSync(opts.schema).isDirectory();
+if (isDir) {
+    console.log("Checking directory %s for .yaml files...", opts.schema);
+    fs.readdir(opts.schema, function(err, files) {
+        if (err) {
+            errFn(err); // Won't return
+        }
+        files.forEach(function(f) {
+            var suffix = ".yaml";
+            if (f.indexOf(suffix, f.length - suffix.length) > 0) {
+                parser.validate(path.join(opts.schema, f), makeHandler(f));
+            }
+        });
+    });
+}
+else{
+    parser.validate(opts.schema, makeHandler(opts.schema));
+}
+