From d790d3cd6d0cd0d8203c8da2ee40b52f2e8ee774 Mon Sep 17 00:00:00 2001
From: Colton Leekley-Winslow <coltonlw@flywheel.io>
Date: Wed, 24 Aug 2016 16:07:15 -0500
Subject: [PATCH] Add shell scripts to install, run and test the API

---
 .gitignore                                    |   1 +
 .travis.yml                                   |  16 +-
 CONTRIBUTING.md                               |   2 +-
 Dockerfile                                    |   2 -
 README.md                                     |  23 +-
 TESTING.md                                    |  15 +-
 api/config.py                                 |   2 +-
 bin/api.wsgi                                  |   5 +
 bin/bootstrap.py                              | 145 +++--------
 bin/install-dev-osx.sh                        |  98 ++++++++
 bin/install-python-requirements.sh            |   9 +
 bin/install-ubuntu.sh                         |  25 ++
 bin/install.sh                                |  10 -
 bin/run-dev-osx.sh                            | 172 +++++++++++++
 bin/run.sh                                    | 230 ------------------
 bin/runtests.sh                               |  99 --------
 ...strap.json.sample => bootstrap.sample.json |   0
 raml/schemas/mongo/user.json                  |   2 +-
 requirements.txt                              |   1 +
 test/{ => bin}/lint.sh                        |   2 +-
 test/bin/run-integration-tests.sh             |  45 ++++
 test/bin/run-tests-osx.sh                     |  33 +++
 test/bin/run-tests-ubuntu.sh                  |  29 +++
 test/bin/run-unit-tests.sh                    |  14 ++
 test/bin/setup-integration-tests-ubuntu.sh    |  24 ++
 test/bootstrap_test_db.sh                     |  16 --
 .../integration_tests/abao/abao_test_hooks.js |   6 +-
 .../bootstrap-data.json}                      |   5 +-
 .../integration_tests.postman_collection      |  69 ------
 ... => integration_tests.postman_environment} |  10 +-
 .../integration_tests.postman_collection      |  62 ++---
 test/integration_tests/python/conftest.py     |  67 +++--
 test/integration_tests/python/test_roles.py   |  16 +-
 test/integration_tests/requirements.txt       |   8 +
 test/requirements-integration-test.txt        |  22 --
 test/unit_tests/{ => python}/test_files.py    |   0
 test/unit_tests/{ => python}/test_rules.py    |   0
 .../{ => python}/test_validators.py           |  19 +-
 38 files changed, 660 insertions(+), 644 deletions(-)
 mode change 100644 => 100755 bin/api.wsgi
 create mode 100755 bin/install-dev-osx.sh
 create mode 100755 bin/install-python-requirements.sh
 create mode 100755 bin/install-ubuntu.sh
 delete mode 100755 bin/install.sh
 create mode 100755 bin/run-dev-osx.sh
 delete mode 100755 bin/run.sh
 delete mode 100755 bin/runtests.sh
 rename bootstrap.json.sample => bootstrap.sample.json (100%)
 rename test/{ => bin}/lint.sh (84%)
 create mode 100755 test/bin/run-integration-tests.sh
 create mode 100755 test/bin/run-tests-osx.sh
 create mode 100755 test/bin/run-tests-ubuntu.sh
 create mode 100755 test/bin/run-unit-tests.sh
 create mode 100755 test/bin/setup-integration-tests-ubuntu.sh
 delete mode 100755 test/bootstrap_test_db.sh
 rename test/{test_bootstrap.json => integration_tests/bootstrap-data.json} (82%)
 delete mode 100644 test/integration_tests/postman/environments/integration_tests.postman_collection
 rename test/integration_tests/postman/environments/{travis-ci.postman_environment => integration_tests.postman_environment} (61%)
 create mode 100644 test/integration_tests/requirements.txt
 delete mode 100644 test/requirements-integration-test.txt
 rename test/unit_tests/{ => python}/test_files.py (100%)
 rename test/unit_tests/{ => python}/test_rules.py (100%)
 rename test/unit_tests/{ => python}/test_validators.py (63%)

diff --git a/.gitignore b/.gitignore
index 61aa6cfe..4ade22dd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,4 @@ bootstrap.json
 .cache
 .coverage
 coverage.xml
+/virtualenv
diff --git a/.travis.yml b/.travis.yml
index a9fabd0f..843a6ade 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,18 +2,12 @@
 # "This computer doesn't have VT-X/AMD-v enabled."
 sudo: required
 dist: trusty
+services:
+  - mongodb
 install:
-  - sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
-  - sudo sh -c "echo 'deb https://apt.dockerproject.org/repo ubuntu-trusty main' > /etc/apt/sources.list.d/docker.list"
-  - sudo apt-get update -qq
-  - sudo apt-get -o Dpkg::Options::="--force-confnew" install -y -q docker-engine
-  - sudo curl -o /usr/local/bin/docker-compose -L https://github.com/docker/compose/releases/download/1.6.2/docker-compose-`uname -s`-`uname -m`
-  - sudo chmod +x /usr/local/bin/docker-compose
-before_script:
-  - sudo bin/install.sh --ci
+  - bin/install-ubuntu.sh
+  - test/bin/setup-integration-tests-ubuntu.sh
 script:
-  - bin/runtests.sh unit --ci
-  - bin/runtests.sh integration --ci
-  - ./test/lint.sh api
+  - SCITRAN_PERSISTENT_DB_PORT=27017 test/bin/run-tests-ubuntu.sh
 after_success:
   - coveralls
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index ec6f403a..1cd21a11 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -23,7 +23,7 @@ Changes to `requirements.txt` should always be by pull request.
 - Add docstrings to all functions with a one-line description of its purpose.
 
 ### Format
-- Ensure that `./test/lint.sh api` exits without errors.
+Ensure that `./test/bin/lint.sh api` exits without errors.
 
 ### Commit Messages
 1. The subject line should be a phrase describing the commit and limited to 50 characters
diff --git a/Dockerfile b/Dockerfile
index e1377fdc..5ee66414 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -73,8 +73,6 @@ RUN pip install --upgrade pip wheel setuptools \
 #
 COPY . /var/scitran/code/api/
 
-
-
 COPY docker/uwsgi-entrypoint.sh /var/scitran/
 COPY docker/uwsgi-config.ini /var/scitran/config/
 COPY docker/newrelic.ini /var/scitran/config/
diff --git a/README.md b/README.md
index 64e3b1eb..e0d983bb 100644
--- a/README.md
+++ b/README.md
@@ -19,10 +19,27 @@ SciTran Core is a RESTful HTTP API, written in Python and backed by MongoDB. It
 
 
 ### Usage
+**Currently Python 2 Only**  
+
+#### OSX
 ```
-./bin/run.sh [config file]
+$ ./bin/run-dev-osx.sh --help
+Run a development instance of scitran-core
+ Also starts mongo on port 9001 by default
+
+ Usage:
+
+ -C, --config-file <shell-script>: Source a shell script to set environemnt variables
+ -I, --no-install: Do not attempt install the application first
+ -R, --reload <interval>: Enable live reload, specifying interval in seconds
+ -T, --no-testdata: do not bootstrap testdata
+ -U, --no-user: do not bootstrap users and groups
 ```
-or
+
+#### Ubuntu
 ```
-PYTHONPATH=. uwsgi --http :8443 --virtualenv ./runtime --master --wsgi-file bin/api.wsgi
+mkvirtualenv scitran-core
+./bin/install-ubuntu.sh
+uwsgi --http :8080 --master --wsgi-file bin/api.wsgi -H $VIRTUAL_ENV \
+    --env SCITRAN_PERSISTENT_DB_URI="mongodb://localhost:27017/scitran-core"
 ```
diff --git a/TESTING.md b/TESTING.md
index 16a0b09d..913e20fe 100644
--- a/TESTING.md
+++ b/TESTING.md
@@ -1,3 +1,17 @@
+## Run the tests
+### OSX
+```
+./test/bin/run-tests-osx.sh
+```
+
+### Ubuntu
+```
+# Follow installation instructions in README first
+workon scitran-core
+./test/bin/setup-integration-tests-ubuntu.sh
+./test/bin/run-tests-ubuntu.sh
+```
+
 ### Tools
 - [abao](https://github.com/cybertk/abao/)
 - [postman](https://www.getpostman.com/docs/)
@@ -25,4 +39,3 @@ Postman Links
 - http://blog.getpostman.com/2014/03/07/writing-automated-tests-for-apis-using-postman/
 - https://www.getpostman.com/docs/environments
 - https://www.getpostman.com/docs/newman_intro
-
diff --git a/api/config.py b/api/config.py
index 6d7b5824..c94c7340 100644
--- a/api/config.py
+++ b/api/config.py
@@ -5,8 +5,8 @@ import logging
 import pymongo
 import datetime
 import elasticsearch
-from . import util
 
+from . import util
 
 logging.basicConfig(
     format='%(asctime)s %(name)16.16s %(filename)24.24s %(lineno)5d:%(levelname)4.4s %(message)s',
diff --git a/bin/api.wsgi b/bin/api.wsgi
old mode 100644
new mode 100755
index c23c90cf..a8dfa915
--- a/bin/api.wsgi
+++ b/bin/api.wsgi
@@ -1,4 +1,9 @@
 # vim: filetype=python
+import sys
+import os.path
+
+repo_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../'))
+sys.path.append(repo_path)
 
 from api import api
 
diff --git a/bin/bootstrap.py b/bin/bootstrap.py
index c12a9516..363ca076 100755
--- a/bin/bootstrap.py
+++ b/bin/bootstrap.py
@@ -3,126 +3,49 @@
 """This script helps bootstrap users and data"""
 
 import os
+import os.path
 import sys
 import json
 import logging
 import argparse
 import datetime
-import requests
 
-logging.basicConfig(
-    format='%(asctime)s %(levelname)8.8s %(message)s',
-    datefmt='%Y-%m-%d %H:%M:%S',
-    level=logging.DEBUG,
-)
-log = logging.getLogger('scitran.bootstrap')
+import jsonschema
 
-logging.getLogger('requests').setLevel(logging.WARNING) # silence Requests library
+repo_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../'))
+sys.path.append(repo_path)
 
+from api import config, validators
 
-def _upsert_user(request_session, api_url, user_doc):
-    """
-    Insert user, or update if insert fails due to user already existing.
-
-    Returns:
-        requests.Response: API response.
+def bootstrap_users_and_groups(bootstrap_json_file_path):
+    """Loads users and groups directly into the database.
 
     Args:
-        request_session (requests.Session): Session to use for the request.
-        api_url (str): Base url for the API eg. 'https://localhost:8443/api'
-        user_doc (dict): Valid user doc defined in user input schema.
-    """
-    new_user_resp = request_session.post(api_url + '/users', json=user_doc)
-    if new_user_resp.status_code != 409:
-        return new_user_resp
-
-    # Already exists, update instead
-    return request_session.put(api_url + '/users/' + user_doc['_id'], json=user_doc)
-
-
-def _upsert_role(request_session, api_url, role_doc, group_id):
-    """
-    Insert group role, or update if insert fails due to group role already existing.
-
-    Returns:
-        requests.Response: API response.
-
-    Args:
-        request_session (requests.Session): Session to use for the request.
-        api_url -- (str): Base url for the API eg. 'https://localhost:8443/api'
-        role_doc -- (dict) Valid permission doc defined in permission input schema.
-    """
-    base_role_url = "{0}/groups/{1}/roles".format(api_url, group_id)
-    new_role_resp = request_session.post(base_role_url , json=role_doc)
-    if new_role_resp.status_code != 409:
-        return new_role_resp
-
-    # Already exists, update instead
-    full_role_url = "{0}/{1}/{2}".format(base_role_url, role_doc['site'], role_doc['_id'])
-    return request_session.put(full_role_url, json=role_doc)
-
-
-def users(filepath, api_url, http_headers, insecure):
+        bootstrap_json_file_path (str): Path to json file with users and groups
     """
-    Upserts the users/groups/roles defined in filepath parameter.
-
-    Raises:
-        requests.HTTPError: Upsert failed.
-    """
-    now = datetime.datetime.utcnow()
-    with open(filepath) as fd:
-        input_data = json.load(fd)
-    with requests.Session() as rs:
-        log.info('bootstrapping users...')
-        rs.verify = not insecure
-        rs.headers = http_headers
-        for u in input_data.get('users', []):
-            log.info('    {0}'.format(u['_id']))
-            r = _upsert_user(request_session=rs, api_url=api_url, user_doc=u)
-            r.raise_for_status()
-
-        log.info('bootstrapping groups...')
-        r = rs.get(api_url + '/config')
-        r.raise_for_status()
-        site_id = r.json()['site']['id']
-        for g in input_data.get('groups', []):
-            roles = g.pop('roles')
-            log.info('    {0}'.format(g['_id']))
-            r = rs.post(api_url + '/groups' , json=g)
-            r.raise_for_status()
-            for role in roles:
-                role.setdefault('site', site_id)
-                r = _upsert_role(request_session=rs, api_url=api_url, role_doc=role, group_id=g['_id'])
-                r.raise_for_status()
-    log.info('bootstrapping complete')
-
-
-ap = argparse.ArgumentParser()
-ap.description = 'Bootstrap SciTran users and groups'
-ap.add_argument('url', help='API URL')
-ap.add_argument('json', help='JSON file containing users and groups')
-ap.add_argument('--insecure', action='store_true', help='do not verify SSL connections')
-ap.add_argument('--secret', help='shared API secret')
-args = ap.parse_args()
-
-if args.insecure:
-    requests.packages.urllib3.disable_warnings()
-
-http_headers = {
-    'X-SciTran-Method': 'bootstrapper',
-    'X-SciTran-Name': 'Bootstrapper',
-}
-if args.secret:
-    http_headers['X-SciTran-Auth'] = args.secret
-# TODO: extend this to support oauth tokens
-
-try:
-    users(args.json, args.url, http_headers, args.insecure)
-except requests.HTTPError as ex:
-    log.error(ex)
-    log.error("request_body={0}".format(ex.response.request.body))
-    sys.exit(1)
-except Exception as ex:
-    log.error('Unexpected error:')
-    log.error(ex)
-    sys.exit(1)
+    log = logging.getLogger('scitran.bootstrap')
+    with open(bootstrap_json_file_path, "r") as bootstrap_data_file:
+        bootstrap_data = json.load(bootstrap_data_file)
+    user_schema_path = validators.schema_uri("mongo", "user.json")
+    user_schema, user_resolver = validators._resolve_schema(user_schema_path)
+    for user in bootstrap_data.get("users", []):
+        config.log.info("Bootstrapping user: {0}".format(user["email"]))
+        user["created"] = user["modified"] = datetime.datetime.utcnow()
+        if user.get("api_key"):
+            user["api_key"]["created"] = datetime.datetime.utcnow()
+        validators._validate_json(user, user_schema, user_resolver)
+        config.db.users.insert_one(user)
+    group_schema_path = validators.schema_uri("mongo", "group.json")
+    group_schema, group_resolver = validators._resolve_schema(group_schema_path)
+    for group in bootstrap_data.get("groups", []):
+        config.log.info("Bootstrapping group: {0}".format(group["name"]))
+        group["created"] = group["modified"] = datetime.datetime.utcnow()
+        validators._validate_json(group, group_schema, group_resolver)
+        config.db.groups.insert_one(group)
+
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser()
+    ap.description = 'Bootstrap SciTran users and groups'
+    ap.add_argument('json', help='JSON file containing users and groups')
+    args = ap.parse_args()
+    bootstrap_users_and_groups(args.json)
diff --git a/bin/install-dev-osx.sh b/bin/install-dev-osx.sh
new file mode 100755
index 00000000..f699b6e2
--- /dev/null
+++ b/bin/install-dev-osx.sh
@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+set -e
+
+unset CDPATH
+cd "$( dirname "${BASH_SOURCE[0]}" )/.."
+
+VIRTUALENV_PATH=${VIRTUALENV_PATH:-"./virtualenv"}
+
+if [ -f "`which brew`" ]; then
+    echo "Homebrew is installed"
+else
+    echo "Installing Homebrew"
+    ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
+    echo "Installed Homebrew"
+fi
+
+if brew list | grep -q openssl; then
+    echo "OpenSSL is installed"
+else
+    echo "Installing OpenSSL"
+    brew install openssl
+    echo "Installed OpenSSL"
+fi
+
+if brew list | grep -q python; then
+    echo "Python is installed"
+else
+    echo "Installing Python"
+    brew install python
+    echo "Installed Python"
+fi
+
+if [ -f "`which virtualenv`" ]; then
+    echo "Virtualenv is installed"
+else
+    echo "Installing Virtualenv"
+    pip install virtualenv
+    echo "Installed Virtualenv"
+fi
+
+if [ -d "$VIRTUALENV_PATH" ]; then
+    echo "Virtualenv exists at $VIRTUALENV_PATH"
+else
+    echo "Creating 'scitran' Virtualenv at $VIRTUALENV_PATH"
+    virtualenv -p `brew --prefix`/bin/python --prompt="(scitran) " $VIRTUALENV_PATH
+    echo "Created 'scitran' Virtualenv at $VIRTUALENV_PATH"
+fi
+
+echo "Activating Virtualenv"
+set -a
+. $VIRTUALENV_PATH/bin/activate
+
+pip install -U pip
+env LDFLAGS="-L$(brew --prefix openssl)/lib" \
+  CFLAGS="-I$(brew --prefix openssl)/include" \
+  pip install cryptography
+
+echo "Installing Python requirements"
+./bin/install-python-requirements.sh
+
+echo "Installing node and dev dependencies"
+if [ ! -f "$VIRTUALENV_PATH/bin/node" ]; then
+  # Node doesn't exist in the virtualenv, install
+  echo "Installing nodejs"
+  node_source_dir=`mktemp -d`
+  curl https://nodejs.org/dist/v6.4.0/node-v6.4.0-darwin-x64.tar.gz | tar xvz -C "$node_source_dir"
+  mv $node_source_dir/node-v6.4.0-darwin-x64/bin/* "$VIRTUALENV_PATH/bin"
+  mv $node_source_dir/node-v6.4.0-darwin-x64/lib/* "$VIRTUALENV_PATH/lib"
+  rm -rf "$node_source_dir"
+  npm config set prefix "$VIRTUALENV_PATH"
+fi
+
+pip install -U -r "test/integration_tests/requirements.txt"
+if [ ! -f "`which abao`" ]; then
+  npm install -g git+https://github.com/flywheel-io/abao.git#better-jsonschema-ref
+fi
+if [ ! -f "`which newman`" ]; then
+  npm install -g newman@3.0.1
+fi
+
+install_mongo() {
+    curl $MONGODB_URL | tar xz -C $VIRTUAL_ENV/bin --strip-components 2
+    echo "MongoDB version $MONGODB_VERSION installed"
+}
+
+MONGODB_VERSION=$(cat mongodb_version.txt)
+MONGODB_URL="https://fastdl.mongodb.org/osx/mongodb-osx-x86_64-$MONGODB_VERSION.tgz"
+if [ -x "$VIRTUAL_ENV/bin/mongod" ]; then
+    INSTALLED_MONGODB_VERSION=$($VIRTUAL_ENV/bin/mongod --version | grep "db version" | cut -d "v" -f 3)
+    echo "MongoDB version $INSTALLED_MONGODB_VERSION is installed"
+    if [ "$INSTALLED_MONGODB_VERSION" != "$MONGODB_VERSION" ]; then
+        echo "Upgrading MongoDB to version $MONGODB_VERSION"
+        install_mongo
+    fi
+else
+    echo "Installing MongoDB"
+    install_mongo
+fi
diff --git a/bin/install-python-requirements.sh b/bin/install-python-requirements.sh
new file mode 100755
index 00000000..6093d9e6
--- /dev/null
+++ b/bin/install-python-requirements.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -e
+
+unset CDPATH
+cd "$( dirname "${BASH_SOURCE[0]}" )/.."
+
+pip install -U pip wheel setuptools
+
+pip install -U -r requirements.txt
diff --git a/bin/install-ubuntu.sh b/bin/install-ubuntu.sh
new file mode 100755
index 00000000..3bc835a8
--- /dev/null
+++ b/bin/install-ubuntu.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -e
+
+unset CDPATH
+cd "$( dirname "${BASH_SOURCE[0]}" )/.."
+
+SCITRAN_USER="scitran-core"
+
+sudo apt-get update
+sudo apt-get install -y \
+    build-essential \
+    ca-certificates \
+    curl \
+    libatlas3-base \
+    numactl \
+    python-dev \
+    libffi-dev \
+    libssl-dev \
+    libpcre3 \
+    libpcre3-dev \
+    git
+
+sudo useradd -d /var/scitran -m -r "$SCITRAN_USER"
+
+./bin/install-python-requirements.sh
diff --git a/bin/install.sh b/bin/install.sh
deleted file mode 100755
index 20c4e5c2..00000000
--- a/bin/install.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-unset CDPATH
-cd "$( dirname "${BASH_SOURCE[0]}" )/.."
-
-pip install -U pip
-pip install -r requirements.txt
-pip install -r requirements_dev.txt
diff --git a/bin/run-dev-osx.sh b/bin/run-dev-osx.sh
new file mode 100755
index 00000000..c13f5ccc
--- /dev/null
+++ b/bin/run-dev-osx.sh
@@ -0,0 +1,172 @@
+#!/usr/bin/env bash
+set -e
+
+unset CDPATH
+cd "$( dirname "${BASH_SOURCE[0]}" )/.."
+
+USAGE="
+    Run a development instance of scitran-core\n
+    Also starts mongo, on port 9001 by default\n
+\n
+    Usage:\n
+    \n
+    -C, --config-file <shell-script>: Source a shell script to set environemnt variables\n
+    -I, --no-install: Do not attempt install the application first\n
+    -R, --reload <interval>: Enable live reload, specifying interval in seconds\n
+    -T, --no-testdata: do not bootstrap testdata\n
+    -U, --no-user: do not bootstrap users and groups\n
+"
+
+CONFIG_FILE=""
+BOOTSTRAP_USERS=1
+BOOTSTRAP_TESTDATA=1
+AUTO_RELOAD=0
+INSTALL_APP=1
+
+while [[ "$#" -gt 0 ]]; do
+  key="$1"
+
+  case $key in
+      -C|--config-file)
+      CONFIG_FILE="$1"
+      shift
+      ;;
+      --help)
+      echo -e $USAGE >&2
+      exit 1
+      ;;
+      -I|--no-install)
+      INSTALL_APP=0
+      ;;
+      -R|--reload)
+      AUTO_RELOAD=1
+      AUTO_RELOAD_INTERVAL=$1
+      shift
+      ;;
+      -T|--no-testdata)
+      BOOTSTRAP_TESTDATA=0
+      ;;
+      -U|--no-users)
+      BOOTSTRAP_USERS=0
+      ;;
+      *)
+      echo "Invalid option: $key" >&2
+      echo -e $USAGE >&2
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+set -a
+
+VIRTUALENV_PATH=${VIRTUALENV_PATH:-"$( pwd )/virtualenv"}
+MONGODB_DATA_DIR="$VIRTUALENV_PATH/mongo_data"
+MONGODB_LOG_FILE="$VIRTUALENV_PATH/mongodb.log"
+MONGOD_EXECUTABLE="$VIRTUALENV_PATH/bin/mongod"
+
+SCITRAN_RUNTIME_HOST=${SCITRAN_RUNTIME_HOST:-"127.0.0.1"}
+SCITRAN_RUNTIME_PORT=${SCITRAN_RUNTIME_PORT:-"8080"}
+SCITRAN_RUNTIME_UWSGI_INI=${SCITRAN_RUNTIME_UWSGI_INI:-""}
+SCITRAN_RUNTIME_BOOTSTRAP=${SCITRAN_RUNTIME_BOOTSTRAP:-"./bootstrap.json"}
+
+SCITRAN_CORE_DRONE_SECRET=${SCITRAN_CORE_DRONE_SECRET:-$(  openssl rand -base64 32 )}
+
+SCITRAN_PERSISTENT_PATH="$VIRTUALENV_PATH/scitran-persistent"
+SCITRAN_PERSISTENT_DATA_PATH="$SCITRAN_PERSISTENT_PATH/data"
+SCITRAN_PERSISTENT_DB_PATH=${SCITRAN_PERSISTENT_DB_PATH:-"$SCITRAN_PERSISTENT_PATH/db"}
+SCITRAN_PERSISTENT_DB_PORT=${SCITRAN_PERSISTENT_DB_PORT:-"9001"}
+SCITRAN_PERSISTENT_DB_URI=${SCITRAN_PERSISTENT_DB_URI:-"mongodb://localhost:$SCITRAN_PERSISTENT_DB_PORT/scitran"}
+
+SCITRAN_SITE_API_URL="http://$SCITRAN_RUNTIME_HOST:$SCITRAN_RUNTIME_PORT/api"
+
+if [ $INSTALL_APP -eq 1 ]; then
+  ./bin/install-dev-osx.sh
+fi
+
+clean_up () {
+  kill $MONGOD_PID || true
+  kill $UWSGI_PID || true
+  deactivate || true
+}
+trap clean_up EXIT
+
+. "$VIRTUALENV_PATH/bin/activate"
+
+ulimit -n 1024
+mkdir -p "$SCITRAN_PERSISTENT_DB_PATH"
+"$MONGOD_EXECUTABLE" --port $SCITRAN_PERSISTENT_DB_PORT --logpath "$MONGODB_LOG_FILE" --dbpath "$SCITRAN_PERSISTENT_DB_PATH" --smallfiles &
+MONGOD_PID=$!
+
+sleep 2
+
+# Always drop integration-tests db on startup
+echo -e "use integration-tests \n db.dropDatabase()" | mongo "$SCITRAN_PERSISTENT_DB_URI"
+
+if [ "$SCITRAN_RUNTIME_UWSGI_INI" == "" ]; then
+  "$VIRTUALENV_PATH/bin/uwsgi" \
+    --http "$SCITRAN_RUNTIME_HOST:$SCITRAN_RUNTIME_PORT" \
+    --master --http-keepalive \
+    --so-keepalive --add-header "Connection: Keep-Alive" \
+    --processes 1 --threads 1 \
+    --enable-threads \
+    --wsgi-file "bin/api.wsgi" \
+    -H "$VIRTUALENV_PATH" \
+    --die-on-term \
+    --py-autoreload $AUTO_RELOAD \
+    --env "SCITRAN_CORE_DRONE_SECRET=$SCITRAN_CORE_DRONE_SECRET" \
+    --env "SCITRAN_PERSISTENT_DB_URI=$SCITRAN_PERSISTENT_DB_URI" \
+    --env "SCITRAN_PERSISTENT_PATH=$SCITRAN_PERSISTENT_PATH" \
+    --env "SCITRAN_PERSISTENT_DATA_PATH=$SCITRAN_PERSISTENT_DATA_PATH" &
+    UWSGI_PID=$!
+else
+  "$VIRTUALENV_PATH/bin/uwsgi" --ini "$SCITRAN_RUNTIME_UWSGI_INI" &
+  UWSGI_PID=$!
+fi
+
+until $(curl --output /dev/null --silent --head --fail "$SCITRAN_SITE_API_URL"); do
+    printf '.'
+    sleep 1
+done
+
+# Bootstrap users
+if [ $BOOTSTRAP_USERS -eq 1 ]; then
+    if [ -f "$SCITRAN_PERSISTENT_DB_PATH/.bootstrapped" ]; then
+        echo "Users previously bootstrapped. Remove $SCITRAN_PERSISTENT_DB_PATH to re-bootstrap."
+    else
+        echo "Bootstrapping users"
+        SCITRAN_PERSISTENT_DB_URI="$SCITRAN_PERSISTENT_DB_URI" \
+          bin/bootstrap.py "$SCITRAN_RUNTIME_BOOTSTRAP"
+        echo "Bootstrapped users"
+        touch "$SCITRAN_PERSISTENT_DB_PATH/.bootstrapped"
+    fi
+else
+    echo "NOT bootstrapping users"
+fi
+
+# Boostrap test data
+TESTDATA_REPO="https://github.com/scitran/testdata.git"
+if [ $BOOTSTRAP_TESTDATA -eq 1 ]; then
+    if [ -f "$SCITRAN_PERSISTENT_DATA_PATH/.bootstrapped" ]; then
+        echo "Data previously bootstrapped. Remove $SCITRAN_PERSISTENT_DATA_PATH to re-bootstrap."
+    else
+        if [ ! -d "$SCITRAN_PERSISTENT_PATH/testdata" ]; then
+            echo "Cloning testdata to $SCITRAN_PERSISTENT_PATH/testdata"
+            git clone --single-branch $TESTDATA_REPO $SCITRAN_PERSISTENT_PATH/testdata
+        else
+            echo "Updating testdata in $SCITRAN_PERSISTENT_PATH/testdata"
+            git -C $SCITRAN_PERSISTENT_PATH/testdata pull
+        fi
+        echo "Ensuring reaper is up to date with master branch"
+        pip install -U git+https://github.com/scitran/reaper.git
+        echo "Bootstrapping testdata"
+        UPLOAD_URI="$SCITRAN_SITE_API_URL?secret=$SCITRAN_CORE_DRONE_SECRET"
+        folder_sniper --yes --insecure "$SCITRAN_PERSISTENT_PATH/testdata" $UPLOAD_URI
+        echo "Bootstrapped testdata"
+        touch "$SCITRAN_PERSISTENT_DATA_PATH/.bootstrapped"
+    fi
+else
+    echo "NOT bootstrapping testdata"
+fi
+
+wait
diff --git a/bin/run.sh b/bin/run.sh
deleted file mode 100755
index 6247b642..00000000
--- a/bin/run.sh
+++ /dev/null
@@ -1,230 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-unset CDPATH
-cd "$( dirname "${BASH_SOURCE[0]}" )/.."
-
-echo() { builtin echo -e "\e[1;7mSCITRAN\e[0;7m $@\e[27m"; }
-
-
-USAGE="
-    Usage:\n
-    $0 [-T] [-U] [config file]\n
-    \n
-    -T: do not bootstrap testdata\n
-    -U: do not users and groups
-"
-
-BOOTSTRAP_USERS=1
-BOOTSTRAP_TESTDATA=1
-
-while getopts ":TU" opt; do
-    case $opt in
-        T)
-            BOOTSTRAP_TESTDATA=0;
-            shift $((OPTIND-1));;
-        U)
-            BOOTSTRAP_USERS=0;
-            shift $((OPTIND-1));;
-        \?)
-            echo "Invalid option: -$OPTARG" >&2
-            echo $USAGE >&2
-            exit 1
-            ;;
-    esac
-done
-
-set -o allexport
-
-
-if [ "$#" -eq 1 ]; then
-    EXISTING_ENV=$(env | grep "SCITRAN_" | cat)
-    source "$1"
-    eval "$EXISTING_ENV"
-fi
-if [ "$#" -gt 1 ]; then
-    echo "Too many positional arguments"
-    echo $USAGE >&2
-    exit 1
-fi
-
-
-# Minimal default config values
-SCITRAN_RUNTIME_HOST=${SCITRAN_RUNTIME_HOST:-"127.0.0.1"}
-SCITRAN_RUNTIME_PORT=${SCITRAN_RUNTIME_PORT:-"8080"}
-SCITRAN_RUNTIME_PATH=${SCITRAN_RUNTIME_PATH:-"./runtime"}
-SCITRAN_RUNTIME_BOOTSTRAP=${SCITRAN_RUNTIME_BOOTSTRAP:-"bootstrap.json"}
-SCITRAN_PERSISTENT_PATH=${SCITRAN_PERSISTENT_PATH:-"./persistent"}
-SCITRAN_PERSISTENT_DATA_PATH=${SCITRAN_PERSISTENT_DATA_PATH:-"$SCITRAN_PERSISTENT_PATH/data"}
-SCITRAN_PERSISTENT_DB_PATH=${SCITRAN_PERSISTENT_DB_PATH:-"$SCITRAN_PERSISTENT_PATH/db"}
-SCITRAN_PERSISTENT_DB_PORT=${SCITRAN_PERSISTENT_DB_PORT:-"9001"}
-SCITRAN_PERSISTENT_DB_URI=${SCITRAN_PERSISTENT_DB_URI:-"mongodb://localhost:$SCITRAN_PERSISTENT_DB_PORT/scitran"}
-SCITRAN_CORE_DRONE_SECRET=${SCITRAN_CORE_DRONE_SECRET:-"change-me"}
-
-[ -z "$SCITRAN_RUNTIME_SSL_PEM" ] && SCITRAN_SITE_API_URL="http" || SCITRAN_SITE_API_URL="https"
-SCITRAN_SITE_API_URL="$SCITRAN_SITE_API_URL://$SCITRAN_RUNTIME_HOST:$SCITRAN_RUNTIME_PORT/api"
-
-set +o allexport
-
-
-if [ ! -f "$SCITRAN_RUNTIME_BOOTSTRAP" ]; then
-    echo "Aborting. Please create $SCITRAN_RUNTIME_BOOTSTRAP from bootstrap.json.sample."
-    exit 1
-fi
-
-
-if [ -f "`which brew`" ]; then
-    echo "Homebrew is installed"
-else
-    echo "Installing Homebrew"
-    ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
-    echo "Installed Homebrew"
-fi
-
-if brew list | grep -q openssl; then
-    echo "OpenSSL is installed"
-else
-    echo "Installing OpenSSL"
-    brew install openssl
-    echo "Installed OpenSSL"
-fi
-
-if brew list | grep -q python; then
-    echo "Python is installed"
-else
-    echo "Installing Python"
-    brew install python
-    echo "Installed Python"
-fi
-
-if [ -f "`which virtualenv`" ]; then
-    echo "Virtualenv is installed"
-else
-    echo "Installing Virtualenv"
-    pip install virtualenv
-    echo "Installed Virtualenv"
-fi
-
-if [ -d "$SCITRAN_RUNTIME_PATH" ]; then
-    echo "Virtualenv exists at $SCITRAN_RUNTIME_PATH"
-else
-    echo "Creating 'scitran' Virtualenv at $SCITRAN_RUNTIME_PATH"
-    virtualenv -p `brew --prefix`/bin/python --prompt="(scitran) " $SCITRAN_RUNTIME_PATH
-    echo "Created 'scitran' Virtualenv at $SCITRAN_RUNTIME_PATH"
-fi
-
-
-echo "Activating Virtualenv"
-source $SCITRAN_RUNTIME_PATH/bin/activate
-
-echo "Installing Python requirements"
-bin/install.sh
-
-
-# Install and launch MongoDB
-install_mongo() {
-    curl $MONGODB_URL | tar xz -C $VIRTUAL_ENV/bin --strip-components 2
-    echo "MongoDB version $MONGODB_VERSION installed"
-}
-
-if [ ! -f "$SCITRAN_PERSISTENT_DB_PATH/mongod.lock" ]; then
-    echo "Creating database location at $SCITRAN_PERSISTENT_DB_PATH"
-    mkdir -p $SCITRAN_PERSISTENT_DB_PATH
-fi
-
-MONGODB_VERSION=$(cat mongodb_version.txt)
-MONGODB_URL="https://fastdl.mongodb.org/osx/mongodb-osx-x86_64-$MONGODB_VERSION.tgz"
-if [ -x "$VIRTUAL_ENV/bin/mongod" ]; then
-    INSTALLED_MONGODB_VERSION=$($VIRTUAL_ENV/bin/mongod --version | grep "db version" | cut -d "v" -f 3)
-    echo "MongoDB version $INSTALLED_MONGODB_VERSION is installed"
-    if [ "$INSTALLED_MONGODB_VERSION" != "$MONGODB_VERSION" ]; then
-        echo "Upgrading MongoDB to version $MONGODB_VERSION"
-        install_mongo
-    fi
-else
-    echo "Installing MongoDB"
-    install_mongo
-fi
-
-ulimit -n 1024
-mongod --dbpath $SCITRAN_PERSISTENT_DB_PATH --smallfiles --port $SCITRAN_PERSISTENT_DB_PORT &
-MONGOD_PID=$!
-
-
-# Set python path so scripts can work
-export PYTHONPATH=.
-
-
-# Serve API with PasteScript
-TEMP_INI_FILE=$(mktemp -t scitran_api)
-cat << EOF > $TEMP_INI_FILE
-[server:main]
-use = egg:Paste#http
-host = $SCITRAN_RUNTIME_HOST
-port = $SCITRAN_RUNTIME_PORT
-ssl_pem=$SCITRAN_RUNTIME_SSL_PEM
-
-[app:main]
-paste.app_factory = api.api:app_factory
-EOF
-
-echo "Launching Paster application server"
-paster serve --reload $TEMP_INI_FILE &
-PASTER_PID=$!
-
-
-# Set up exit and error trap to shutdown mongod and paster
-trap "{
-    echo 'Exit signal trapped';
-    kill $MONGOD_PID $PASTER_PID; wait;
-    rm -f $TEMP_INI_FILE
-    deactivate
-}" EXIT ERR
-
-
-# Wait for everything to come up
-sleep 2
-
-
-# Boostrap users and groups
-if [ $BOOTSTRAP_USERS -eq 1 ]; then
-    if [ -f "$SCITRAN_PERSISTENT_DB_PATH/.bootstrapped" ]; then
-        echo "Users previously bootstrapped. Remove $SCITRAN_PERSISTENT_DB_PATH to re-bootstrap."
-    else
-        echo "Bootstrapping users"
-        bin/bootstrap.py --insecure --secret "$SCITRAN_CORE_DRONE_SECRET" $SCITRAN_SITE_API_URL "$SCITRAN_RUNTIME_BOOTSTRAP"
-        echo "Bootstrapped users"
-        touch "$SCITRAN_PERSISTENT_DB_PATH/.bootstrapped"
-    fi
-else
-    echo "NOT bootstrapping users"
-fi
-
-
-# Boostrap test data
-TESTDATA_REPO="https://github.com/scitran/testdata.git"
-if [ $BOOTSTRAP_TESTDATA -eq 1 ]; then
-    if [ -f "$SCITRAN_PERSISTENT_DATA_PATH/.bootstrapped" ]; then
-        echo "Data previously bootstrapped. Remove $SCITRAN_PERSISTENT_DATA_PATH to re-bootstrap."
-    else
-        if [ ! -d "$SCITRAN_PERSISTENT_PATH/testdata" ]; then
-            echo "Cloning testdata to $SCITRAN_PERSISTENT_PATH/testdata"
-            git clone --single-branch $TESTDATA_REPO $SCITRAN_PERSISTENT_PATH/testdata
-        else
-            echo "Updating testdata in $SCITRAN_PERSISTENT_PATH/testdata"
-            git -C $SCITRAN_PERSISTENT_PATH/testdata pull
-        fi
-        echo "Bootstrapping testdata"
-        UPLOAD_URI=$SCITRAN_SITE_API_URL/upload/label?secret=$SCITRAN_CORE_DRONE_SECRET
-        folder_uploader --yes --insecure "$SCITRAN_PERSISTENT_PATH/testdata" $UPLOAD_URI
-        echo "Bootstrapped testdata"
-        touch "$SCITRAN_PERSISTENT_DATA_PATH/.bootstrapped"
-    fi
-else
-    echo "NOT bootstrapping testdata"
-fi
-
-
-# Wait for good or bad things to happen until exit or error trap catches
-wait
diff --git a/bin/runtests.sh b/bin/runtests.sh
deleted file mode 100755
index 1dd5a7a9..00000000
--- a/bin/runtests.sh
+++ /dev/null
@@ -1,99 +0,0 @@
-#!/bin/bash
-
-# Convenience script for unit and integration test execution consumed by
-# continous integration workflow (travis)
-#
-# Must return non-zero on any failure.
-set -e
-
-unit_test_path=test/unit_tests/
-integration_test_path=test/integration_tests/python
-code_path=api/
-
-cd "$( dirname "${BASH_SOURCE[0]}" )/.."
-
-(
-case "$1-$2" in
-  unit-)
-    PYTHONPATH=. py.test $unit_test_path
-    ;;
-  unit---ci)
-    PYTHONPATH=. py.test --cov=api --cov-report=term-missing $unit_test_path
-    ;;
-  unit---watch)
-    PYTHONPATH=. ptw $unit_test_path $code_path --poll -- $unit_test_path
-    ;;
-  integration---ci|integration-)
-    # Bootstrap and run integration test.
-    #  - always stop and remove docker containers
-    #  - always exit non-zero if either bootstrap or integration tests fail
-    #  - only execute tests after core is confirmed up
-    #  - only run integration tests on bootstrap success
-
-    # launch core
-    docker-compose \
-      -f test/docker-compose.yml \
-      up \
-      -d \
-      scitran-core &&
-    # wait for core to be ready.
-    (
-      for((i=1;i<=30;i++))
-      do
-        # ignore return code
-        apiResponse=$(docker-compose -f test/docker-compose.yml run --rm core-check) && true
-
-        # reformat response string for comparison
-        apiResponse="${apiResponse//[$'\r\n ']}"
-        if [ "${apiResponse}" == "200" ]  ; then
-          >&2 echo "INFO: Core API is available."
-          exit 0
-        fi
-        >&2 echo "INFO (${apiResponse}): Waiting for Core API to become available after ${i} attempts to connect."
-        sleep 1
-      done
-      exit 1
-    ) &&
-    # execute tests
-    
-    docker-compose \
-      -f test/docker-compose.yml \
-      run \
-      --rm \
-      bootstrap  &&
-    docker-compose \
-      -f test/docker-compose.yml \
-      run \
-      --rm \
-      integration-test &&
-    docker-compose \
-      -f test/docker-compose.yml \
-      run \
-      --rm \
-      --entrypoint "/bin/bash -c 'cd /usr/src/raml/schemas/definitions && abao /usr/src/raml/api.raml --server=http://scitran-core:8080/api --hookfiles=/usr/src/tests/abao/abao_test_hooks.js'" \
-      integration-test &&
-    docker-compose \
-      -f test/docker-compose.yml \
-      run \
-      --rm \
-      --entrypoint "newman run /usr/src/tests/postman/integration_tests.postman_collection -e /usr/src/tests/postman/environments/travis-ci.postman_environment" \
-      integration-test &&
-    echo "Checking number of files with DOS encoding:" &&
-    ! find * -type f -exec file {} \; | \
-      grep -I "with CRLF line terminators" &&
-    echo "Checking for files with windows style newline:" &&
-    ! grep -rI $'\r' * ||
-    # set failure exit code in the event any previous commands in chain failed.
-    exit_code=1
-
-    docker-compose -f test/docker-compose.yml down -v
-    exit $exit_code
-    ;;
-  integration---watch)
-    echo "Not implemented"
-    ;;
-  *)
-    echo "Usage: $0 unit|integration [--ci|--watch]"
-    ;;
-esac
-)
diff --git a/bootstrap.json.sample b/bootstrap.sample.json
similarity index 100%
rename from bootstrap.json.sample
rename to bootstrap.sample.json
diff --git a/raml/schemas/mongo/user.json b/raml/schemas/mongo/user.json
index bd5f959e..280feab4 100644
--- a/raml/schemas/mongo/user.json
+++ b/raml/schemas/mongo/user.json
@@ -32,7 +32,7 @@
                           "title": "Preferences",
                           "type": "object"
                         },
-    "api_keys":         {
+    "api_key":         {
                           "type": "object",
                           "properties": {
                               "key":            {"type": "string"},
diff --git a/requirements.txt b/requirements.txt
index 93f6e3e4..d46526b9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,6 +10,7 @@ requests==2.9.1
 requests-toolbelt==0.6.0
 rfc3987==1.3.4
 strict-rfc3339==0.7
+uwsgi==2.0.13.1
 webapp2==2.5.2
 WebOb==1.5.1
 elasticsearch==1.9.0
diff --git a/test/lint.sh b/test/bin/lint.sh
similarity index 84%
rename from test/lint.sh
rename to test/bin/lint.sh
index 4df82dfa..9a4f37bc 100755
--- a/test/lint.sh
+++ b/test/bin/lint.sh
@@ -3,7 +3,7 @@
 set -eu
 
 unset CDPATH
-cd "$( dirname "${BASH_SOURCE[0]}" )/.."
+cd "$( dirname "${BASH_SOURCE[0]}" )/../.."
 
 echo "Running pylint ..."
 # TODO: Enable Refactor and Convention reports
diff --git a/test/bin/run-integration-tests.sh b/test/bin/run-integration-tests.sh
new file mode 100755
index 00000000..2975b74f
--- /dev/null
+++ b/test/bin/run-integration-tests.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -e
+
+unset CDPATH
+cd "$( dirname "${BASH_SOURCE[0]}" )/../.."
+
+USAGE="
+    Usage:\n
+    $0 <api-base-url> <mongodb-uri>\n
+    \n
+"
+
+if [ "$#" -eq 2 ]; then
+  SCITRAN_SITE_API_URL=$1
+  MONGODB_URI=$2
+else
+    echo "Wrong number of positional arguments"
+    echo $USAGE >&2
+    exit 1
+fi
+
+echo "Connecting to API"
+until $(curl --output /dev/null --silent --head --fail "$SCITRAN_SITE_API_URL"); do
+    printf '.'
+    sleep 1
+done
+
+echo "Bootstrapping test data..."
+# Don't call things bootstrap.json because that's in root .gitignore
+
+SCITRAN_PERSISTENT_DB_URI="$MONGODB_URI" \
+  python "bin/bootstrap.py" \
+  "test/integration_tests/bootstrap-data.json"
+
+BASE_URL="$SCITRAN_SITE_API_URL" \
+    MONGO_PATH="$MONGODB_URI" \
+    py.test test/integration_tests/python
+
+# Have to change into definitions directory to resolve
+# relative $ref's in the jsonschema's
+pushd raml/schemas/definitions
+abao ../../api.raml "--server=$SCITRAN_SITE_API_URL" "--hookfiles=../../../test/integration_tests/abao/abao_test_hooks.js"
+popd
+
+newman run test/integration_tests/postman/integration_tests.postman_collection -e test/integration_tests/postman/environments/integration_tests.postman_environment
diff --git a/test/bin/run-tests-osx.sh b/test/bin/run-tests-osx.sh
new file mode 100755
index 00000000..576bcc9c
--- /dev/null
+++ b/test/bin/run-tests-osx.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -e
+
+unset CDPATH
+cd "$( dirname "${BASH_SOURCE[0]}" )/../.."
+
+set -a
+VIRTUALENV_PATH=${VIRTUALENV_PATH:-"$( pwd )/virtualenv"}
+# Use port 9003 to hopefully avoid conflicts
+SCITRAN_PERSISTENT_DB_PORT=9003
+SCITRAN_PERSISTENT_DB_URI="mongodb://localhost:$SCITRAN_PERSISTENT_DB_PORT/integration-tests"
+
+./bin/install-dev-osx.sh
+
+. "$VIRTUALENV_PATH/bin/activate"
+
+./test/bin/lint.sh api
+
+./test/bin/run-unit-tests.sh
+
+clean_up () {
+  kill $API_PID || true
+}
+trap clean_up EXIT
+
+SCITRAN_RUNTIME_PORT=8081 \
+    SCITRAN_CORE_DRONE_SECRET=integration-tests \
+    ./bin/run-dev-osx.sh -T -U -I &
+API_PID=$!
+
+./test/bin/run-integration-tests.sh \
+    "http://localhost:8081/api" \
+    "$SCITRAN_PERSISTENT_DB_URI"
diff --git a/test/bin/run-tests-ubuntu.sh b/test/bin/run-tests-ubuntu.sh
new file mode 100755
index 00000000..e5887622
--- /dev/null
+++ b/test/bin/run-tests-ubuntu.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -e
+
+unset CDPATH
+cd "$( dirname "${BASH_SOURCE[0]}" )/../.."
+
+./test/bin/lint.sh api
+
+./test/bin/run-unit-tests.sh
+
+API_BASE_URL="http://localhost:8081/api"
+SCITRAN_PERSISTENT_DB_PORT=${SCITRAN_PERSISTENT_DB_PORT:-"9001"}
+SCITRAN_PERSISTENT_DB_URI=${SCITRAN_PERSISTENT_DB_URI:-"mongodb://localhost:$SCITRAN_PERSISTENT_DB_PORT/scitran"}
+SCITRAN_PERSISTENT_PATH=`mktemp -d`
+SCITRAN_PERSISTENT_DATA_PATH="$SCITRAN_PERSISTENT_PATH/data"
+
+uwsgi --http "localhost:8081" --master --http-keepalive \
+  --so-keepalive --add-header "Connection: Keep-Alive" \
+  --processes 1 --threads 1 \
+  --enable-threads \
+  --wsgi-file bin/api.wsgi \
+  --die-on-term \
+  --env "SCITRAN_PERSISTENT_DB_URI=$SCITRAN_PERSISTENT_DB_URI" \
+  --env "SCITRAN_PERSISTENT_PATH=$SCITRAN_PERSISTENT_PATH" \
+  --env "SCITRAN_PERSISTENT_DATA_PATH=$SCITRAN_PERSISTENT_DATA_PATH" &
+
+./test/bin/run-integration-tests.sh \
+    "$API_BASE_URL" \
+    "$SCITRAN_PERSISTENT_DB_URI"
diff --git a/test/bin/run-unit-tests.sh b/test/bin/run-unit-tests.sh
new file mode 100755
index 00000000..6b61bfb3
--- /dev/null
+++ b/test/bin/run-unit-tests.sh
@@ -0,0 +1,14 @@
+set -e
+
+unset CDPATH
+cd "$( dirname "${BASH_SOURCE[0]}" )/../.."
+
+echo "Checking for files with DOS encoding:"
+! find * -path "virtualenv" -prune -o -path "persisten" -prune -o \
+  -type f -exec file {} \; | grep -I "with CRLF line terminators"
+
+echo "Checking for files with windows style newline:"
+! find * -path "virtualenv" -prune -o -path "persisten" -prune -o -type f \
+  -exec grep -rI $'\r' {} \+
+
+PYTHONPATH="$( pwd )" py.test test/unit_tests/python
diff --git a/test/bin/setup-integration-tests-ubuntu.sh b/test/bin/setup-integration-tests-ubuntu.sh
new file mode 100755
index 00000000..eb9ba76e
--- /dev/null
+++ b/test/bin/setup-integration-tests-ubuntu.sh
@@ -0,0 +1,24 @@
+set -e
+
+unset CDPATH
+cd "$( dirname "${BASH_SOURCE[0]}" )/../.."
+
+pip install -U -r "test/integration_tests/requirements.txt"
+
+
+node_source_dir=`mktemp -d`
+curl https://nodejs.org/dist/v6.4.0/node-v6.4.0-linux-x64.tar.gz | tar xvz -C "$node_source_dir"
+
+if [ -z "$VIRTUAL_ENV" ]; then
+    sudo mv $node_source_dir/node-v6.4.0-linux-x64/bin/* /usr/local/bin
+    sudo mv $node_source_dir/node-v6.4.0-linux-x64/lib/* /usr/local/lib
+    sudo npm install -g git+https://github.com/flywheel-io/abao.git#better-jsonschema-ref
+    sudo npm install -g newman@3.0.1
+else
+    mv $node_source_dir/node-v6.4.0-linux-x64/bin/* "$VIRTUAL_ENV/bin"
+    mv $node_source_dir/node-v6.4.0-linux-x64/lib/* "$VIRTUAL_ENV/lib"
+    rm -rf "$node_source_dir"
+    npm config set prefix "$VIRTUAL_ENV"
+    npm install -g git+https://github.com/flywheel-io/abao.git#better-jsonschema-ref
+    npm install -g newman@3.0.1
+fi
diff --git a/test/bootstrap_test_db.sh b/test/bootstrap_test_db.sh
deleted file mode 100755
index 07395f11..00000000
--- a/test/bootstrap_test_db.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-if [ -z "$1" ]
-  then
-    echo "Usage ./bootstrap_test_db.sh <site_id>"
-    exit 1
-fi
-
-(
-	# Set cwd
-	unset CDPATH
-	cd "$( dirname "${BASH_SOURCE[0]}" )"
-
-	../../../live.sh cmd PYTHONPATH=code/api:code/data code/api/bin/bootstrap.py users -f mongodb://localhost:9001/scitran code/api/test/test_bootstrap.json $1
-)
diff --git a/test/integration_tests/abao/abao_test_hooks.js b/test/integration_tests/abao/abao_test_hooks.js
index 2426cbd2..0295e2ad 100644
--- a/test/integration_tests/abao/abao_test_hooks.js
+++ b/test/integration_tests/abao/abao_test_hooks.js
@@ -54,10 +54,8 @@ hooks.skip("POST /upload/uid-match -> 404");
 hooks.skip("POST /engine -> 200");
 
 hooks.beforeEach(function (test, done) {
-    test.request.query = {
-      user: 'admin@user.com',
-      root: 'true'
-    };
+    test.request.query.root = "true"
+    test.request.headers.Authorization = "scitran-user XZpXI40Uk85eozjQkU1zHJ6yZHpix+j0mo1TMeGZ4dPzIqVPVGPmyfeK";
     done();
 });
 
diff --git a/test/test_bootstrap.json b/test/integration_tests/bootstrap-data.json
similarity index 82%
rename from test/test_bootstrap.json
rename to test/integration_tests/bootstrap-data.json
index c997fb0a..0d9983cf 100644
--- a/test/test_bootstrap.json
+++ b/test/integration_tests/bootstrap-data.json
@@ -17,7 +17,10 @@
                     "email": "admin@user.com",
                     "firstname": "Admin",
                     "lastname": "User",
-                    "root": true
+                    "root": true,
+                    "api_key":{
+                        "key":"XZpXI40Uk85eozjQkU1zHJ6yZHpix+j0mo1TMeGZ4dPzIqVPVGPmyfeK"
+                    }
             },
             {
                     "_id": "test@user.com",
diff --git a/test/integration_tests/postman/environments/integration_tests.postman_collection b/test/integration_tests/postman/environments/integration_tests.postman_collection
deleted file mode 100644
index ac330ac7..00000000
--- a/test/integration_tests/postman/environments/integration_tests.postman_collection
+++ /dev/null
@@ -1,69 +0,0 @@
-{
-	"id": "a8f4f3da-c945-3c88-f6a2-6d77e69506ca",
-	"name": "test",
-	"description": "",
-	"order": [
-		"8ed30abc-627c-333f-d929-d0abf0db5aa7",
-		"3200a331-89b8-70f4-82af-5a96f32876e9"
-	],
-	"folders": [],
-	"timestamp": 1471364887347,
-	"owner": 0,
-	"public": false,
-	"published": false,
-	"requests": [
-		{
-			"id": "3200a331-89b8-70f4-82af-5a96f32876e9",
-			"headers": "",
-			"url": "{{baseUri}}/engine?user={{test_user}}&level=analysis&root=true&id=57ac736ca16b3e715b930200",
-			"preRequestScript": null,
-			"pathVariables": {},
-			"method": "POST",
-			"data": [
-				{
-					"key": "file1",
-					"value": "engine-analyses-1.txt",
-					"type": "file",
-					"enabled": true
-				},
-				{
-					"key": "metadata",
-					"value": "{\"label\":\"test\"}",
-					"type": "text",
-					"enabled": true
-				}
-			],
-			"dataMode": "params",
-			"version": 2,
-			"tests": null,
-			"currentHelper": "normal",
-			"helperAttributes": {},
-			"time": 1471607394844,
-			"name": "Test /engine upload - type \"analysis\"",
-			"description": "",
-			"collectionId": "a8f4f3da-c945-3c88-f6a2-6d77e69506ca",
-			"responses": []
-		},
-		{
-			"id": "8ed30abc-627c-333f-d929-d0abf0db5aa7",
-			"headers": "Content-Type: application/json\n",
-			"url": "{{baseUri}}/users?user={{test_user}}",
-			"pathVariables": {},
-			"preRequestScript": "",
-			"method": "GET",
-			"collectionId": "a8f4f3da-c945-3c88-f6a2-6d77e69506ca",
-			"data": [],
-			"dataMode": "raw",
-			"name": "/users",
-			"description": "List users\n\n",
-			"descriptionFormat": "html",
-			"time": 1471364908152,
-			"version": 2,
-			"responses": [],
-			"tests": "tests[\"Status code is 200\"] = responseCode.code === 200;",
-			"currentHelper": "normal",
-			"helperAttributes": {},
-			"rawModeData": "{\"_id\":\"jane.doe@gmail.com\",\"firstname\":\"Jane\",\"lastname\":\"Doe\",\"email\":\"jane.doe@gmail.com\"}"
-		}
-	]
-}
\ No newline at end of file
diff --git a/test/integration_tests/postman/environments/travis-ci.postman_environment b/test/integration_tests/postman/environments/integration_tests.postman_environment
similarity index 61%
rename from test/integration_tests/postman/environments/travis-ci.postman_environment
rename to test/integration_tests/postman/environments/integration_tests.postman_environment
index b43d0504..623b6a64 100644
--- a/test/integration_tests/postman/environments/travis-ci.postman_environment
+++ b/test/integration_tests/postman/environments/integration_tests.postman_environment
@@ -4,20 +4,20 @@
 	"values": [
 		{
 			"key": "baseUri",
-			"value": "http://scitran-core:8080/api",
+			"value": "http://localhost:8081/api",
 			"type": "text",
 			"enabled": true
 		},
 		{
-			"key": "test_user",
-			"value": "admin@user.com",
+			"key": "test_user_api_key",
+			"value": "XZpXI40Uk85eozjQkU1zHJ6yZHpix+j0mo1TMeGZ4dPzIqVPVGPmyfeK",
 			"type": "text",
 			"enabled": true
 		}
 	],
-	"timestamp": 1471459823996,
+	"timestamp": 1472144623917,
 	"synced": false,
 	"syncedFilename": "",
 	"team": null,
 	"isDeleted": false
-}
\ No newline at end of file
+}
diff --git a/test/integration_tests/postman/integration_tests.postman_collection b/test/integration_tests/postman/integration_tests.postman_collection
index 2544d4c7..7437d0e1 100644
--- a/test/integration_tests/postman/integration_tests.postman_collection
+++ b/test/integration_tests/postman/integration_tests.postman_collection
@@ -1,30 +1,51 @@
 {
-	"id": "a8f4f3da-c945-3c88-f6a2-6d77e69506ca",
+	"id": "40551e1a-7213-8417-7834-25c7f986d14d",
 	"name": "test",
 	"description": "",
 	"order": [
-		"8ed30abc-627c-333f-d929-d0abf0db5aa7",
-		"3200a331-89b8-70f4-82af-5a96f32876e9"
+		"0706ef61-46b2-95d4-02b5-64d9ec6ac7ff",
+		"e7b8fc49-f494-427f-e29b-86d8e601b025"
 	],
 	"folders": [],
 	"timestamp": 1471364887347,
 	"owner": 0,
 	"public": false,
 	"published": false,
+	"hasRequests": true,
 	"requests": [
 		{
-			"id": "3200a331-89b8-70f4-82af-5a96f32876e9",
-			"headers": "",
-			"url": "{{baseUri}}/engine?user={{test_user}}&level=analysis&root=true&id=57ac736ca16b3e715b930200",
+			"id": "0706ef61-46b2-95d4-02b5-64d9ec6ac7ff",
+			"headers": "Content-Type: application/json\nAuthorization: scitran-user {{test_user_api_key}}\n",
+			"url": "{{baseUri}}/users",
+			"preRequestScript": "",
+			"pathVariables": {},
+			"method": "GET",
+			"data": [],
+			"dataMode": "raw",
+			"version": 2,
+			"tests": "tests[\"Status code is 200\"] = responseCode.code === 200;",
+			"currentHelper": "normal",
+			"helperAttributes": {},
+			"time": 1472144768788,
+			"name": "/users",
+			"description": "List users\n\n",
+			"collectionId": "40551e1a-7213-8417-7834-25c7f986d14d",
+			"responses": [],
+			"rawModeData": "{\"_id\":\"jane.doe@gmail.com\",\"firstname\":\"Jane\",\"lastname\":\"Doe\",\"email\":\"jane.doe@gmail.com\"}"
+		},
+		{
+			"id": "e7b8fc49-f494-427f-e29b-86d8e601b025",
+			"headers": "Authorization: scitran-user {{test_user_api_key}}\n",
+			"url": "{{baseUri}}/engine?&level=analysis&root=true&id=57ac736ca16b3e715b930200",
 			"preRequestScript": null,
 			"pathVariables": {},
 			"method": "POST",
 			"data": [
 				{
 					"key": "file1",
-					"value": "test_files/engine-analyses-1.txt",
 					"type": "file",
-					"enabled": true
+					"enabled": true,
+                    "value":"test_files/engine-analyses-1.txt"
 				},
 				{
 					"key": "metadata",
@@ -38,32 +59,11 @@
 			"tests": "tests[\"Status code is 200\"] = responseCode.code === 200;",
 			"currentHelper": "normal",
 			"helperAttributes": {},
-			"time": 1471607560876,
+			"time": 1472144790445,
 			"name": "Test /engine upload - type \"analysis\"",
 			"description": "",
-			"collectionId": "a8f4f3da-c945-3c88-f6a2-6d77e69506ca",
+			"collectionId": "40551e1a-7213-8417-7834-25c7f986d14d",
 			"responses": []
-		},
-		{
-			"id": "8ed30abc-627c-333f-d929-d0abf0db5aa7",
-			"headers": "Content-Type: application/json\n",
-			"url": "{{baseUri}}/users?user={{test_user}}",
-			"pathVariables": {},
-			"preRequestScript": "",
-			"method": "GET",
-			"collectionId": "a8f4f3da-c945-3c88-f6a2-6d77e69506ca",
-			"data": [],
-			"dataMode": "raw",
-			"name": "/users",
-			"description": "List users\n\n",
-			"descriptionFormat": "html",
-			"time": 1471364908152,
-			"version": 2,
-			"responses": [],
-			"tests": "tests[\"Status code is 200\"] = responseCode.code === 200;",
-			"currentHelper": "normal",
-			"helperAttributes": {},
-			"rawModeData": "{\"_id\":\"jane.doe@gmail.com\",\"firstname\":\"Jane\",\"lastname\":\"Doe\",\"email\":\"jane.doe@gmail.com\"}"
 		}
 	]
 }
diff --git a/test/integration_tests/python/conftest.py b/test/integration_tests/python/conftest.py
index 18ed318e..ae901543 100644
--- a/test/integration_tests/python/conftest.py
+++ b/test/integration_tests/python/conftest.py
@@ -1,6 +1,7 @@
+import json
 import os
 import time
-import json
+
 import pytest
 import pymongo
 import requests
@@ -28,9 +29,14 @@ def base_url():
 
 
 class RequestsAccessor(object):
-    def __init__(self, base_url, default_params):
+    def __init__(self, base_url, default_params=None, default_headers=None):
         self.base_url = base_url
+        if default_params is None:
+            default_params = {}
         self.default_params = default_params
+        if default_headers is None:
+            default_headers = {}
+        self.default_headers = default_headers
 
     def _get_params(self, **kwargs):
         params = self.default_params.copy()
@@ -38,40 +44,73 @@ class RequestsAccessor(object):
             params.update(kwargs["params"])
         return params
 
-    def post(self, url_path, *args, **kwargs):
+    def get_headers(self, user_headers):
+        request_headers = self.default_headers.copy()
+        request_headers.update(user_headers)
+        return request_headers
+
+    def get(self, url_path, **kwargs):
+        url = self.get_url_from_path(url_path)
         kwargs['params'] = self._get_params(**kwargs)
-        return requests.post(self.base_url + url_path, verify=False, *args, **kwargs)
+        headers = self.get_headers(kwargs.get("headers", {}))
+        return requests.get(url, verify=False,
+                            headers=headers, **kwargs)
 
-    def delete(self, url_path, *args, **kwargs):
+    def post(self, url_path, **kwargs):
+        url = self.get_url_from_path(url_path)
         kwargs['params'] = self._get_params(**kwargs)
-        return requests.delete(self.base_url + url_path, verify=False, *args, **kwargs)
+        headers = self.get_headers(kwargs.get("headers", {}))
+        return requests.post(url, verify=False,
+                             headers=headers, **kwargs)
 
-    def get(self, url_path, *args, **kwargs):
+    def put(self, url_path, **kwargs):
+        url = self.get_url_from_path(url_path)
         kwargs['params'] = self._get_params(**kwargs)
-        return requests.get(self.base_url + url_path, verify=False, *args, **kwargs)
+        headers = self.get_headers(kwargs.get("headers", {}))
+        return requests.put(url, verify=False,
+                            headers=headers, **kwargs)
 
-    def put(self, url_path, *args, **kwargs):
+    def delete(self, url_path, **kwargs):
         kwargs['params'] = self._get_params(**kwargs)
-        return requests.put(self.base_url + url_path, verify=False, *args, **kwargs)
+        headers = self.get_headers(kwargs.get("headers", {}))
+        url = self.get_url_from_path(url_path)
+        return requests.delete(url, verify=False,
+                               headers=headers, **kwargs)
 
+    def get_url_from_path(self, path):
+        return "{0}{1}".format(self.base_url, path)
 
 @pytest.fixture(scope="module")
 def api_as_admin(base_url):
-    accessor = RequestsAccessor(base_url, {"user": "admin@user.com", "root": "true"})
+    accessor = RequestsAccessor(base_url,
+        {"user": "admin@user.com", "root": "true"},
+        default_headers={
+            "Authorization":"scitran-user XZpXI40Uk85eozjQkU1zHJ6yZHpix+j0mo1TMeGZ4dPzIqVPVGPmyfeK"
+            }
+        )
     return accessor
 
 
 @pytest.fixture(scope="module")
 def api_as_user(base_url):
-    accessor = RequestsAccessor(base_url, {"user": "admin@user.com"})
+    accessor = RequestsAccessor(base_url,
+        {"user": "admin@user.com"},
+        default_headers={
+            "Authorization":"scitran-user XZpXI40Uk85eozjQkU1zHJ6yZHpix+j0mo1TMeGZ4dPzIqVPVGPmyfeK"
+            }
+        )
     return accessor
 
 
 @pytest.fixture(scope="module")
 def api_accessor(base_url):
     class RequestsAccessorWithBaseUrl(RequestsAccessor):
-        def __init__(self, user):
-            super(self.__class__, self).__init__(base_url, {"user": user})
+        def __init__(self, user_api_key):
+            super(self.__class__, self).__init__(
+                base_url,
+                default_headers={
+                    "Authorization":"scitran-user {0}".format(user_api_key)
+                })
 
     return RequestsAccessorWithBaseUrl
 
diff --git a/test/integration_tests/python/test_roles.py b/test/integration_tests/python/test_roles.py
index e6dc78dc..be524df7 100644
--- a/test/integration_tests/python/test_roles.py
+++ b/test/integration_tests/python/test_roles.py
@@ -1,5 +1,7 @@
+import datetime
 import json
 import time
+
 import pytest
 
 
@@ -37,9 +39,19 @@ def create_role_payload(user, site, access):
     })
 
 
-def test_roles(api_as_admin, with_a_group_and_a_user, api_accessor):
+def test_roles(api_as_admin, with_a_group_and_a_user, api_accessor, db):
     data = with_a_group_and_a_user
-    api_as_other_user = api_accessor(data.user_id)
+    user_api_key = "4hOn5aBx/nUiI0blDbTUPpKQsEbEn74rH9z5KctlXw6GrMKdicPGXKQg"
+    api_key_doc = {
+        "key":user_api_key,
+        "created":datetime.datetime.utcnow()
+    }
+    update_result = db.users.update_one(
+        {"_id":data.user_id},
+        {"$set":{"api_key":api_key_doc}}
+        )
+    assert update_result.modified_count == 1
+    api_as_other_user = api_accessor(user_api_key)
 
     roles_path = '/groups/' + data.group_id + '/roles'
     local_user_roles_path = roles_path + '/local/' + data.user_id
diff --git a/test/integration_tests/requirements.txt b/test/integration_tests/requirements.txt
new file mode 100644
index 00000000..d77a7e08
--- /dev/null
+++ b/test/integration_tests/requirements.txt
@@ -0,0 +1,8 @@
+# Development packages
+coverage==4.0.3
+coveralls==1.1
+PasteScript==2.0.2
+pylint==1.5.3
+pytest==2.8.5
+pytest-cov==2.2.0
+pytest-watch==3.8.0
diff --git a/test/requirements-integration-test.txt b/test/requirements-integration-test.txt
deleted file mode 100644
index 6120ac2b..00000000
--- a/test/requirements-integration-test.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-# Development packages
-coverage==4.0.3
-coveralls==1.1
-nose==1.3.7
-PasteScript==2.0.2
-pylint==1.5.3
-pytest==2.8.5
-pytest-cov==2.2.0
-pytest-watch==3.8.0
-pymongo==3.2
-
-# Production packages
-enum==0.4.6
-jsonschema==2.5.1
-Markdown==2.6.5
-pyOpenSSL==0.15.1
-python-dateutil==2.4.2
-pytz==2015.7
-requests==2.9.1
-rfc3987==1.3.4
-webapp2==2.5.2
-WebOb==1.5.1
diff --git a/test/unit_tests/test_files.py b/test/unit_tests/python/test_files.py
similarity index 100%
rename from test/unit_tests/test_files.py
rename to test/unit_tests/python/test_files.py
diff --git a/test/unit_tests/test_rules.py b/test/unit_tests/python/test_rules.py
similarity index 100%
rename from test/unit_tests/test_rules.py
rename to test/unit_tests/python/test_rules.py
diff --git a/test/unit_tests/test_validators.py b/test/unit_tests/python/test_validators.py
similarity index 63%
rename from test/unit_tests/test_validators.py
rename to test/unit_tests/python/test_validators.py
index 2c8b5f8e..d5626563 100644
--- a/test/unit_tests/test_validators.py
+++ b/test/unit_tests/python/test_validators.py
@@ -1,6 +1,10 @@
-from api import validators
 import logging
-import nose.tools
+
+import jsonschema.exceptions
+import pytest
+
+from api import validators
+
 log = logging.getLogger(__name__)
 sh = logging.StreamHandler()
 log.addHandler(sh)
@@ -12,7 +16,6 @@ class StubHandler:
 
 default_handler = StubHandler()
 
-@nose.tools.raises(Exception)
 def test_payload():
     payload = {
         'files': [],
@@ -22,9 +25,7 @@ def test_payload():
         'permissions': [],
         'extra_params': 'testtest'
     }
-    payload_validator = validators.payload_from_schema_file(default_handler, 'input/project.json')
-    payload_validator(payload, 'POST')
-
-
-
-
+    schema_uri = validators.schema_uri("input", "project.json")
+    schema, resolver = validators._resolve_schema(schema_uri)
+    with pytest.raises(jsonschema.exceptions.ValidationError):
+        validators._validate_json(payload, schema, resolver)
-- 
GitLab