Add shell scripts to install, run and test the API

.gitignore

@@ -8,3 +8,4 @@ bootstrap.json
 .cache
 .coverage
 coverage.xml
+/virtualenv

.travis.yml

@@ -2,18 +2,12 @@
 # "This computer doesn't have VT-X/AMD-v enabled."
 sudo: required
 dist: trusty
+services:
+  - mongodb
 install:
-  - sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
-  - sudo sh -c "echo 'deb https://apt.dockerproject.org/repo ubuntu-trusty main' > /etc/apt/sources.list.d/docker.list"
-  - sudo apt-get update -qq
-  - sudo apt-get -o Dpkg::Options::="--force-confnew" install -y -q docker-engine
-  - sudo curl -o /usr/local/bin/docker-compose -L https://github.com/docker/compose/releases/download/1.6.2/docker-compose-`uname -s`-`uname -m`
-  - sudo chmod +x /usr/local/bin/docker-compose
-before_script:
-  - sudo bin/install.sh --ci
+  - bin/install-ubuntu.sh
+  - test/bin/setup-integration-tests-ubuntu.sh
 script:
-  - bin/runtests.sh unit --ci
-  - bin/runtests.sh integration --ci
-  - ./test/lint.sh api
+  - SCITRAN_PERSISTENT_DB_PORT=27017 test/bin/run-tests-ubuntu.sh
 after_success:
   - coveralls

CONTRIBUTING.md

@@ -23,7 +23,7 @@ Changes to `requirements.txt` should always be by pull request.
 - Add docstrings to all functions with a one-line description of its purpose.
 
 ### Format
-- Ensure that `./test/lint.sh api` exits without errors.
+Ensure that `./test/bin/lint.sh api` exits without errors.
 
 ### Commit Messages
 1. The subject line should be a phrase describing the commit and limited to 50 characters

Dockerfile

@@ -73,8 +73,6 @@ RUN pip install --upgrade pip wheel setuptools \
 #
 COPY . /var/scitran/code/api/
 
-
-
 COPY docker/uwsgi-entrypoint.sh /var/scitran/
 COPY docker/uwsgi-config.ini /var/scitran/config/
 COPY docker/newrelic.ini /var/scitran/config/

README.md

@@ -19,10 +19,27 @@ SciTran Core is a RESTful HTTP API, written in Python and backed by MongoDB. It
 
 
 ### Usage
+**Currently Python 2 Only**  
+
+#### OSX
 ```
-./bin/run.sh [config file]
+$ ./bin/run-dev-osx.sh --help
+Run a development instance of scitran-core
+ Also starts mongo on port 9001 by default
+
+ Usage:
+
+ -C, --config-file <shell-script>: Source a shell script to set environemnt variables
+ -I, --no-install: Do not attempt install the application first
+ -R, --reload <interval>: Enable live reload, specifying interval in seconds
+ -T, --no-testdata: do not bootstrap testdata
+ -U, --no-user: do not bootstrap users and groups
 ```
-or
+
+#### Ubuntu
 ```
-PYTHONPATH=. uwsgi --http :8443 --virtualenv ./runtime --master --wsgi-file bin/api.wsgi
+mkvirtualenv scitran-core
+./bin/install-ubuntu.sh
+uwsgi --http :8080 --master --wsgi-file bin/api.wsgi -H $VIRTUAL_ENV \
+    --env SCITRAN_PERSISTENT_DB_URI="mongodb://localhost:27017/scitran-core"
 ```

TESTING.md

+## Run the tests
+### OSX
+```
+./test/bin/run-tests-osx.sh
+```
+
+### Ubuntu
+```
+# Follow installation instructions in README first
+workon scitran-core
+./test/bin/setup-integration-tests-ubuntu.sh
+./test/bin/run-tests-ubuntu.sh
+```
+
 ### Tools
 - [abao](https://github.com/cybertk/abao/)
 - [postman](https://www.getpostman.com/docs/)
@@ -25,4 +39,3 @@ Postman Links
 - http://blog.getpostman.com/2014/03/07/writing-automated-tests-for-apis-using-postman/
 - https://www.getpostman.com/docs/environments
 - https://www.getpostman.com/docs/newman_intro
-

api/config.py

@@ -5,8 +5,8 @@ import logging
 import pymongo
 import datetime
 import elasticsearch
-from . import util
 
+from . import util
 
 logging.basicConfig(
     format='%(asctime)s %(name)16.16s %(filename)24.24s %(lineno)5d:%(levelname)4.4s %(message)s',

bin/api.wsgi

 # vim: filetype=python
+import sys
+import os.path
+
+repo_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../'))
+sys.path.append(repo_path)
 
 from api import api
 

bin/bootstrap.py

@@ -3,126 +3,49 @@
 """This script helps bootstrap users and data"""
 
 import os
+import os.path
 import sys
 import json
 import logging
 import argparse
 import datetime
-import requests
 
-logging.basicConfig(
-    format='%(asctime)s %(levelname)8.8s %(message)s',
-    datefmt='%Y-%m-%d %H:%M:%S',
-    level=logging.DEBUG,
-)
-log = logging.getLogger('scitran.bootstrap')
+import jsonschema
 
-logging.getLogger('requests').setLevel(logging.WARNING) # silence Requests library
+repo_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../'))
+sys.path.append(repo_path)
 
+from api import config, validators
 
-def _upsert_user(request_session, api_url, user_doc):
-    """
-    Insert user, or update if insert fails due to user already existing.
-
-    Returns:
-        requests.Response: API response.
+def bootstrap_users_and_groups(bootstrap_json_file_path):
+    """Loads users and groups directly into the database.
 
     Args:
-        request_session (requests.Session): Session to use for the request.
-        api_url (str): Base url for the API eg. 'https://localhost:8443/api'
-        user_doc (dict): Valid user doc defined in user input schema.
-    """
-    new_user_resp = request_session.post(api_url + '/users', json=user_doc)
-    if new_user_resp.status_code != 409:
-        return new_user_resp
-
-    # Already exists, update instead
-    return request_session.put(api_url + '/users/' + user_doc['_id'], json=user_doc)
-
-
-def _upsert_role(request_session, api_url, role_doc, group_id):
-    """
-    Insert group role, or update if insert fails due to group role already existing.
-
-    Returns:
-        requests.Response: API response.
-
-    Args:
-        request_session (requests.Session): Session to use for the request.
-        api_url -- (str): Base url for the API eg. 'https://localhost:8443/api'
-        role_doc -- (dict) Valid permission doc defined in permission input schema.
-    """
-    base_role_url = "{0}/groups/{1}/roles".format(api_url, group_id)
-    new_role_resp = request_session.post(base_role_url , json=role_doc)
-    if new_role_resp.status_code != 409:
-        return new_role_resp
-
-    # Already exists, update instead
-    full_role_url = "{0}/{1}/{2}".format(base_role_url, role_doc['site'], role_doc['_id'])
-    return request_session.put(full_role_url, json=role_doc)
-
-
-def users(filepath, api_url, http_headers, insecure):
+        bootstrap_json_file_path (str): Path to json file with users and groups
     """
-    Upserts the users/groups/roles defined in filepath parameter.
-
-    Raises:
-        requests.HTTPError: Upsert failed.
-    """
-    now = datetime.datetime.utcnow()
-    with open(filepath) as fd:
-        input_data = json.load(fd)
-    with requests.Session() as rs:
-        log.info('bootstrapping users...')
-        rs.verify = not insecure
-        rs.headers = http_headers
-        for u in input_data.get('users', []):
-            log.info('    {0}'.format(u['_id']))
-            r = _upsert_user(request_session=rs, api_url=api_url, user_doc=u)
-            r.raise_for_status()
-
-        log.info('bootstrapping groups...')
-        r = rs.get(api_url + '/config')
-        r.raise_for_status()
-        site_id = r.json()['site']['id']
-        for g in input_data.get('groups', []):
-            roles = g.pop('roles')
-            log.info('    {0}'.format(g['_id']))
-            r = rs.post(api_url + '/groups' , json=g)
-            r.raise_for_status()
-            for role in roles:
-                role.setdefault('site', site_id)
-                r = _upsert_role(request_session=rs, api_url=api_url, role_doc=role, group_id=g['_id'])
-                r.raise_for_status()
-    log.info('bootstrapping complete')
-
-
-ap = argparse.ArgumentParser()
-ap.description = 'Bootstrap SciTran users and groups'
-ap.add_argument('url', help='API URL')
-ap.add_argument('json', help='JSON file containing users and groups')
-ap.add_argument('--insecure', action='store_true', help='do not verify SSL connections')
-ap.add_argument('--secret', help='shared API secret')
-args = ap.parse_args()
-
-if args.insecure:
-    requests.packages.urllib3.disable_warnings()
-
-http_headers = {
-    'X-SciTran-Method': 'bootstrapper',
-    'X-SciTran-Name': 'Bootstrapper',
-}
-if args.secret:
-    http_headers['X-SciTran-Auth'] = args.secret
-# TODO: extend this to support oauth tokens
-
-try:
-    users(args.json, args.url, http_headers, args.insecure)
-except requests.HTTPError as ex:
-    log.error(ex)
-    log.error("request_body={0}".format(ex.response.request.body))
-    sys.exit(1)
-except Exception as ex:
-    log.error('Unexpected error:')
-    log.error(ex)
-    sys.exit(1)
+    log = logging.getLogger('scitran.bootstrap')
+    with open(bootstrap_json_file_path, "r") as bootstrap_data_file:
+        bootstrap_data = json.load(bootstrap_data_file)
+    user_schema_path = validators.schema_uri("mongo", "user.json")
+    user_schema, user_resolver = validators._resolve_schema(user_schema_path)
+    for user in bootstrap_data.get("users", []):
+        config.log.info("Bootstrapping user: {0}".format(user["email"]))
+        user["created"] = user["modified"] = datetime.datetime.utcnow()
+        if user.get("api_key"):
+            user["api_key"]["created"] = datetime.datetime.utcnow()
+        validators._validate_json(user, user_schema, user_resolver)
+        config.db.users.insert_one(user)
+    group_schema_path = validators.schema_uri("mongo", "group.json")
+    group_schema, group_resolver = validators._resolve_schema(group_schema_path)
+    for group in bootstrap_data.get("groups", []):
+        config.log.info("Bootstrapping group: {0}".format(group["name"]))
+        group["created"] = group["modified"] = datetime.datetime.utcnow()
+        validators._validate_json(group, group_schema, group_resolver)
+        config.db.groups.insert_one(group)
+
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser()
+    ap.description = 'Bootstrap SciTran users and groups'
+    ap.add_argument('json', help='JSON file containing users and groups')
+    args = ap.parse_args()
+    bootstrap_users_and_groups(args.json)

bin/install-dev-osx.sh

+#!/usr/bin/env bash
+set -e
+
+unset CDPATH
+cd "$( dirname "${BASH_SOURCE[0]}" )/.."
+
+VIRTUALENV_PATH=${VIRTUALENV_PATH:-"./virtualenv"}
+
+if [ -f "`which brew`" ]; then
+    echo "Homebrew is installed"
+else
+    echo "Installing Homebrew"
+    ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
+    echo "Installed Homebrew"
+fi
+
+if brew list | grep -q openssl; then
+    echo "OpenSSL is installed"
+else
+    echo "Installing OpenSSL"
+    brew install openssl
+    echo "Installed OpenSSL"
+fi
+
+if brew list | grep -q python; then
+    echo "Python is installed"
+else
+    echo "Installing Python"
+    brew install python
+    echo "Installed Python"
+fi
+
+if [ -f "`which virtualenv`" ]; then
+    echo "Virtualenv is installed"
+else
+    echo "Installing Virtualenv"
+    pip install virtualenv
+    echo "Installed Virtualenv"
+fi
+
+if [ -d "$VIRTUALENV_PATH" ]; then
+    echo "Virtualenv exists at $VIRTUALENV_PATH"
+else
+    echo "Creating 'scitran' Virtualenv at $VIRTUALENV_PATH"
+    virtualenv -p `brew --prefix`/bin/python --prompt="(scitran) " $VIRTUALENV_PATH
+    echo "Created 'scitran' Virtualenv at $VIRTUALENV_PATH"
+fi
+
+echo "Activating Virtualenv"
+set -a
+. $VIRTUALENV_PATH/bin/activate
+
+pip install -U pip
+env LDFLAGS="-L$(brew --prefix openssl)/lib" \
+  CFLAGS="-I$(brew --prefix openssl)/include" \
+  pip install cryptography
+
+echo "Installing Python requirements"
+./bin/install-python-requirements.sh
+
+echo "Installing node and dev dependencies"
+if [ ! -f "$VIRTUALENV_PATH/bin/node" ]; then
+  # Node doesn't exist in the virtualenv, install
+  echo "Installing nodejs"
+  node_source_dir=`mktemp -d`
+  curl https://nodejs.org/dist/v6.4.0/node-v6.4.0-darwin-x64.tar.gz | tar xvz -C "$node_source_dir"
+  mv $node_source_dir/node-v6.4.0-darwin-x64/bin/* "$VIRTUALENV_PATH/bin"
+  mv $node_source_dir/node-v6.4.0-darwin-x64/lib/* "$VIRTUALENV_PATH/lib"
+  rm -rf "$node_source_dir"
+  npm config set prefix "$VIRTUALENV_PATH"
+fi
+
+pip install -U -r "test/integration_tests/requirements.txt"
+if [ ! -f "`which abao`" ]; then
+  npm install -g git+https://github.com/flywheel-io/abao.git#better-jsonschema-ref
+fi
+if [ ! -f "`which newman`" ]; then
+  npm install -g newman@3.0.1
+fi
+
+install_mongo() {
+    curl $MONGODB_URL | tar xz -C $VIRTUAL_ENV/bin --strip-components 2
+    echo "MongoDB version $MONGODB_VERSION installed"
+}
+
+MONGODB_VERSION=$(cat mongodb_version.txt)
+MONGODB_URL="https://fastdl.mongodb.org/osx/mongodb-osx-x86_64-$MONGODB_VERSION.tgz"
+if [ -x "$VIRTUAL_ENV/bin/mongod" ]; then
+    INSTALLED_MONGODB_VERSION=$($VIRTUAL_ENV/bin/mongod --version | grep "db version" | cut -d "v" -f 3)
+    echo "MongoDB version $INSTALLED_MONGODB_VERSION is installed"
+    if [ "$INSTALLED_MONGODB_VERSION" != "$MONGODB_VERSION" ]; then
+        echo "Upgrading MongoDB to version $MONGODB_VERSION"
+        install_mongo
+    fi
+else
+    echo "Installing MongoDB"
+    install_mongo
+fi

bin/install.sh → bin/install-python-requirements.sh

 #!/usr/bin/env bash
-
 set -e
 
 unset CDPATH
 cd "$( dirname "${BASH_SOURCE[0]}" )/.."
 
-pip install -U pip
-pip install -r requirements.txt
-pip install -r requirements_dev.txt
+pip install -U pip wheel setuptools
+
+pip install -U -r requirements.txt

bin/install-ubuntu.sh

+#!/usr/bin/env bash
+set -e
+
+unset CDPATH
+cd "$( dirname "${BASH_SOURCE[0]}" )/.."
+
+SCITRAN_USER="scitran-core"
+
+sudo apt-get update
+sudo apt-get install -y \
+    build-essential \
+    ca-certificates \
+    curl \
+    libatlas3-base \
+    numactl \
+    python-dev \
+    libffi-dev \
+    libssl-dev \
+    libpcre3 \
+    libpcre3-dev \
+    git
+
+sudo useradd -d /var/scitran -m -r "$SCITRAN_USER"
+
+./bin/install-python-requirements.sh

bin/run-dev-osx.sh

+#!/usr/bin/env bash
+set -e
+
+unset CDPATH
+cd "$( dirname "${BASH_SOURCE[0]}" )/.."
+
+USAGE="
+    Run a development instance of scitran-core\n
+    Also starts mongo, on port 9001 by default\n
+\n
+    Usage:\n
+    \n
+    -C, --config-file <shell-script>: Source a shell script to set environemnt variables\n
+    -I, --no-install: Do not attempt install the application first\n
+    -R, --reload <interval>: Enable live reload, specifying interval in seconds\n
+    -T, --no-testdata: do not bootstrap testdata\n
+    -U, --no-user: do not bootstrap users and groups\n
+"
+
+CONFIG_FILE=""
+BOOTSTRAP_USERS=1
+BOOTSTRAP_TESTDATA=1
+AUTO_RELOAD=0
+INSTALL_APP=1
+
+while [[ "$#" -gt 0 ]]; do
+  key="$1"
+
+  case $key in
+      -C|--config-file)
+      CONFIG_FILE="$1"
+      shift
+      ;;
+      --help)
+      echo -e $USAGE >&2
+      exit 1
+      ;;
+      -I|--no-install)
+      INSTALL_APP=0
+      ;;
+      -R|--reload)
+      AUTO_RELOAD=1
+      AUTO_RELOAD_INTERVAL=$1
+      shift
+      ;;
+      -T|--no-testdata)
+      BOOTSTRAP_TESTDATA=0
+      ;;
+      -U|--no-users)
+      BOOTSTRAP_USERS=0
+      ;;
+      *)
+      echo "Invalid option: $key" >&2
+      echo -e $USAGE >&2
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+set -a
+
+VIRTUALENV_PATH=${VIRTUALENV_PATH:-"$( pwd )/virtualenv"}
+MONGODB_DATA_DIR="$VIRTUALENV_PATH/mongo_data"
+MONGODB_LOG_FILE="$VIRTUALENV_PATH/mongodb.log"
+MONGOD_EXECUTABLE="$VIRTUALENV_PATH/bin/mongod"
+
+SCITRAN_RUNTIME_HOST=${SCITRAN_RUNTIME_HOST:-"127.0.0.1"}
+SCITRAN_RUNTIME_PORT=${SCITRAN_RUNTIME_PORT:-"8080"}
+SCITRAN_RUNTIME_UWSGI_INI=${SCITRAN_RUNTIME_UWSGI_INI:-""}
+SCITRAN_RUNTIME_BOOTSTRAP=${SCITRAN_RUNTIME_BOOTSTRAP:-"./bootstrap.json"}
+
+SCITRAN_CORE_DRONE_SECRET=${SCITRAN_CORE_DRONE_SECRET:-$(  openssl rand -base64 32 )}
+
+SCITRAN_PERSISTENT_PATH="$VIRTUALENV_PATH/scitran-persistent"
+SCITRAN_PERSISTENT_DATA_PATH="$SCITRAN_PERSISTENT_PATH/data"
+SCITRAN_PERSISTENT_DB_PATH=${SCITRAN_PERSISTENT_DB_PATH:-"$SCITRAN_PERSISTENT_PATH/db"}
+SCITRAN_PERSISTENT_DB_PORT=${SCITRAN_PERSISTENT_DB_PORT:-"9001"}
+SCITRAN_PERSISTENT_DB_URI=${SCITRAN_PERSISTENT_DB_URI:-"mongodb://localhost:$SCITRAN_PERSISTENT_DB_PORT/scitran"}
+
+SCITRAN_SITE_API_URL="http://$SCITRAN_RUNTIME_HOST:$SCITRAN_RUNTIME_PORT/api"
+
+if [ $INSTALL_APP -eq 1 ]; then
+  ./bin/install-dev-osx.sh
+fi
+
+clean_up () {
+  kill $MONGOD_PID || true
+  kill $UWSGI_PID || true
+  deactivate || true
+}
+trap clean_up EXIT
+
+. "$VIRTUALENV_PATH/bin/activate"
+
+ulimit -n 1024
+mkdir -p "$SCITRAN_PERSISTENT_DB_PATH"
+"$MONGOD_EXECUTABLE" --port $SCITRAN_PERSISTENT_DB_PORT --logpath "$MONGODB_LOG_FILE" --dbpath "$SCITRAN_PERSISTENT_DB_PATH" --smallfiles &
+MONGOD_PID=$!
+
+sleep 2
+
+# Always drop integration-tests db on startup
+echo -e "use integration-tests \n db.dropDatabase()" | mongo "$SCITRAN_PERSISTENT_DB_URI"
+
+if [ "$SCITRAN_RUNTIME_UWSGI_INI" == "" ]; then
+  "$VIRTUALENV_PATH/bin/uwsgi" \
+    --http "$SCITRAN_RUNTIME_HOST:$SCITRAN_RUNTIME_PORT" \
+    --master --http-keepalive \
+    --so-keepalive --add-header "Connection: Keep-Alive" \
+    --processes 1 --threads 1 \
+    --enable-threads \
+    --wsgi-file "bin/api.wsgi" \
+    -H "$VIRTUALENV_PATH" \
+    --die-on-term \
+    --py-autoreload $AUTO_RELOAD \
+    --env "SCITRAN_CORE_DRONE_SECRET=$SCITRAN_CORE_DRONE_SECRET" \
+    --env "SCITRAN_PERSISTENT_DB_URI=$SCITRAN_PERSISTENT_DB_URI" \
+    --env "SCITRAN_PERSISTENT_PATH=$SCITRAN_PERSISTENT_PATH" \
+    --env "SCITRAN_PERSISTENT_DATA_PATH=$SCITRAN_PERSISTENT_DATA_PATH" &
+    UWSGI_PID=$!
+else
+  "$VIRTUALENV_PATH/bin/uwsgi" --ini "$SCITRAN_RUNTIME_UWSGI_INI" &
+  UWSGI_PID=$!
+fi
+
+until $(curl --output /dev/null --silent --head --fail "$SCITRAN_SITE_API_URL"); do
+    printf '.'
+    sleep 1
+done
+
+# Bootstrap users
+if [ $BOOTSTRAP_USERS -eq 1 ]; then
+    if [ -f "$SCITRAN_PERSISTENT_DB_PATH/.bootstrapped" ]; then
+        echo "Users previously bootstrapped. Remove $SCITRAN_PERSISTENT_DB_PATH to re-bootstrap."
+    else
+        echo "Bootstrapping users"
+        SCITRAN_PERSISTENT_DB_URI="$SCITRAN_PERSISTENT_DB_URI" \
+          bin/bootstrap.py "$SCITRAN_RUNTIME_BOOTSTRAP"
+        echo "Bootstrapped users"
+        touch "$SCITRAN_PERSISTENT_DB_PATH/.bootstrapped"
+    fi
+else
+    echo "NOT bootstrapping users"
+fi
+
+# Boostrap test data
+TESTDATA_REPO="https://github.com/scitran/testdata.git"
+if [ $BOOTSTRAP_TESTDATA -eq 1 ]; then
+    if [ -f "$SCITRAN_PERSISTENT_DATA_PATH/.bootstrapped" ]; then
+        echo "Data previously bootstrapped. Remove $SCITRAN_PERSISTENT_DATA_PATH to re-bootstrap."
+    else
+        if [ ! -d "$SCITRAN_PERSISTENT_PATH/testdata" ]; then
+            echo "Cloning testdata to $SCITRAN_PERSISTENT_PATH/testdata"
+            git clone --single-branch $TESTDATA_REPO $SCITRAN_PERSISTENT_PATH/testdata
+        else
+            echo "Updating testdata in $SCITRAN_PERSISTENT_PATH/testdata"
+            git -C $SCITRAN_PERSISTENT_PATH/testdata pull
+        fi
+        echo "Ensuring reaper is up to date with master branch"
+        pip install -U git+https://github.com/scitran/reaper.git
+        echo "Bootstrapping testdata"
+        UPLOAD_URI="$SCITRAN_SITE_API_URL?secret=$SCITRAN_CORE_DRONE_SECRET"
+        folder_sniper --yes --insecure "$SCITRAN_PERSISTENT_PATH/testdata" $UPLOAD_URI
+        echo "Bootstrapped testdata"
+        touch "$SCITRAN_PERSISTENT_DATA_PATH/.bootstrapped"
+    fi
+else
+    echo "NOT bootstrapping testdata"
+fi
+
+wait

bin/run.sh

-#!/usr/bin/env bash
-
-set -e
-
-unset CDPATH
-cd "$( dirname "${BASH_SOURCE[0]}" )/.."
-
-echo() { builtin echo -e "\e[1;7mSCITRAN\e[0;7m $@\e[27m"; }
-
-
-USAGE="
-    Usage:\n
-    $0 [-T] [-U] [config file]\n
-    \n
-    -T: do not bootstrap testdata\n
-    -U: do not users and groups
-"
-
-BOOTSTRAP_USERS=1
-BOOTSTRAP_TESTDATA=1
-
-while getopts ":TU" opt; do
-    case $opt in
-        T)
-            BOOTSTRAP_TESTDATA=0;
-            shift $((OPTIND-1));;
-        U)
-            BOOTSTRAP_USERS=0;
-            shift $((OPTIND-1));;
-        \?)
-            echo "Invalid option: -$OPTARG" >&2
-            echo $USAGE >&2
-            exit 1
-            ;;
-    esac
-done
-
-set -o allexport
-
-
-if [ "$#" -eq 1 ]; then
-    EXISTING_ENV=$(env | grep "SCITRAN_" | cat)
-    source "$1"
-    eval "$EXISTING_ENV"
-fi
-if [ "$#" -gt 1 ]; then
-    echo "Too many positional arguments"
-    echo $USAGE >&2
-    exit 1
-fi
-
-
-# Minimal default config values
-SCITRAN_RUNTIME_HOST=${SCITRAN_RUNTIME_HOST:-"127.0.0.1"}
-SCITRAN_RUNTIME_PORT=${SCITRAN_RUNTIME_PORT:-"8080"}
-SCITRAN_RUNTIME_PATH=${SCITRAN_RUNTIME_PATH:-"./runtime"}
-SCITRAN_RUNTIME_BOOTSTRAP=${SCITRAN_RUNTIME_BOOTSTRAP:-"bootstrap.json"}
-SCITRAN_PERSISTENT_PATH=${SCITRAN_PERSISTENT_PATH:-"./persistent"}
-SCITRAN_PERSISTENT_DATA_PATH=${SCITRAN_PERSISTENT_DATA_PATH:-"$SCITRAN_PERSISTENT_PATH/data"}
-SCITRAN_PERSISTENT_DB_PATH=${SCITRAN_PERSISTENT_DB_PATH:-"$SCITRAN_PERSISTENT_PATH/db"}
-SCITRAN_PERSISTENT_DB_PORT=${SCITRAN_PERSISTENT_DB_PORT:-"9001"}
-SCITRAN_PERSISTENT_DB_URI=${SCITRAN_PERSISTENT_DB_URI:-"mongodb://localhost:$SCITRAN_PERSISTENT_DB_PORT/scitran"}
-SCITRAN_CORE_DRONE_SECRET=${SCITRAN_CORE_DRONE_SECRET:-"change-me"}
-
-[ -z "$SCITRAN_RUNTIME_SSL_PEM" ] && SCITRAN_SITE_API_URL="http" || SCITRAN_SITE_API_URL="https"
-SCITRAN_SITE_API_URL="$SCITRAN_SITE_API_URL://$SCITRAN_RUNTIME_HOST:$SCITRAN_RUNTIME_PORT/api"
-
-set +o allexport
-
-
-if [ ! -f "$SCITRAN_RUNTIME_BOOTSTRAP" ]; then
-    echo "Aborting. Please create $SCITRAN_RUNTIME_BOOTSTRAP from bootstrap.json.sample."
-    exit 1
-fi
-
-
-if [ -f "`which brew`" ]; then
-    echo "Homebrew is installed"
-else
-    echo "Installing Homebrew"
-    ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
-    echo "Installed Homebrew"
-fi
-
-if brew list | grep -q openssl; then
-    echo "OpenSSL is installed"
-else
-    echo "Installing OpenSSL"
-    brew install openssl
-    echo "Installed OpenSSL"
-fi
-
-if brew list | grep -q python; then
-    echo "Python is installed"
-else
-    echo "Installing Python"
-    brew install python
-    echo "Installed Python"
-fi
-
-if [ -f "`which virtualenv`" ]; then
-    echo "Virtualenv is installed"
-else
-    echo "Installing Virtualenv"
-    pip install virtualenv
-    echo "Installed Virtualenv"
-fi
-
-if [ -d "$SCITRAN_RUNTIME_PATH" ]; then
-    echo "Virtualenv exists at $SCITRAN_RUNTIME_PATH"
-else
-    echo "Creating 'scitran' Virtualenv at $SCITRAN_RUNTIME_PATH"
-    virtualenv -p `brew --prefix`/bin/python --prompt="(scitran) " $SCITRAN_RUNTIME_PATH
-    echo "Created 'scitran' Virtualenv at $SCITRAN_RUNTIME_PATH"
-fi
-
-
-echo "Activating Virtualenv"
-source $SCITRAN_RUNTIME_PATH/bin/activate
-
-echo "Installing Python requirements"
-bin/install.sh
-
-
-# Install and launch MongoDB
-install_mongo() {
-    curl $MONGODB_URL | tar xz -C $VIRTUAL_ENV/bin --strip-components 2
-    echo "MongoDB version $MONGODB_VERSION installed"
-}
-
-if [ ! -f "$SCITRAN_PERSISTENT_DB_PATH/mongod.lock" ]; then
-    echo "Creating database location at $SCITRAN_PERSISTENT_DB_PATH"
-    mkdir -p $SCITRAN_PERSISTENT_DB_PATH
-fi
-
-MONGODB_VERSION=$(cat mongodb_version.txt)
-MONGODB_URL="https://fastdl.mongodb.org/osx/mongodb-osx-x86_64-$MONGODB_VERSION.tgz"
-if [ -x "$VIRTUAL_ENV/bin/mongod" ]; then
-    INSTALLED_MONGODB_VERSION=$($VIRTUAL_ENV/bin/mongod --version | grep "db version" | cut -d "v" -f 3)
-    echo "MongoDB version $INSTALLED_MONGODB_VERSION is installed"
-    if [ "$INSTALLED_MONGODB_VERSION" != "$MONGODB_VERSION" ]; then
-        echo "Upgrading MongoDB to version $MONGODB_VERSION"
-        install_mongo
-    fi
-else
-    echo "Installing MongoDB"
-    install_mongo
-fi
-
-ulimit -n 1024
-mongod --dbpath $SCITRAN_PERSISTENT_DB_PATH --smallfiles --port $SCITRAN_PERSISTENT_DB_PORT &
-MONGOD_PID=$!
-
-
-# Set python path so scripts can work
-export PYTHONPATH=.
-
-
-# Serve API with PasteScript
-TEMP_INI_FILE=$(mktemp -t scitran_api)
-cat << EOF > $TEMP_INI_FILE
-[server:main]
-use = egg:Paste#http
-host = $SCITRAN_RUNTIME_HOST
-port = $SCITRAN_RUNTIME_PORT
-ssl_pem=$SCITRAN_RUNTIME_SSL_PEM
-
-[app:main]
-paste.app_factory = api.api:app_factory
-EOF
-
-echo "Launching Paster application server"
-paster serve --reload $TEMP_INI_FILE &
-PASTER_PID=$!
-
-
-# Set up exit and error trap to shutdown mongod and paster
-trap "{
-    echo 'Exit signal trapped';
-    kill $MONGOD_PID $PASTER_PID; wait;
-    rm -f $TEMP_INI_FILE
-    deactivate
-}" EXIT ERR
-
-
-# Wait for everything to come up
-sleep 2
-
-
-# Boostrap users and groups
-if [ $BOOTSTRAP_USERS -eq 1 ]; then
-    if [ -f "$SCITRAN_PERSISTENT_DB_PATH/.bootstrapped" ]; then
-        echo "Users previously bootstrapped. Remove $SCITRAN_PERSISTENT_DB_PATH to re-bootstrap."
-    else
-        echo "Bootstrapping users"
-        bin/bootstrap.py --insecure --secret "$SCITRAN_CORE_DRONE_SECRET" $SCITRAN_SITE_API_URL "$SCITRAN_RUNTIME_BOOTSTRAP"
-        echo "Bootstrapped users"
-        touch "$SCITRAN_PERSISTENT_DB_PATH/.bootstrapped"
-    fi
-else
-    echo "NOT bootstrapping users"
-fi
-
-
-# Boostrap test data
-TESTDATA_REPO="https://github.com/scitran/testdata.git"
-if [ $BOOTSTRAP_TESTDATA -eq 1 ]; then
-    if [ -f "$SCITRAN_PERSISTENT_DATA_PATH/.bootstrapped" ]; then
-        echo "Data previously bootstrapped. Remove $SCITRAN_PERSISTENT_DATA_PATH to re-bootstrap."
-    else
-        if [ ! -d "$SCITRAN_PERSISTENT_PATH/testdata" ]; then
-            echo "Cloning testdata to $SCITRAN_PERSISTENT_PATH/testdata"
-            git clone --single-branch $TESTDATA_REPO $SCITRAN_PERSISTENT_PATH/testdata
-        else
-            echo "Updating testdata in $SCITRAN_PERSISTENT_PATH/testdata"
-            git -C $SCITRAN_PERSISTENT_PATH/testdata pull
-        fi
-        echo "Bootstrapping testdata"
-        UPLOAD_URI=$SCITRAN_SITE_API_URL/upload/label?secret=$SCITRAN_CORE_DRONE_SECRET
-        folder_uploader --yes --insecure "$SCITRAN_PERSISTENT_PATH/testdata" $UPLOAD_URI
-        echo "Bootstrapped testdata"
-        touch "$SCITRAN_PERSISTENT_DATA_PATH/.bootstrapped"
-    fi
-else
-    echo "NOT bootstrapping testdata"
-fi
-
-
-# Wait for good or bad things to happen until exit or error trap catches
-wait

bin/runtests.sh

-#!/bin/bash
-
-# Convenience script for unit and integration test execution consumed by
-# continous integration workflow (travis)
-#
-# Must return non-zero on any failure.
-set -e
-
-unit_test_path=test/unit_tests/
-integration_test_path=test/integration_tests/python
-code_path=api/
-
-cd "$( dirname "${BASH_SOURCE[0]}" )/.."
-
-(
-case "$1-$2" in
-  unit-)
-    PYTHONPATH=. py.test $unit_test_path
-    ;;
-  unit---ci)
-    PYTHONPATH=. py.test --cov=api --cov-report=term-missing $unit_test_path
-    ;;
-  unit---watch)
-    PYTHONPATH=. ptw $unit_test_path $code_path --poll -- $unit_test_path
-    ;;
-  integration---ci|integration-)
-    # Bootstrap and run integration test.
-    #  - always stop and remove docker containers
-    #  - always exit non-zero if either bootstrap or integration tests fail
-    #  - only execute tests after core is confirmed up
-    #  - only run integration tests on bootstrap success
-
-    # launch core
-    docker-compose \
-      -f test/docker-compose.yml \
-      up \
-      -d \
-      scitran-core &&
-    # wait for core to be ready.
-    (
-      for((i=1;i<=30;i++))
-      do
-        # ignore return code
-        apiResponse=$(docker-compose -f test/docker-compose.yml run --rm core-check) && true
-
-        # reformat response string for comparison
-        apiResponse="${apiResponse//[$'\r\n ']}"
-        if [ "${apiResponse}" == "200" ]  ; then
-          >&2 echo "INFO: Core API is available."
-          exit 0
-        fi
-        >&2 echo "INFO (${apiResponse}): Waiting for Core API to become available after ${i} attempts to connect."
-        sleep 1
-      done
-      exit 1
-    ) &&
-    # execute tests
-    
-    docker-compose \
-      -f test/docker-compose.yml \
-      run \
-      --rm \
-      bootstrap  &&
-    docker-compose \
-      -f test/docker-compose.yml \
-      run \
-      --rm \
-      integration-test &&
-    docker-compose \
-      -f test/docker-compose.yml \
-      run \
-      --rm \
-      --entrypoint "/bin/bash -c 'cd /usr/src/raml/schemas/definitions && abao /usr/src/raml/api.raml --server=http://scitran-core:8080/api --hookfiles=/usr/src/tests/abao/abao_test_hooks.js'" \
-      integration-test &&
-    docker-compose \
-      -f test/docker-compose.yml \
-      run \
-      --rm \
-      --entrypoint "newman run /usr/src/tests/postman/integration_tests.postman_collection -e /usr/src/tests/postman/environments/travis-ci.postman_environment" \
-      integration-test &&
-    echo "Checking number of files with DOS encoding:" &&
-    ! find * -type f -exec file {} \; | \
-      grep -I "with CRLF line terminators" &&
-    echo "Checking for files with windows style newline:" &&
-    ! grep -rI $'\r' * ||
-    # set failure exit code in the event any previous commands in chain failed.
-    exit_code=1
-
-    docker-compose -f test/docker-compose.yml down -v
-    exit $exit_code
-    ;;
-  integration---watch)
-    echo "Not implemented"
-    ;;
-  *)
-    echo "Usage: $0 unit|integration [--ci|--watch]"
-    ;;
-esac
-)

raml/schemas/mongo/user.json

@@ -32,7 +32,7 @@
                           "title": "Preferences",
                           "type": "object"
                         },
-    "api_keys":         {
+    "api_key":         {
                           "type": "object",
                           "properties": {
                               "key":            {"type": "string"},

requirements.txt

@@ -10,6 +10,7 @@ requests==2.9.1
 requests-toolbelt==0.6.0
 rfc3987==1.3.4
 strict-rfc3339==0.7
+uwsgi==2.0.13.1
 webapp2==2.5.2
 WebOb==1.5.1
 elasticsearch==1.9.0

test/lint.sh → test/bin/lint.sh

@@ -3,7 +3,7 @@
 set -eu
 
 unset CDPATH
-cd "$( dirname "${BASH_SOURCE[0]}" )/.."
+cd "$( dirname "${BASH_SOURCE[0]}" )/../.."
 
 echo "Running pylint ..."
 # TODO: Enable Refactor and Convention reports

test/bin/run-integration-tests.sh

+#!/usr/bin/env bash
+set -e
+
+unset CDPATH
+cd "$( dirname "${BASH_SOURCE[0]}" )/../.."
+
+USAGE="
+    Usage:\n
+    $0 <api-base-url> <mongodb-uri>\n
+    \n
+"
+
+if [ "$#" -eq 2 ]; then
+  SCITRAN_SITE_API_URL=$1
+  MONGODB_URI=$2
+else
+    echo "Wrong number of positional arguments"
+    echo $USAGE >&2
+    exit 1
+fi
+
+echo "Connecting to API"
+until $(curl --output /dev/null --silent --head --fail "$SCITRAN_SITE_API_URL"); do
+    printf '.'
+    sleep 1
+done
+
+echo "Bootstrapping test data..."
+# Don't call things bootstrap.json because that's in root .gitignore
+
+SCITRAN_PERSISTENT_DB_URI="$MONGODB_URI" \
+  python "bin/bootstrap.py" \
+  "test/integration_tests/bootstrap-data.json"
+
+BASE_URL="$SCITRAN_SITE_API_URL" \
+    MONGO_PATH="$MONGODB_URI" \
+    py.test test/integration_tests/python
+
+# Have to change into definitions directory to resolve
+# relative $ref's in the jsonschema's
+pushd raml/schemas/definitions
+abao ../../api.raml "--server=$SCITRAN_SITE_API_URL" "--hookfiles=../../../test/integration_tests/abao/abao_test_hooks.js"
+popd
+
+newman run test/integration_tests/postman/integration_tests.postman_collection -e test/integration_tests/postman/environments/integration_tests.postman_environment