Add config file option and default env vars

api/api.py

@@ -124,10 +124,10 @@ def dispatcher(router, request, response):
 application = webapp2.WSGIApplication(routes)
 application.router.set_dispatcher(dispatcher)
 
-if config.get_item('system', 'newrelic'):
+if config.get_item('core', 'newrelic'):
     try:
         import newrelic.agent, newrelic.api.exceptions
-        newrelic.agent.initialize(config.get_item('system', 'newrelic'))
+        newrelic.agent.initialize(config.get_item('core', 'newrelic'))
         application = newrelic.agent.WSGIApplicationWrapper(application)
         log.info('New Relic detected and loaded. Monitoring enabled.')
     except ImportError:

api/base.py

@@ -18,7 +18,7 @@ class RequestHandler(webapp2.RequestHandler):
 
     def __init__(self, request=None, response=None):
         self.initialize(request, response)
-        self.debug = config.get_item('system', 'insecure')
+        self.debug = config.get_item('core', 'insecure')
         request_start = datetime.datetime.utcnow()
         provider_avatar = None
 

api/config.py

@@ -20,8 +20,8 @@ logging.getLogger('paste.httpserver').setLevel(logging.WARNING) # silence Paste
 
 
 DEFAULT_CONFIG = {
-    'system': {
-        'log_level': 'debug',
+    'core': {
+        'log_level': 'info',
         'insecure': False,
         'persisted': False,
         'newrelic': None,
@@ -44,7 +44,6 @@ DEFAULT_CONFIG = {
     },
     'persistent': {
         'db_uri': 'mongodb://localhost:9001/scitran',
-        'db_path': os.path.join(os.path.dirname(__file__), '../persistent/db'),
         'data_path': os.path.join(os.path.dirname(__file__), '../persistent/data'),
     },
 }
@@ -90,23 +89,23 @@ db.groups.update_one({'_id': 'unknown'}, {'$setOnInsert': { 'created': now, 'mod
 db.sites.replace_one({'_id': __config['site']['_id']}, {'name': __config['site']['name'], 'site_url': __config['site']['url']}, upsert=True)
 
 
-def get_config(projection=None):
+def get_config():
     global __last_update, __config, environment_read
     now = datetime.datetime.utcnow()
-    if not __config['system']['persisted']:
+    if not __config['core']['persisted']:
         __config['modified'] = now
         flat_config= util.mongo_dict(__config)
         r = db.config.update_one({'latest': True}, {'$set': flat_config, '$setOnInsert': {'created': now}}, upsert=True)
-        __config['system']['persisted'] = bool(r.modified_count)
+        __config['core']['persisted'] = bool(r.modified_count)
     elif now - __last_update > datetime.timedelta(seconds=120):
-        __config = db.config.find_one({'latest': True}, projection)
+        __config = db.config.find_one({'latest': True})
         __last_update = now
-        log.setLevel(getattr(logging, __config['system']['log_level'].upper()))
+        log.setLevel(getattr(logging, __config['core']['log_level'].upper()))
     return __config
 
 def get_public_config():
-    projection = None # FIXME: define public config whitelist
-    return get_config(projection)
+    projection = ['created', 'modified', 'site', 'auth']
+    return db.config.find_one({'latest': True}, projection)
 
 def get_item(outer, inner):
     return get_config()[outer][inner]

dev.ini

-[server:main]
-use = egg:Paste#http
-host = 127.0.0.1
-port = 8080
-ssl_pem=/Users/gsfr/Applications/localhost.pem
-
-[app:main]
-paste.app_factory = api.api:app_factory

run.sh

@@ -2,27 +2,54 @@
 
 set -e
 
-RUNTIME_DIR="./runtime"
-PERSITENT_DIR="./persistent"
+unset CDPATH
+cd "$( dirname "${BASH_SOURCE[0]}" )"
+
+
+if [ "$#" -eq 1 ]; then
+    TEMP_ENV_FILE=$(mktemp -t scitran_env)
+    env > $TEMP_ENV_FILE
+    set -o allexport
+    source "$1"
+    source $TEMP_ENV_FILE
+    rm -f $TEMP_ENV_FILE
+    set +o allexport
+fi
+if [ "$#" -gt 1 ]; then
+    echo "Usage: $0 [config file]"
+    exit 1
+fi
+
 
-if [ "$#" -ge 1 ]; then
-    PERSITENT_DIR="$1"
+# Default config values
+if [ -z "$SCITRAN_SYSTEM_HOST" ]; then
+    SCITRAN_SYSTEM_HOST="127.0.0.1"
 fi
-if [ "$#" -eq 2 ]; then
-    RUNTIME_DIR="$2"
+if [ -z "$SCITRAN_SYSTEM_PORT" ]; then
+    SCITRAN_SYSTEM_PORT="8080"
 fi
-if [ "$#" -gt 2 ]; then
-    echo "Usage: $0 persistent runtime"
-    exit 1
+if [ -z "$SCITRAN_SYSTEM_RUNTIME" ]; then
+    SCITRAN_SYSTEM_RUNTIME="./runtime"
+fi
+if [ -z "$SCITRAN_SYSTEM_SSL_PEM" ]; then
+    SCITRAN_SYSTEM_SSL_PEM=""
+fi
+if [ -z "$SCITRAN_PERSISTENT_PATH" ]; then
+    SCITRAN_PERSISTENT_PATH="./persistent"
+fi
+if [ -z "$SCITRAN_PERSISTENT_DB_PORT" ]; then
+    SCITRAN_PERSISTENT_DB_PORT="9001"
+fi
+if [ -z "$SCITRAN_PERSISTENT_DB_URI" ]; then
+    SCITRAN_PERSISTENT_DB_URI="mongodb://localhost:$SCITRAN_PERSISTENT_DB_PORT/scitran"
 fi
 
 
-if [ -f "$PERSITENT_DIR/db/mongod.lock" ]; then
-    echo "Database exists at $PERSITENT_DIR/db. Not bootstrapping users."
+if [ -f "$SCITRAN_PERSISTENT_PATH/db/mongod.lock" ]; then
     BOOTSTRAP_USERS=0
 else
-    echo "Creating database location at $PERSITENT_DIR/db"
-    mkdir -p $PERSITENT_DIR/db
+    echo "Creating database location at $SCITRAN_PERSISTENT_PATH/db"
+    mkdir -p $SCITRAN_PERSISTENT_PATH/db
     if ! [ -f "bootstrap.json" ]; then
         echo "Cannot bootstrap users. Please create bootstrap.json from bootstrap.json.sample."
         exit 1
@@ -63,32 +90,32 @@ else
     echo "Installed Virtualenv"
 fi
 
-if [ -d "$RUNTIME_DIR" ]; then
-    echo "Virtualenv exists present at $RUNTIME_DIR"
+if [ -d "$SCITRAN_SYSTEM_RUNTIME" ]; then
+    echo "Virtualenv exists present at $SCITRAN_SYSTEM_RUNTIME"
 else
-    echo "Creating 'scitran' Virtualenv at $RUNTIME_DIR"
-    virtualenv -p `brew --prefix`/bin/python --prompt="(scitran)" $RUNTIME_DIR
-    echo "Created 'scitran' Virtualenv at $RUNTIME_DIR"
+    echo "Creating 'scitran' Virtualenv at $SCITRAN_SYSTEM_RUNTIME"
+    virtualenv -p `brew --prefix`/bin/python --prompt="(scitran)" $SCITRAN_SYSTEM_RUNTIME
+    echo "Created 'scitran' Virtualenv at $SCITRAN_SYSTEM_RUNTIME"
 fi
 
-if [ -f "$RUNTIME_DIR/bin/mongod" ]; then
+if [ -f "$SCITRAN_SYSTEM_RUNTIME/bin/mongod" ]; then
     echo "MongoDB is installed"
 else
     echo "Installing MongoDB"
-    curl https://fastdl.mongodb.org/osx/mongodb-osx-x86_64-3.0.7.tgz | tar xz -C $RUNTIME_DIR --strip-components 1
+    curl https://fastdl.mongodb.org/osx/mongodb-osx-x86_64-3.0.7.tgz | tar xz -C $SCITRAN_SYSTEM_RUNTIME --strip-components 1
     echo "MongoDB installed"
 fi
 
 
 echo "Activating Virtualenv"
-source $RUNTIME_DIR/bin/activate
+source $SCITRAN_SYSTEM_RUNTIME/bin/activate
 
 echo "Installing Python requirements"
 pip install -U -r requirements.txt
 
 
 # Launch mongod
-mongod --dbpath $PERSITENT_DIR/db --smallfiles --port 9001 &
+mongod --dbpath $SCITRAN_PERSISTENT_PATH/db --smallfiles --port $SCITRAN_PERSISTENT_DB_PORT &
 MONGO_PID=$!
 
 # Set python path so scripts can work
@@ -98,28 +125,51 @@ export PYTHONPATH=.
 if [ $BOOTSTRAP_USERS -eq 1 ]; then
     echo "Bootstrapping users"
     bin/bootstrap.py users bootstrap.json
+else
+    echo "Database exists at $SCITRAN_PERSISTENT_PATH/db. Not bootstrapping users."
 fi
 
-if [ ! -d "$PERSITENT_DIR/testdata-master" ]; then
-    echo "Downloading testdata"
-    curl https://codeload.github.com/scitran/testdata/tar.gz/master | tar xz -C $PERSITENT_DIR
+TESTDATA_VERSION=$(curl -sLI https://github.com/scitran/testdata/archive/master.tar.gz | grep ETag | tail -n 1 | cut -f 2 -d '"')
+if [ ! -d "$SCITRAN_PERSISTENT_PATH/testdata" ]; then
+    echo "Downloading testdata to $SCITRAN_PERSISTENT_PATH/testdata"
+    mkdir "$SCITRAN_PERSISTENT_PATH/testdata"
+    curl -L https://github.com/scitran/testdata/archive/master.tar.gz | tar xz -C "$SCITRAN_PERSISTENT_PATH/testdata" --strip-components 1
+else
+    if [ "$TESTDATA_VERSION" != "$(cat $SCITRAN_PERSISTENT_PATH/.testdata_version)" ]; then
+        echo "Testdata out of data; downloading"
+        curl -L https://github.com/scitran/testdata/archive/master.tar.gz | tar xz -C "$SCITRAN_PERSISTENT_PATH/testdata" --strip-components 1
+    else
+        echo "Testdata up to date"
+    fi
 fi
+echo "$TESTDATA_VERSION" > "$SCITRAN_PERSISTENT_PATH/.testdata_version"
 
-if [ -d "$PERSITENT_DIR/data" ]; then
-    echo "Persistence store exists at $PERSITENT_DIR/data. Not bootstrapping data. Remove to re-bootstrap."
+if [ -d "$SCITRAN_PERSISTENT_PATH/data" ]; then
+    echo "Persistence store exists at $SCITRAN_PERSISTENT_PATH/data. Not bootstrapping data. Remove to re-bootstrap."
 else
     echo "Bootstrapping testdata"
-    bin/bootstrap.py data --copy $PERSITENT_DIR/testdata-master $PERSITENT_DIR/data
+    bin/bootstrap.py data --copy $SCITRAN_PERSISTENT_PATH/testdata $SCITRAN_PERSISTENT_PATH/data
     echo "Bootstrapped testdata"
 fi
 
-# Serve API with paste
-# python bin/api.wsgi --data_path $PERSITENT_DIR/data --ssl --insecure --log_level debug --drone_secret scitran_drone --db_uri mongodb://localhost/scitran
 
 # Serve API with PasteScript
-paster serve dev.ini --reload
-
-# Exit out of the python virtualenv
+TEMP_INI_FILE=$(mktemp -t scitran_api)
+cat << EOF > $TEMP_INI_FILE
+[server:main]
+use = egg:Paste#http
+host = $SCITRAN_SYSTEM_HOST
+port = $SCITRAN_SYSTEM_PORT
+ssl_pem=$SCITRAN_SYSTEM_SSL_PEM
+
+[app:main]
+paste.app_factory = api.api:app_factory
+EOF
+
+paster serve --reload $TEMP_INI_FILE
+
+# Clean up and exit out of the python virtualenv
+rm -f $TEMP_INI_FILE
 deactivate
 
 # Shutdown mongod on ctrl+C

sample.config

+# vim: filetype=sh
+
+#SCITRAN_SYSTEM_HOST="127.0.0.1"
+#SCITRAN_SYSTEM_PORT="8080"
+#SCITRAN_SYSTEM_RUNTIME="./runtime"
+#SCITRAN_SYSTEM_SSL_PEM="*"
+
+#SCITRAN_CORE_INSECURE=false
+#SCITRAN_CORE_LOG_LEVEL=debug
+#SCITRAN_CORE_NEWRELIC=none
+
+#SCITRAN_SITE__ID=""
+#SCITRAN_SITE_NAME=""
+#SCITRAN_SITE_URL=""
+#SCITRAN_SITE_CENTRAL_URL=""
+#SCITRAN_SITE_REGISTERED=""
+#SCITRAN_SITE_SSL_CERT=""
+
+#SCITRAN_PERSISTENT_PATH="./persistent"
+#SCITRAN_PERSISTENT_DB_PORT=9001
+#SCITRAN_PERSISTENT_DB_URI="mongodb://localhost:$SCITRAN_PERSISTENT_DB_PORT/scitran"
+
+#SCITRAN_AUTH_AUTH_ENDPOINT=""
+#SCITRAN_AUTH_CLIENT_ID=""
+#SCITRAN_AUTH_DRONE_SECRET=""
+#SCITRAN_AUTH_ID_ENDPOINT=""
+#SCITRAN_AUTH_VERIFY_ENDPOINT=""