diff --git a/api/api.py b/api/api.py index 3ef6ebd13fe69ac242dc34e5143da6d62c90ceb0..26e3f86565eec6050eaf64a659f2ae6a307284c4 100644 --- a/api/api.py +++ b/api/api.py @@ -124,10 +124,10 @@ def dispatcher(router, request, response): application = webapp2.WSGIApplication(routes) application.router.set_dispatcher(dispatcher) -if config.get_item('system', 'newrelic'): +if config.get_item('core', 'newrelic'): try: import newrelic.agent, newrelic.api.exceptions - newrelic.agent.initialize(config.get_item('system', 'newrelic')) + newrelic.agent.initialize(config.get_item('core', 'newrelic')) application = newrelic.agent.WSGIApplicationWrapper(application) log.info('New Relic detected and loaded. Monitoring enabled.') except ImportError: diff --git a/api/base.py b/api/base.py index d035fd19195fb2dda1bca7c7a1d9bdf97274cf5b..1746ab327d0a6ad3d1c62799ec9c4e1979250473 100644 --- a/api/base.py +++ b/api/base.py @@ -18,7 +18,7 @@ class RequestHandler(webapp2.RequestHandler): def __init__(self, request=None, response=None): self.initialize(request, response) - self.debug = config.get_item('system', 'insecure') + self.debug = config.get_item('core', 'insecure') request_start = datetime.datetime.utcnow() provider_avatar = None diff --git a/api/config.py b/api/config.py index 798e5daa8a7c3dcc6488697408c602a7eb0cb4b3..773cf81d85024ad579656b62b5f01bef2b6d8974 100644 --- a/api/config.py +++ b/api/config.py @@ -20,8 +20,8 @@ logging.getLogger('paste.httpserver').setLevel(logging.WARNING) # silence Paste DEFAULT_CONFIG = { - 'system': { - 'log_level': 'debug', + 'core': { + 'log_level': 'info', 'insecure': False, 'persisted': False, 'newrelic': None, @@ -44,7 +44,6 @@ DEFAULT_CONFIG = { }, 'persistent': { 'db_uri': 'mongodb://localhost:9001/scitran', - 'db_path': os.path.join(os.path.dirname(__file__), '../persistent/db'), 'data_path': os.path.join(os.path.dirname(__file__), '../persistent/data'), }, } @@ -90,23 +89,23 @@ db.groups.update_one({'_id': 'unknown'}, {'$setOnInsert': { 'created': now, 'mod db.sites.replace_one({'_id': __config['site']['_id']}, {'name': __config['site']['name'], 'site_url': __config['site']['url']}, upsert=True) -def get_config(projection=None): +def get_config(): global __last_update, __config, environment_read now = datetime.datetime.utcnow() - if not __config['system']['persisted']: + if not __config['core']['persisted']: __config['modified'] = now flat_config= util.mongo_dict(__config) r = db.config.update_one({'latest': True}, {'$set': flat_config, '$setOnInsert': {'created': now}}, upsert=True) - __config['system']['persisted'] = bool(r.modified_count) + __config['core']['persisted'] = bool(r.modified_count) elif now - __last_update > datetime.timedelta(seconds=120): - __config = db.config.find_one({'latest': True}, projection) + __config = db.config.find_one({'latest': True}) __last_update = now - log.setLevel(getattr(logging, __config['system']['log_level'].upper())) + log.setLevel(getattr(logging, __config['core']['log_level'].upper())) return __config def get_public_config(): - projection = None # FIXME: define public config whitelist - return get_config(projection) + projection = ['created', 'modified', 'site', 'auth'] + return db.config.find_one({'latest': True}, projection) def get_item(outer, inner): return get_config()[outer][inner] diff --git a/dev.ini b/dev.ini deleted file mode 100644 index c11224f99917383af6bf9dbc9113ddd8362c3958..0000000000000000000000000000000000000000 --- a/dev.ini +++ /dev/null @@ -1,8 +0,0 @@ -[server:main] -use = egg:Paste#http -host = 127.0.0.1 -port = 8080 -ssl_pem=/Users/gsfr/Applications/localhost.pem - -[app:main] -paste.app_factory = api.api:app_factory diff --git a/run.sh b/run.sh index 831d6f99f50099fcf21211b48ea0ff84bddb89b6..dc22dd1d0d022cbc6ed7cb61069416a58d5a7eee 100755 --- a/run.sh +++ b/run.sh @@ -2,27 +2,54 @@ set -e -RUNTIME_DIR="./runtime" -PERSITENT_DIR="./persistent" +unset CDPATH +cd "$( dirname "${BASH_SOURCE[0]}" )" + + +if [ "$#" -eq 1 ]; then + TEMP_ENV_FILE=$(mktemp -t scitran_env) + env > $TEMP_ENV_FILE + set -o allexport + source "$1" + source $TEMP_ENV_FILE + rm -f $TEMP_ENV_FILE + set +o allexport +fi +if [ "$#" -gt 1 ]; then + echo "Usage: $0 [config file]" + exit 1 +fi + -if [ "$#" -ge 1 ]; then - PERSITENT_DIR="$1" +# Default config values +if [ -z "$SCITRAN_SYSTEM_HOST" ]; then + SCITRAN_SYSTEM_HOST="127.0.0.1" fi -if [ "$#" -eq 2 ]; then - RUNTIME_DIR="$2" +if [ -z "$SCITRAN_SYSTEM_PORT" ]; then + SCITRAN_SYSTEM_PORT="8080" fi -if [ "$#" -gt 2 ]; then - echo "Usage: $0 persistent runtime" - exit 1 +if [ -z "$SCITRAN_SYSTEM_RUNTIME" ]; then + SCITRAN_SYSTEM_RUNTIME="./runtime" +fi +if [ -z "$SCITRAN_SYSTEM_SSL_PEM" ]; then + SCITRAN_SYSTEM_SSL_PEM="" +fi +if [ -z "$SCITRAN_PERSISTENT_PATH" ]; then + SCITRAN_PERSISTENT_PATH="./persistent" +fi +if [ -z "$SCITRAN_PERSISTENT_DB_PORT" ]; then + SCITRAN_PERSISTENT_DB_PORT="9001" +fi +if [ -z "$SCITRAN_PERSISTENT_DB_URI" ]; then + SCITRAN_PERSISTENT_DB_URI="mongodb://localhost:$SCITRAN_PERSISTENT_DB_PORT/scitran" fi -if [ -f "$PERSITENT_DIR/db/mongod.lock" ]; then - echo "Database exists at $PERSITENT_DIR/db. Not bootstrapping users." +if [ -f "$SCITRAN_PERSISTENT_PATH/db/mongod.lock" ]; then BOOTSTRAP_USERS=0 else - echo "Creating database location at $PERSITENT_DIR/db" - mkdir -p $PERSITENT_DIR/db + echo "Creating database location at $SCITRAN_PERSISTENT_PATH/db" + mkdir -p $SCITRAN_PERSISTENT_PATH/db if ! [ -f "bootstrap.json" ]; then echo "Cannot bootstrap users. Please create bootstrap.json from bootstrap.json.sample." exit 1 @@ -63,32 +90,32 @@ else echo "Installed Virtualenv" fi -if [ -d "$RUNTIME_DIR" ]; then - echo "Virtualenv exists present at $RUNTIME_DIR" +if [ -d "$SCITRAN_SYSTEM_RUNTIME" ]; then + echo "Virtualenv exists present at $SCITRAN_SYSTEM_RUNTIME" else - echo "Creating 'scitran' Virtualenv at $RUNTIME_DIR" - virtualenv -p `brew --prefix`/bin/python --prompt="(scitran)" $RUNTIME_DIR - echo "Created 'scitran' Virtualenv at $RUNTIME_DIR" + echo "Creating 'scitran' Virtualenv at $SCITRAN_SYSTEM_RUNTIME" + virtualenv -p `brew --prefix`/bin/python --prompt="(scitran)" $SCITRAN_SYSTEM_RUNTIME + echo "Created 'scitran' Virtualenv at $SCITRAN_SYSTEM_RUNTIME" fi -if [ -f "$RUNTIME_DIR/bin/mongod" ]; then +if [ -f "$SCITRAN_SYSTEM_RUNTIME/bin/mongod" ]; then echo "MongoDB is installed" else echo "Installing MongoDB" - curl https://fastdl.mongodb.org/osx/mongodb-osx-x86_64-3.0.7.tgz | tar xz -C $RUNTIME_DIR --strip-components 1 + curl https://fastdl.mongodb.org/osx/mongodb-osx-x86_64-3.0.7.tgz | tar xz -C $SCITRAN_SYSTEM_RUNTIME --strip-components 1 echo "MongoDB installed" fi echo "Activating Virtualenv" -source $RUNTIME_DIR/bin/activate +source $SCITRAN_SYSTEM_RUNTIME/bin/activate echo "Installing Python requirements" pip install -U -r requirements.txt # Launch mongod -mongod --dbpath $PERSITENT_DIR/db --smallfiles --port 9001 & +mongod --dbpath $SCITRAN_PERSISTENT_PATH/db --smallfiles --port $SCITRAN_PERSISTENT_DB_PORT & MONGO_PID=$! # Set python path so scripts can work @@ -98,28 +125,51 @@ export PYTHONPATH=. if [ $BOOTSTRAP_USERS -eq 1 ]; then echo "Bootstrapping users" bin/bootstrap.py users bootstrap.json +else + echo "Database exists at $SCITRAN_PERSISTENT_PATH/db. Not bootstrapping users." fi -if [ ! -d "$PERSITENT_DIR/testdata-master" ]; then - echo "Downloading testdata" - curl https://codeload.github.com/scitran/testdata/tar.gz/master | tar xz -C $PERSITENT_DIR +TESTDATA_VERSION=$(curl -sLI https://github.com/scitran/testdata/archive/master.tar.gz | grep ETag | tail -n 1 | cut -f 2 -d '"') +if [ ! -d "$SCITRAN_PERSISTENT_PATH/testdata" ]; then + echo "Downloading testdata to $SCITRAN_PERSISTENT_PATH/testdata" + mkdir "$SCITRAN_PERSISTENT_PATH/testdata" + curl -L https://github.com/scitran/testdata/archive/master.tar.gz | tar xz -C "$SCITRAN_PERSISTENT_PATH/testdata" --strip-components 1 +else + if [ "$TESTDATA_VERSION" != "$(cat $SCITRAN_PERSISTENT_PATH/.testdata_version)" ]; then + echo "Testdata out of data; downloading" + curl -L https://github.com/scitran/testdata/archive/master.tar.gz | tar xz -C "$SCITRAN_PERSISTENT_PATH/testdata" --strip-components 1 + else + echo "Testdata up to date" + fi fi +echo "$TESTDATA_VERSION" > "$SCITRAN_PERSISTENT_PATH/.testdata_version" -if [ -d "$PERSITENT_DIR/data" ]; then - echo "Persistence store exists at $PERSITENT_DIR/data. Not bootstrapping data. Remove to re-bootstrap." +if [ -d "$SCITRAN_PERSISTENT_PATH/data" ]; then + echo "Persistence store exists at $SCITRAN_PERSISTENT_PATH/data. Not bootstrapping data. Remove to re-bootstrap." else echo "Bootstrapping testdata" - bin/bootstrap.py data --copy $PERSITENT_DIR/testdata-master $PERSITENT_DIR/data + bin/bootstrap.py data --copy $SCITRAN_PERSISTENT_PATH/testdata $SCITRAN_PERSISTENT_PATH/data echo "Bootstrapped testdata" fi -# Serve API with paste -# python bin/api.wsgi --data_path $PERSITENT_DIR/data --ssl --insecure --log_level debug --drone_secret scitran_drone --db_uri mongodb://localhost/scitran # Serve API with PasteScript -paster serve dev.ini --reload - -# Exit out of the python virtualenv +TEMP_INI_FILE=$(mktemp -t scitran_api) +cat << EOF > $TEMP_INI_FILE +[server:main] +use = egg:Paste#http +host = $SCITRAN_SYSTEM_HOST +port = $SCITRAN_SYSTEM_PORT +ssl_pem=$SCITRAN_SYSTEM_SSL_PEM + +[app:main] +paste.app_factory = api.api:app_factory +EOF + +paster serve --reload $TEMP_INI_FILE + +# Clean up and exit out of the python virtualenv +rm -f $TEMP_INI_FILE deactivate # Shutdown mongod on ctrl+C diff --git a/sample.config b/sample.config new file mode 100644 index 0000000000000000000000000000000000000000..d5aa18285ab385fd89b53e29108d23bef23e09ad --- /dev/null +++ b/sample.config @@ -0,0 +1,27 @@ +# vim: filetype=sh + +#SCITRAN_SYSTEM_HOST="127.0.0.1" +#SCITRAN_SYSTEM_PORT="8080" +#SCITRAN_SYSTEM_RUNTIME="./runtime" +#SCITRAN_SYSTEM_SSL_PEM="*" + +#SCITRAN_CORE_INSECURE=false +#SCITRAN_CORE_LOG_LEVEL=debug +#SCITRAN_CORE_NEWRELIC=none + +#SCITRAN_SITE__ID="" +#SCITRAN_SITE_NAME="" +#SCITRAN_SITE_URL="" +#SCITRAN_SITE_CENTRAL_URL="" +#SCITRAN_SITE_REGISTERED="" +#SCITRAN_SITE_SSL_CERT="" + +#SCITRAN_PERSISTENT_PATH="./persistent" +#SCITRAN_PERSISTENT_DB_PORT=9001 +#SCITRAN_PERSISTENT_DB_URI="mongodb://localhost:$SCITRAN_PERSISTENT_DB_PORT/scitran" + +#SCITRAN_AUTH_AUTH_ENDPOINT="" +#SCITRAN_AUTH_CLIENT_ID="" +#SCITRAN_AUTH_DRONE_SECRET="" +#SCITRAN_AUTH_ID_ENDPOINT="" +#SCITRAN_AUTH_VERIFY_ENDPOINT=""