Merge branch 'new-deploy'

api.py

@@ -99,6 +99,7 @@ app.config = {
         'ssl_cert':         None,
         'insecure':         False,
         'log_path':         None,
+        'demo':             False,
         }
 
 
@@ -119,6 +120,7 @@ if __name__ == '__main__':
     arg_parser.add_argument('--site_id', help='InterNIMS site ID')
     arg_parser.add_argument('--site_name', help='InterNIMS site name')
     arg_parser.add_argument('--oauth2_id_endpoint', help='OAuth2 provider ID endpoint')
+    arg_parser.add_argument('--demo', help='demo mode, enables auto user creation')
     args = arg_parser.parse_args()
 
     app.config['here'] = os.path.dirname(os.path.abspath(args.config_file))
@@ -135,6 +137,7 @@ if __name__ == '__main__':
     app.config['oauth2_id_endpoint'] = args.oauth2_id_endpoint or config.get('oauth2', 'id_endpoint')
     app.config['insecure'] = config.getboolean('nims', 'insecure')
     app.config['ssl_cert'] = args.ssl_cert or config.get('nims', 'ssl_cert')     # to give to requests
+    app.config['demo'] = arg.demo or config.getboolean('nims', 'demo')
 
     if not app.config['ssl_cert']:
         log.warning('SSL certificate not specified, interNIMS functionality disabled')

api.wsgi

@@ -3,9 +3,11 @@
 import os
 import sys
 import site
+import time
+
 import ConfigParser
 
-configfile = '../production.ini'
+configfile = '/service/config/production.ini'
 config = ConfigParser.ConfigParser(allow_no_value=True)
 config.read(configfile)
 
@@ -35,6 +37,7 @@ application.config['site_id'] = config.get('nims', 'site_id')
 application.config['ssl_cert'] = config.get('nims', 'ssl_cert')
 application.config['oauth2_id_endpoint'] = config.get('oauth2', 'id_endpoint')
 application.config['insecure'] = config.getboolean('nims', 'insecure')
+application.config['demo'] = config.getboolean('nims', 'demo')
 
 if not os.path.exists(application.config['data_path']):
     os.makedirs(application.config['data_path'])
@@ -44,8 +47,20 @@ if not os.path.exists(application.config['quarantine_path']):
 # connect to db
 kwargs = dict(tz_aware=True)
 db_uri = config.get('nims', 'db_uri')
-db_client = pymongo.MongoReplicaSetClient(db_uri, **kwargs) if 'replicaSet' in db_uri else pymongo.MongoClient(db_uri, **kwargs)
-application.db = db_client.get_default_database()
+db_client = None
+application.db = None
+
+for x in range(0, 30):
+    try:
+        db_client = pymongo.MongoReplicaSetClient(db_uri, **kwargs) if 'replicaSet' in db_uri else pymongo.MongoClient(db_uri, **kwargs)
+        application.db = db_client.get_default_database()
+    except:
+        time.sleep(1)
+        pass
+    else:
+        break
+else:
+    raise Exception("Could not connect to MongoDB")
 
 
 # internims, send is-alive signals

base.py

@@ -6,6 +6,7 @@ logging.getLogger('requests').setLevel(logging.WARNING) # silence Requests libra
 
 import copy
 import json
+import hashlib
 import webapp2
 import datetime
 import requests
@@ -23,6 +24,8 @@ class RequestHandler(webapp2.RequestHandler):
 
         # set uid, source_site, public_request, and superuser
         self.uid = None
+        firstname = None
+        lastname = None
         self.source_site = None
         access_token = self.request.headers.get('Authorization', None)
         if access_token and self.app.config['oauth2_id_endpoint']:
@@ -34,7 +37,10 @@ class RequestHandler(webapp2.RequestHandler):
             else:
                 r = requests.get(self.app.config['oauth2_id_endpoint'], headers={'Authorization': 'Bearer ' + access_token})
                 if r.status_code == 200:
-                    self.uid = json.loads(r.content)['email']
+                    identity = json.loads(r.content)
+                    self.uid = identity['email']
+                    firstname = identity['given_name']
+                    lastname = identity['family_name']
                     self.app.db.tokens.insert({'_id': access_token, 'uid': self.uid, 'timestamp': datetime.datetime.utcnow()})
                     log.debug('looked up remote token in %dms' % ((datetime.datetime.now() - token_request_time).total_seconds() * 1000.))
                 else:
@@ -56,7 +62,19 @@ class RequestHandler(webapp2.RequestHandler):
         else:
             user = self.app.db.users.find_one({'_id': self.uid}, ['root', 'wheel'])
             if not user:
-                self.abort(403, 'user ' + self.uid + ' does not exist')
+                if self.app.config['demo']:
+                    self.app.db.users.insert({
+                        '_id': self.uid,
+                        'email': self.uid,
+                        'email_hash': hashlib.md5(self.uid).hexdigest(),
+                        'firstname': firstname or 'DEMO',
+                        'lastname': lastname or 'DEMO',
+                        'wheel': True,
+                        'root': True,
+                    })
+                    user = self.app.db.users.find_one({'_id': self.uid}, ['root', 'wheel'])
+                else:
+                    self.abort(403, 'user ' + self.uid + ' does not exist')
             self.superuser_request = user.get('root') and user.get('wheel')
 
     def dispatch(self):

bootstrap.py

@@ -4,14 +4,29 @@
 
 import os
 import json
+import time
 import pymongo
 import hashlib
 import logging
 import argparse
 
 
+def connect_db(db_uri, **kwargs):
+    for x in range(0, 30):
+        try:
+            db_client = pymongo.MongoReplicaSetClient(db_uri, **kwargs) if 'replicaSet' in db_uri else pymongo.MongoClient(db_uri, **kwargs)
+        except:
+            time.sleep(1)
+            pass
+        else:
+            break
+    else:
+        raise Exception("Could not connect to MongoDB")
+    return db_client
+
+
 def rsinit(args):
-    db_client = pymongo.MongoClient(args.uri)
+    db_client = pymongo.MongoClient(args.db_uri)
     repl_conf = eval(args.config)
     db_client.admin.command('replSetInitiate', repl_conf)
 
@@ -27,10 +42,10 @@ example:
 
 
 def authinit(args):
-    db_client = pymongo.MongoClient(args.uri)
+    db_client = pymongo.MongoClient(args.db_uri)
     db_client['admin'].add_user(name='admin', password=args.password, roles=['userAdminAnyDatabase'])
     db_client['nims'].add_user(name=args.username, password=args.password, roles=['readWrite', 'dbAdmin'])
-    uri_parts = args.uri.partition('://')
+    uri_parts = args.db_uri.partition('://')
     print 'You must now restart mongod with the "--auth" parameter and modify your URI as follows:'
     print '    %s%s:%s@%s' % (uri_parts[0] + uri_parts[1], args.username, args.password, uri_parts[2])
 
@@ -43,7 +58,7 @@ example:
 
 
 def dbinit(args):
-    db_client = pymongo.MongoReplicaSetClient(args.uri) if 'replicaSet' in args.uri else pymongo.MongoClient(args.uri)
+    db_client = connect_db(args.db_uri)
     db = db_client.get_default_database()
 
     if args.force:
@@ -87,7 +102,7 @@ def sort(args):
         os.makedirs(quarantine_path)
     print 'initializing DB'
     kwargs = dict(tz_aware=True)
-    db_client = pymongo.MongoReplicaSetClient(args.db_uri, **kwargs) if 'replicaSet' in args.db_uri else pymongo.MongoClient(args.db_uri, **kwargs)
+    db_client = connect_db(args.db_uri, **kwargs)
     db = db_client.get_default_database()
     print 'inspecting %s' % args.path
     files = []
@@ -114,6 +129,15 @@ example:
 ./scripts/bootstrap.py sort mongodb://localhost/nims /tmp/data /tmp/sorted
 """
 
+def dbinitsort(args):
+    logging.basicConfig(level=logging.WARNING)
+    dbinit(args)
+    upload(args)
+
+dbinitsort_desc = """
+example:
+./scripts/bootstrap.py dbinitsort mongodb://localhost/nims -j bootstrap.json /tmp/data https://example.com/api/upload
+"""
 
 def upload(args):
     import util
@@ -139,7 +163,7 @@ def upload(args):
             headers = {'User-Agent': 'bootstrapper', 'Content-MD5': hash_.hexdigest()}
             try:
                 start = datetime.datetime.now()
-                r = requests.put(args.url + '?filename=%s' % filename, data=fd, headers=headers)
+                r = requests.put(args.url + '?filename=%s' % filename, data=fd, headers=headers, verify=not args.no_verify)
                 upload_duration = (datetime.datetime.now() - start).total_seconds()
             except requests.exceptions.ConnectionError as e:
                 print 'error       %s: %s' % (filename, e)
@@ -147,7 +171,7 @@ def upload(args):
                 if r.status_code == 200:
                     print 'success     %s [%s/s]' % (filename, util.hrsize(os.path.getsize(filepath)/upload_duration))
                 else:
-                    print 'failure     %s: %s %s, %s' % (log_info, filename, r.status_code, r.reason, r.text)
+                    print 'failure     %s: %s %s, %s' % (filename, r.status_code, r.reason, r.text)
 
 upload_desc = """
 example:
@@ -164,7 +188,7 @@ rsinit_parser = subparsers.add_parser(
         description=rsinit_desc,
         formatter_class=argparse.RawDescriptionHelpFormatter,
         )
-rsinit_parser.add_argument('uri', help='DB URI')
+rsinit_parser.add_argument('db_uri', help='DB URI')
 rsinit_parser.add_argument('config', help='replication set config')
 rsinit_parser.set_defaults(func=rsinit)
 
@@ -176,7 +200,7 @@ authinit_parser = subparsers.add_parser(
         )
 authinit_parser.add_argument('username', help='DB username')
 authinit_parser.add_argument('password', help='DB password')
-authinit_parser.add_argument('uri', help='DB URI')
+authinit_parser.add_argument('db_uri', help='DB URI')
 authinit_parser.set_defaults(func=authinit)
 
 dbinit_parser = subparsers.add_parser(
@@ -187,7 +211,7 @@ dbinit_parser = subparsers.add_parser(
         )
 dbinit_parser.add_argument('-f', '--force', action='store_true', help='wipe out any existing data')
 dbinit_parser.add_argument('-j', '--json', help='JSON file containing users and groups')
-dbinit_parser.add_argument('uri', help='DB URI')
+dbinit_parser.add_argument('db_uri', help='DB URI')
 dbinit_parser.set_defaults(func=dbinit)
 
 sort_parser = subparsers.add_parser(
@@ -202,6 +226,20 @@ sort_parser.add_argument('path', help='filesystem path to data')
 sort_parser.add_argument('sort_path', help='filesystem path to sorted data')
 sort_parser.set_defaults(func=sort)
 
+dbinitsort_parser = subparsers.add_parser(
+    name='dbinitsort',
+    help='initialize database, then sort all files in a directory tree',
+    description=dbinitsort_desc,
+    formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+dbinitsort_parser.add_argument('db_uri', help='database URI')
+dbinitsort_parser.add_argument('path', help='filesystem path to data')
+dbinitsort_parser.add_argument('url', help='upload URL')
+dbinitsort_parser.add_argument('-j', '--json', help='JSON file container users and groups')
+dbinitsort_parser.add_argument('-f', '--force', action='store_true', help='wipe out any existing db data')
+dbinitsort_parser.add_argument('-n', '--no_verify', help='disable SSL verification', action='store_true')
+dbinitsort_parser.set_defaults(func=dbinitsort)
+
 upload_parser = subparsers.add_parser(
         name='upload',
         help='upload all files in a directory tree',
@@ -210,6 +248,7 @@ upload_parser = subparsers.add_parser(
         )
 upload_parser.add_argument('path', help='filesystem path to data')
 upload_parser.add_argument('url', help='upload URL')
+upload_parser.add_argument('-n', '--no_verify', help='disable SSL verification', action='store_true')
 upload_parser.set_defaults(func=upload)
 
 args = parser.parse_args()

users.py

@@ -181,13 +181,26 @@ class Groups(base.RequestHandler):
 
     """/nimsapi/groups """
 
+    def __init__(self, request=None, response=None):
+        super(Groups, self).__init__(request, response)
+        self.dbc = self.app.db.groups
+
     def count(self):
         """Return the number of Groups."""
         self.response.write(self.app.db.groups.count())
 
     def post(self):
         """Create a new Group"""
-        self.response.write('groups post\n')
+        if not self.superuser_request:
+            self.abort(403, 'must be superuser to create new group')
+        try:
+            json_body = self.request.json_body
+            jsonschema.validate(json_body, Group.json_schema)
+            self.dbc.insert(json_body)
+        except (ValueError, jsonschema.ValidationError) as e:
+            self.abort(400, str(e))
+        except pymongo.errors.DuplicateKeyError as e:
+            self.abort(400, 'Groups ID %s already exists' % json_body['_id'])
 
     def get(self, _id=None):
         """Return the list of Groups."""
@@ -219,7 +232,7 @@ class Group(base.RequestHandler):
         'type': 'object',
         'properties': {
             '_id': {
-                'title': 'Database ID',
+                'title': 'Group ID',
                 'type': 'string',
             },
             'name': {
@@ -234,14 +247,16 @@ class Group(base.RequestHandler):
                 'items': {
                     'type': 'object',
                     'properties': {
-                        '_id': {
+                        'access': {
                             'type': 'string',
+                            'enum': [role['rid'] for role in ROLES],
                         },
-                        'access': {
+                        '_id': {
                             'type': 'string',
-                            'enum': [k for k, v in sorted(INTEGER_ROLES.iteritems(), key=lambda (k, v): v)],
                         },
                     },
+                    'required': ['access', '_id'],
+                    'additionalProperties': False,
                 },
                 'uniqueItems': True,
             },
@@ -249,6 +264,10 @@ class Group(base.RequestHandler):
         'required': ['_id'],
     }
 
+    def __init__(self, request=None, response=None):
+        super(Group, self).__init__(request, response)
+        self.dbc = self.app.db.groups
+
     def get(self, _id):
         """Return Group details."""
         group = self.app.db.groups.find_one({'_id': _id})
@@ -266,3 +285,7 @@ class Group(base.RequestHandler):
 
     def delete(self, _id):
         """Delete an Group."""
+        if not self.superuser_request:
+            self.abort(403, 'must be superuser to delete a Group')
+        # TODO: block deletion, if group is referenced by any projects
+        self.dbc.remove({'_id': _id})

util.py

@@ -8,6 +8,7 @@ import copy
 import shutil
 import difflib
 import datetime
+import tempdir as tempfile
 
 import scitran.data