merciless refactoring

collections_.py

 # @author:  Gunnar Schaefer
 
 import re
-import json
 import webapp2
 import bson.json_util
 
@@ -48,7 +47,7 @@ class Collections(nimsapiutil.NIMSRequestHandler):
 
     def count(self):
         """Return the number of Collections."""
-        self.response.write(json.dumps(self.app.db.collections.count()))
+        self.response.write(self.app.db.collections.count())
 
     def post(self):
         """Create a new Collection."""
@@ -63,8 +62,7 @@ class Collections(nimsapiutil.NIMSRequestHandler):
         """Return the list of Collections."""
         query = {'permissions.uid': self.uid} if not self.user_is_superuser else None
         projection = {'curator': 1, 'name': 1, 'notes': 1, 'permissions': {'$elemMatch': {'uid': self.uid}}}
-        collections = list(self.app.db.collections.find(query, projection))
-        self.response.write(json.dumps(collections, default=bson.json_util.default))
+        return list(self.app.db.collections.find(query, projection))
 
     def put(self):
         """Update many Collections."""
@@ -114,8 +112,7 @@ class Collection(nimsapiutil.NIMSRequestHandler):
     def get(self, cid):
         """Return one Collection, conditionally with details."""
         cid = bson.ObjectId(cid)
-        collection = self.get_collection(cid)
-        self.response.write(json.dumps(collection, default=bson.json_util.default))
+        return self.get_collection(cid)
 
     def put(self, cid):
         """Update an existing Collection."""
@@ -171,7 +168,7 @@ class Sessions(nimsapiutil.NIMSRequestHandler):
 
     def count(self):
         """Return the number of Sessions."""
-        self.response.write(json.dumps(self.app.db.sessions.count()))
+        self.response.write(self.app.db.sessions.count())
 
     def post(self):
         """Create a new Session"""
@@ -190,7 +187,7 @@ class Sessions(nimsapiutil.NIMSRequestHandler):
         sessions = list(self.app.db.sessions.find(query, projection))
         for sess in sessions:
             sess['site'] = self.app.config['site_id']
-        self.response.write(json.dumps(sessions, default=bson.json_util.default))
+        return sessions
 
     def put(self):
         """Update many Sessions."""
@@ -230,7 +227,7 @@ class Epochs(nimsapiutil.NIMSRequestHandler):
 
     def count(self):
         """Return the number of Epochs."""
-        self.response.write(json.dumps(self.app.db.epochs.count()))
+        self.response.write(self.app.db.epochs.count())
 
     def post(self):
         """Create a new Epoch."""
@@ -247,9 +244,7 @@ class Epochs(nimsapiutil.NIMSRequestHandler):
         elif sid != '':
             self.abort(400, sid + ' is not a valid ObjectId')
         projection = ['name', 'description', 'datatype', 'notes']
-        print query
-        epochs = list(self.app.db.epochs.find(query, projection))
-        self.response.write(json.dumps(epochs, default=bson.json_util.default))
+        return list(self.app.db.epochs.find(query, projection))
 
     def put(self):
         """Update many Epochs."""

experiments.py

 # @author:  Gunnar Schaefer
 
-import json
 import webapp2
 import bson.json_util
 
@@ -51,7 +50,7 @@ class Experiments(nimsapiutil.NIMSRequestHandler):
 
     def count(self):
         """Return the number of Experiments."""
-        self.response.write(json.dumps(self.app.db.experiments.count()))
+        self.response.write(self.app.db.experiments.count())
 
     def post(self):
         """Create a new Experiment."""
@@ -64,7 +63,7 @@ class Experiments(nimsapiutil.NIMSRequestHandler):
         experiments = list(self.app.db.experiments.find(query, projection))
         for exp in experiments:
             exp['site'] = self.app.config['site_id']
-        self.response.write(json.dumps(experiments, default=bson.json_util.default))
+        return experiments
 
     def put(self):
         """Update many Experiments."""
@@ -117,8 +116,7 @@ class Experiment(nimsapiutil.NIMSRequestHandler):
     def get(self, xid):
         """Return one Experiment, conditionally with details."""
         xid = bson.ObjectId(xid)
-        experiment = self.get_experiment(xid)
-        self.response.write(json.dumps(experiment, default=bson.json_util.default))
+        return self.get_experiment(xid)
 
     def put(self, xid):
         """Update an existing Experiment."""
@@ -168,7 +166,7 @@ class Sessions(nimsapiutil.NIMSRequestHandler):
 
     def count(self):
         """Return the number of Sessions."""
-        self.response.write(json.dumps(self.app.db.sessions.count()))
+        self.response.write(self.app.db.sessions.count())
 
     def post(self):
         """Create a new Session"""
@@ -180,8 +178,7 @@ class Sessions(nimsapiutil.NIMSRequestHandler):
         self.get_experiment(xid) # ensure permissions
         query = {'experiment': xid}
         projection = ['name', 'subject', 'notes']
-        sessions = list(self.app.db.sessions.find(query, projection))
-        self.response.write(json.dumps(sessions, default=bson.json_util.default))
+        return list(self.app.db.sessions.find(query, projection))
 
     def put(self):
         """Update many Sessions."""
@@ -225,13 +222,12 @@ class Session(nimsapiutil.NIMSRequestHandler):
         import copy
         json_schema = copy.deepcopy(self.json_schema)
         json_schema['properties'].update(nimsdata.NIMSData.session_properties)
-        self.response.write(json.dumps(json_schema, default=bson.json_util.default))
+        return json_schema
 
     def get(self, sid):
         """Return one Session, conditionally with details."""
         sid = bson.ObjectId(sid)
-        session = self.get_session(sid)
-        self.response.write(json.dumps(session, default=bson.json_util.default))
+        return self.get_session(sid)
 
     def put(self, sid):
         """Update an existing Session."""
@@ -290,7 +286,7 @@ class Epochs(nimsapiutil.NIMSRequestHandler):
 
     def count(self):
         """Return the number of Epochs."""
-        self.response.write(json.dumps(self.app.db.epochs.count()))
+        self.response.write(self.app.db.epochs.count())
 
     def post(self):
         """Create a new Epoch."""
@@ -302,8 +298,7 @@ class Epochs(nimsapiutil.NIMSRequestHandler):
         self.get_session(sid) # ensure permissions
         query = {'session': sid}
         projection = ['name', 'description', 'datatype', 'notes']
-        epochs = list(self.app.db.epochs.find(query, projection))
-        self.response.write(json.dumps(epochs, default=bson.json_util.default))
+        return list(self.app.db.epochs.find(query, projection))
 
     def put(self):
         """Update many Epochs."""
@@ -343,13 +338,12 @@ class Epoch(nimsapiutil.NIMSRequestHandler):
         import copy
         json_schema = copy.deepcopy(self.json_schema)
         json_schema['properties'].update(nimsdata.nimsdicom.NIMSDicom.epoch_properties)
-        self.response.write(json.dumps(json_schema, default=bson.json_util.default))
+        return json_schema
 
     def get(self, eid):
         """Return one Epoch, conditionally with details."""
         eid = bson.ObjectId(eid)
-        epoch = self.get_epoch(eid)
-        self.response.write(json.dumps(epoch, default=bson.json_util.default))
+        return self.get_epoch(eid)
 
     def put(self, eid):
         """Update an existing Epoch."""

internimsclient.py

@@ -18,7 +18,7 @@ logging.getLogger('requests').setLevel(logging.WARNING)
 
 def update(db, api_uri, site_id, privkey, internims_url):
     """sends is-alive signal to internims central."""
-    db.remotes.ensure_index('UTC', expireAfterSeconds=120)
+    db.remotes.ensure_index('timestamp', expireAfterSeconds=120)
 
     exp_userlist = [e['permissions'] for e in db.experiments.find(None, {'_id': False, 'permissions.uid': True})]
     col_userlist = [c['permissions'] for c in db.collections.find(None, {'_id': False, 'permissions.uid': True})]
@@ -34,18 +34,18 @@ def update(db, api_uri, site_id, privkey, internims_url):
         response = (json.loads(r.content))
         # update remotes entries
         for site in response['sites']:
-            site['UTC'] = datetime.datetime.strptime(site['timestamp'], '%Y-%m-%dT%H:%M:%S.%f')
-            db.remotes.find_and_modify({'_id': site['_id']}, update=site, upsert=True)
-            log.debug('upserting remote: ' + site['_id'])
-
-        # update, add remotes to users
-        new_remotes = response['users']
-        log.debug('users w/ remotes: ' + str(new_remotes))
-        for user in response['users']:
-            db.users.update({'uid': user}, {'$set': {'remotes': new_remotes.get(user, [])}})
-
-        # cannot use new_remotes.viewkeys(). leads to 'bson.errors.InvalidDocument: Cannot encode object: dict_keys([])'
-        db.users.update({'remotes': {'$exists':True}, 'uid': {'$nin': new_remotes.keys()}}, {'$unset': {'remotes': ''}}, multi=True)
+            #FIXME
+            site['timestamp'] = datetime.datetime.strptime(site['timestamp'], '%Y-%m-%dT%H:%M:%S.%fZ')
+            db.remotes.update({'_id': site['_id']}, site, upsert=True)
+        log.debug('updating remotes: ' + ', '.join((r['_id'] for r in response['sites'])))
+
+        # delete remotes from users, who no longer have remotes
+        db.users.update({'remotes': {'$exists':True}, 'uid': {'$nin': response['users'].keys()}}, {'$unset': {'remotes': ''}}, multi=True)
+
+        # add remotes to users
+        log.debug('users w/ remotes: ' + ', '.join(response['users']))
+        for uid, remotes in response['users'].iteritems():
+            db.users.update({'uid': uid}, {'$set': {'remotes': remotes}})
     else:
         log.warning((r.status_code, r.reason))
 

nimsapi.py

@@ -43,10 +43,10 @@ class NIMSAPI(nimsapiutil.NIMSRequestHandler):
             nimsapi/remotes                                     | list of remote instances
             [(nimsapi/log)]                                     | list of uwsgi log messages
             [(nimsapi/users)]                                   | list of users
-            nimsapi/users/current                               | details for currently logged in user
             [(nimsapi/users/count)]                             | count of users
             [(nimsapi/users/listschema)]                        | schema for user list
             [(nimsapi/users/schema)]                            | schema for single user
+            nimsapi/users/current                               | details for currently logged-in user
             nimsapi/users/*<uid>*                               | details for user *<uid>*
             [(nimsapi/groups)]                                  | list of groups
             [(nimsapi/groups/count)]                            | count of groups
@@ -130,16 +130,13 @@ class NIMSAPI(nimsapiutil.NIMSRequestHandler):
             symlinks += _idsymlinks
 
     def remotes(self):
-        """Return the list of remotes where user has membership"""
-        remotes = [remote['_id'] for remote in list(self.app.db.remotes.find({}, []))]
-        self.response.write(json.dumps(remotes, default=bson.json_util.default))
+        """Return the list of all remote sites."""
+        return list(self.app.db.remotes.find(None, []))
 
     def log(self):
-        """Return logs"""
-        # TODO: don't hardcode log path.
-        logfile = '/var/log/uwsgi/app/nims.log'
+        """Return logs."""
         try:
-            logs = open(logfile).readlines()
+            logs = open(app.config['log_path']).readlines()
         except IOError as e:
             log.debug(e)
             if 'Permission denied' in e:
@@ -150,20 +147,11 @@ class NIMSAPI(nimsapiutil.NIMSRequestHandler):
             else:
                 # file does not exist
                 self.abort(500, e)
-        else:
-            logs.reverse()
-            numlines = self.request.get('n', None)
-
-            trimmed = []
-            for line in logs:
-                match = re.search('^[\d\s:-]{17}[\s]+nimsapi:[.]*', line)
-                if match:
-                    trimmed.append(line)
-
-            if not numlines: numlines = len(trimmed)
-
-            self.response.headers['Content-Type'] = 'application/json'
-            self.response.write(json.dumps(trimmed[:int(numlines)]))
+        try:
+            n = int(self.request.get('n', 10000))
+        except:
+            self.abort(400, 'n must be an integer')
+        return [line for line in reversed(logs) if re.match('[\d\s:-]{17}[\s]+nimsapi:[.]*', line)][:n]
 
 
 class Users(nimsapiutil.NIMSRequestHandler):
@@ -208,12 +196,7 @@ class Users(nimsapiutil.NIMSRequestHandler):
 
     def count(self):
         """Return the number of Users."""
-        self.response.write('%d users\n' % self.app.db.users.count())
-
-    def current(self):
-        """Return the current User."""
-        # FIXME: trim this down to not use the self.user object, and only send relevant info
-        self.response.write(json.dumps(self.user, default=bson.json_util.default))
+        self.response.write(self.app.db.users.count())
 
     def post(self):
         """Create a new User"""
@@ -221,9 +204,7 @@ class Users(nimsapiutil.NIMSRequestHandler):
 
     def get(self):
         """Return the list of Users."""
-        projection = ['firstname', 'lastname', 'email_hash']
-        users = list(self.app.db.users.find({}, projection))
-        self.response.write(json.dumps(users, default=bson.json_util.default))
+        return list(self.app.db.users.find({}, ['firstname', 'lastname', 'email_hash']))
 
     def put(self):
         """Update many Users."""
@@ -271,14 +252,28 @@ class User(nimsapiutil.NIMSRequestHandler):
         'required': ['_id'],
     }
 
+    def current(self):
+        """Return details for the current User."""
+        if self.request.method == 'GET':
+            return self.get(self.uid)
+        elif self.request.method == 'PUT':
+            return self.put(self.uid)
+
     def get(self, uid):
         """Return User details."""
-        user = self.app.db.users.find_one({'uid': uid})
-        self.response.write(json.dumps(user, default=bson.json_util.default))
+        projection = []
+        if self.request.get('remotes') in ('1', 'true'):
+            projection += ['remotes']
+        if self.request.get('status') in ('1', 'true'):
+            projection += ['status']
+        if self.request.get('login') in ('1', 'true'):
+            projection += ['firstname', 'lastname', 'superuser']
+            self.app.db.users.update({'uid': uid}, {'$inc': {'logins': 1}})
+        return self.app.db.users.find_one({'uid': uid}, projection or None)
 
     def put(self, uid):
         """Update an existing User."""
-        user = self.app.db.users.find_one({'user_info': uid})
+        user = self.app.db.users.find_one({'uid': uid})
         if not user:
             self.abort(404)
         if uid == self.uid or self.user_is_superuser: # users can only update their own info
@@ -293,7 +288,7 @@ class User(nimsapiutil.NIMSRequestHandler):
                         updates['$set'][k] = False # superuser is tri-state: False indicates granted, but disabled, superuser privileges
                     elif v.lower() not in ('1', 'true'):
                         updates['$unset'][k] = ''
-            self.app.db.users.update({'user_info': uid}, updates)
+            self.app.db.users.update({'uid': uid}, updates)
         else:
             self.abort(403)
 
@@ -324,7 +319,7 @@ class Groups(nimsapiutil.NIMSRequestHandler):
 
     def count(self):
         """Return the number of Groups."""
-        self.response.write('%d groups\n' % self.app.db.groups.count())
+        self.response.write(self.app.db.groups.count())
 
     def post(self):
         """Create a new Group"""
@@ -332,9 +327,7 @@ class Groups(nimsapiutil.NIMSRequestHandler):
 
     def get(self):
         """Return the list of Groups."""
-        projection = ['_id']
-        groups = list(self.app.db.groups.find({}, projection))
-        self.response.write(json.dumps(groups, default=bson.json_util.default))
+        return list(self.app.db.groups.find({}, []))
 
     def put(self):
         """Update many Groups."""
@@ -392,8 +385,7 @@ class Group(nimsapiutil.NIMSRequestHandler):
 
     def get(self, gid):
         """Return Group details."""
-        group = self.app.db.groups.find_one({'_id': gid})
-        self.response.write(json.dumps(group, default=bson.json_util.default))
+        return self.app.db.groups.find_one({'_id': gid})
 
     def put(self, gid):
         """Update an existing Group."""
@@ -411,10 +403,10 @@ routes = [
         webapp2.Route(r'/remotes',                                  NIMSAPI, handler_method='remotes', methods=['GET']),
         webapp2.Route(r'/log',                                      NIMSAPI, handler_method='log', methods=['GET']),
         webapp2.Route(r'/users',                                    Users),
-        webapp2.Route(r'/users/current',                            Users, handler_method='current', methods=['GET']),
         webapp2.Route(r'/users/count',                              Users, handler_method='count', methods=['GET']),
         webapp2.Route(r'/users/listschema',                         Users, handler_method='schema', methods=['GET']),
         webapp2.Route(r'/users/schema',                             User, handler_method='schema', methods=['GET']),
+        webapp2.Route(r'/users/current',                            User, handler_method='current', methods=['GET', 'PUT']),
         webapp2.Route(r'/users/<uid>',                              User),
         webapp2.Route(r'/groups',                                   Groups),
         webapp2.Route(r'/groups/count',                             Groups, handler_method='count', methods=['GET']),
@@ -447,8 +439,14 @@ routes = [
     ]),
 ]
 
+def dispatcher(router, request, response):
+    rv = router.default_dispatcher(request, response)
+    if rv is not None:
+        return webapp2.Response(json.dumps(rv, default=bson.json_util.default))
+
 app = webapp2.WSGIApplication(routes, debug=True)
-app.config = dict(stage_path='', site_id=None, ssl_key=None, insecure=False)
+app.router.set_dispatcher(dispatcher)
+app.config = dict(stage_path='', site_id=None, ssl_key=None, insecure=False, log_path='')
 
 
 if __name__ == '__main__':
@@ -462,6 +460,7 @@ if __name__ == '__main__':
     arg_parser.add_argument('config_file', help='path to config file')
     arg_parser.add_argument('--db_uri', help='NIMS DB URI')
     arg_parser.add_argument('--stage_path', help='path to staging area')
+    arg_parser.add_argument('--log_path', help='path to API log file')
     arg_parser.add_argument('--ssl_key', help='path to private SSL key file')
     arg_parser.add_argument('--site_id', help='InterNIMS site ID')
     arg_parser.add_argument('--oauth2_id_endpoint', help='OAuth2 provider ID endpoint')
@@ -485,6 +484,7 @@ if __name__ == '__main__':
 
     app.config['site_id'] = args.site_id or 'local'
     app.config['stage_path'] = args.stage_path or config.get('nims', 'stage_path')
+    app.config['log_path'] = args.log_path
     app.config['oauth2_id_endpoint'] = args.oauth2_id_endpoint or config.get('oauth2', 'id_endpoint')
     app.config['insecure'] = config.getboolean('nims', 'insecure')
 

nimsapi.wsgi

@@ -48,6 +48,7 @@ else:
 site_id = config.get('nims', 'site_id')
 application = nimsapi.app
 application.config['stage_path'] = config.get('nims', 'stage_path')
+application.config['log_path'] = config.get('nims', 'log_path')
 application.config['site_id'] = site_id
 application.config['ssl_key']  = privkey
 application.config['oauth2_id_endpoint'] = config.get('oauth2', 'id_endpoint')

nimsapiutil.py

@@ -102,24 +102,19 @@ class NIMSRequestHandler(webapp2.RequestHandler):
                     log.debug('remote host ' + requester + ' not in auth list. DENIED')
                     self.abort(403, requester + ' is not authorized')
                 log.debug('request from ' + self.request.user_agent + ', interNIMS p2p initiated')
-                # verify signature
-                self.signature = base64.b64decode(self.request.headers.get('X-Signature'))
 
-                # assemble msg to be hased
+                # assemble msg to be hashed and verify signature
+                self.signature = base64.b64decode(self.request.headers.get('X-Signature'))
                 msg = self.request.method + self.request.path + str(dict(self.request.params)) + self.request.body + self.request.headers.get('Date')
                 key = Crypto.PublicKey.RSA.importKey(target['pubkey'])
                 h = Crypto.Hash.SHA.new(msg)
                 verifier = Crypto.Signature.PKCS1_v1_5.new(key)
-                if verifier.verify(h, self.signature):
-                    super(NIMSRequestHandler, self).dispatch()
-                else:
-                    log.debug('message/signature is not authentic')
-                    self.abort(403, 'authentication failed')
-            # request originates from self
-            else:
-                    super(NIMSRequestHandler, self).dispatch()
+                if not verifier.verify(h, self.signature):
+                    log.debug('remote message/signature is not authentic')
+                    self.abort(403, 'remote message/signature is not authentic')
+            return super(NIMSRequestHandler, self).dispatch()
 
-        # dispatch to remote instance
+        # delegate to remote instance
         elif self.ssl_key is not None and self.site_id is not None:
             log.debug('dispatching to remote ' + self.target_id)
             # is target registered?
@@ -134,10 +129,8 @@ class NIMSRequestHandler(webapp2.RequestHandler):
             headers['X-From'] = self.uid
             headers['Content-Length'] = len(self.request.body)
             del headers['Host']                                                 # delete old host destination
-            try:
-                del headers['Authorization']                                    # delete access_token
-            except KeyError:
-                pass                                                            # not all requests will have access_token
+            if 'Authorization' in headers:
+                del headers['Authorization']                                    # delete access_token, if present
 
             # assemble msg to be hashed
             nonce = str(datetime.datetime.now().strftime('%a, %d %b %Y %H:%M:%S'))
@@ -159,8 +152,8 @@ class NIMSRequestHandler(webapp2.RequestHandler):
         elif self.ssl_key is None or self.site_id is None:
             log.debug('ssl key or site id undefined, cannot dispatch to remote')
 
-    def schema(self, *args, **kwargs):
-        self.response.write(json.dumps(self.json_schema, default=bson.json_util.default))
+    def schema(self):
+        return self.json_schema
 
     def get_collection(self, cid, min_role='anon-read'):
         collection = self.app.db.collections.find_one({'_id': cid})