internims sends/receives site id and name

internimsclient.py

@@ -17,15 +17,14 @@ import Crypto.PublicKey.RSA
 import Crypto.Signature.PKCS1_v1_5
 
 
-def update(db, api_uri, site_id, privkey, internims_url):
+def update(db, api_uri, site_name, site_id, privkey, internims_url):
     """sends is-alive signal to internims central."""
-    db.remotes.ensure_index('timestamp', expireAfterSeconds=120)
-
     exp_userlist = [e['permissions'] for e in db.experiments.find(None, {'_id': False, 'permissions.uid': True})]
     col_userlist = [c['permissions'] for c in db.collections.find(None, {'_id': False, 'permissions.uid': True})]
-    remote_users = list(set([user['uid'] for container in exp_userlist+col_userlist for user in container if '#' in user['uid']]))
+    grp_userlist = [g['roles'] for g in db.groups.find(None, {'_id': False, 'roles.uid': True})]
+    remote_users = list(set([user['uid'] for container in exp_userlist+col_userlist+grp_userlist for user in container if '#' in user['uid']]))
 
-    payload = json.dumps({'iid': site_id, 'api_uri': api_uri, 'users': remote_users})
+    payload = json.dumps({'site': site_id, 'api_uri': api_uri, 'users': remote_users, 'name': site_name})
     h = Crypto.Hash.SHA.new(payload)
     signature = Crypto.Signature.PKCS1_v1_5.new(privkey).sign(h)
     headers = {'Authorization': base64.b64encode(signature)}
@@ -40,17 +39,26 @@ def update(db, api_uri, site_id, privkey, internims_url):
         log.debug('updating remotes: ' + ', '.join((r['_id'] for r in response['sites'])))
 
         # delete remotes from users, who no longer have remotes
-        db.users.update({'remotes': {'$exists':True}, '_id': {'$nin': response['users'].keys()}}, {'$unset': {'remotes': ''}}, multi=True)
+        db.users.update({'remotes': {'$exists': True}, '_id': {'$nin': response['users'].keys()}}, {'$unset': {'remotes': ''}}, multi=True)
 
         # add remotes to users
         log.debug('users w/ remotes: ' + ', '.join(response['users']))
         for uid, remotes in response['users'].iteritems():
             db.users.update({'_id': uid}, {'$set': {'remotes': remotes}})
+        return True
     else:
         # r.reason contains generic description for the specific error code
         # need the part of the error response body that contains the detailed explanation
         reason = re.search('<br /><br />\n(.*)\n\n\n </body>\n</html>', r.content)
         log.warning((r.status_code, reason.group(1)))
+        return False
+
+
+def clean_remotes(db):
+    """removes db.remotes, and removes remotes field from all db.users"""
+    log.debug('removing remotes from users, and remotes collection')
+    db.remotes.remove({})
+    db.users.update({'remotes': {'$exists': True}}, {'$unset': {'remotes': ''}}, multi=True)
 
 
 if __name__ == '__main__':
@@ -67,6 +75,7 @@ if __name__ == '__main__':
     arg_parser.add_argument('--db_uri', help='DB URI')
     arg_parser.add_argument('--api_uri', help='API URL, without http:// or https://')
     arg_parser.add_argument('--site_id', help='instance ID')
+    arg_parser.add_argument('--site_name', help='instance name')
     arg_parser.add_argument('--sleeptime', default=60, type=int, help='time to sleep between is alive signals')
     arg_parser.add_argument('--ssl_key', help='path to privkey file')
     args = arg_parser.parse_args()
@@ -79,7 +88,7 @@ if __name__ == '__main__':
     if privkey_file:
         try:
             privkey = Crypto.PublicKey.RSA.importKey(open(privkey_file).read())
-        except:
+        except Exception:
             log.warn(privkey_file + ' is not a valid private SSL key file, bailing out.')
             sys.exit(1)
         else:
@@ -91,10 +100,19 @@ if __name__ == '__main__':
     db_uri = args.db_uri or config.get('nims', 'db_uri')
     db = (pymongo.MongoReplicaSetClient(db_uri) if 'replicaSet' in db_uri else pymongo.MongoClient(db_uri)).get_default_database()
 
+    site_name = args.site_name or config.get('nims', 'site_name')
     site_id = args.site_id or config.get('nims', 'site_id')
     api_uri = args.api_uri or config.get('nims', 'api_uri')
     internims_url = args.internims_url or config.get('nims', 'internims_url')
 
+    fail_count = 0
+
     while True:
-        update(db, api_uri, site_id, privkey, internims_url)
+        if not update(db, api_uri, site_name, site_id, privkey, internims_url):
+            fail_count += 1
+        else:
+            fail_count = 0
+        if fail_count == 3:
+            log.debug('InterNIMS unreachable, purging all remotes info')
+            clean_remotes(db)
         time.sleep(args.sleeptime)

nimsapi.py

@@ -167,7 +167,7 @@ class NIMSAPI(nimsapiutil.NIMSRequestHandler):
         except IOError as e:
             if 'Permission denied' in e:
                 body_template = '${explanation}<br /><br />${detail}<br /><br />${comment}'
-                comment = 'To fix permissions, run the following command: chmod o+r ' + logfile
+                comment = 'To fix permissions, run the following command: chmod o+r ' + app.config['log_path']
                 self.abort(500, detail=str(e), comment=comment, body_template=body_template)
             else:
                 self.abort(500, e) # file does not exist
@@ -240,7 +240,7 @@ def dispatcher(router, request, response):
 
 app = webapp2.WSGIApplication(routes, debug=True)
 app.router.set_dispatcher(dispatcher)
-app.config = dict(stage_path='', site_id='local', ssl_key=None, insecure=False, log_path='')
+app.config = dict(stage_path='', site_id='local', site_name='Local', ssl_key=None, insecure=False, log_path='')
 
 
 if __name__ == '__main__':
@@ -257,6 +257,7 @@ if __name__ == '__main__':
     arg_parser.add_argument('--log_path', help='path to API log file')
     arg_parser.add_argument('--ssl_key', help='path to private SSL key file')
     arg_parser.add_argument('--site_id', help='InterNIMS site ID')
+    arg_parser.add_argument('--site_name', help='InterNIMS site name')
     arg_parser.add_argument('--oauth2_id_endpoint', help='OAuth2 provider ID endpoint')
     args = arg_parser.parse_args()
 
@@ -275,9 +276,10 @@ if __name__ == '__main__':
             log.debug('successfully loaded private SSL key from ' + args.ssl_key)
             app.config['ssl_key'] = ssl_key
     else:
-        log.warning('private SSL key not specified, internims functionality disabled')
+        log.warning('private SSL key not specified, InterNIMS functionality disabled')
 
     app.config['site_id'] = args.site_id or app.config['site_id']
+    app.config['site_name'] = args.site_name or app.config['site_name']
     app.config['stage_path'] = args.stage_path or config.get('nims', 'stage_path')
     app.config['log_path'] = args.log_path or app.config['log_path']
     app.config['oauth2_id_endpoint'] = args.oauth2_id_endpoint or config.get('oauth2', 'id_endpoint')

nimsapi.wsgi

@@ -13,13 +13,7 @@ site.addsitedir(os.path.join(config.get('nims', 'virtualenv'), 'lib', 'python2.7
 sys.path.append(config.get('nims', 'here'))
 os.environ['PYTHON_EGG_CACHE'] = config.get('nims', 'python_egg_cache')
 
-import json
-import time
-import base64
 import pymongo
-import webapp2
-import datetime
-import requests
 import Crypto.Random
 import Crypto.Hash.SHA
 import uwsgidecorators
@@ -38,7 +32,7 @@ import internimsclient
 privkey_file = config.get('nims', 'ssl_key')
 try:
     privkey = Crypto.PublicKey.RSA.importKey(open(privkey_file).read())
-except:
+except Exception:
     log.warn(privkey_file + 'is not a valid private SSL key file')
     privkey = None
 else:
@@ -46,11 +40,13 @@ else:
 
 # configure uwsgi application
 site_id = config.get('nims', 'site_id')
+site_name = config.get('nims', 'site_name')
 application = nimsapi.app
 application.config['stage_path'] = config.get('nims', 'stage_path')
 application.config['log_path'] = config.get('nims', 'log_path')
+application.config['site_name'] = site_name
 application.config['site_id'] = site_id
-application.config['ssl_key']  = privkey
+application.config['ssl_key'] = privkey
 application.config['oauth2_id_endpoint'] = config.get('oauth2', 'id_endpoint')
 application.config['insecure'] = config.getboolean('nims', 'insecure')
 
@@ -62,9 +58,20 @@ application.db = (pymongo.MongoReplicaSetClient(db_uri) if 'replicaSet' in db_ur
 api_uri = config.get('nims', 'api_uri')
 internims_url = config.get('nims', 'internims_url')
 
+fail_count = 0
+
 @uwsgidecorators.timer(60)
 def internimsclient_timer(signum):
-    internimsclient.update(application.db, api_uri, site_id, privkey, internims_url)
+        global fail_count
+        if not internimsclient.update(application.db, api_uri, site_name, site_id, privkey, internims_url):
+            fail_count += 1
+        else:
+            fail_count = 0
+
+        if fail_count == 3:
+            log.debug('InterNIMS unreachable, purging all remotes info')
+            internimsclient.clean_remotes(application.db)
+
 
 @uwsgidecorators.postfork
 def random_atfork():

nimsapiutil.py

@@ -57,7 +57,7 @@ class NIMSRequestHandler(webapp2.RequestHandler):
 
     def __init__(self, request=None, response=None):
         self.initialize(request, response)
-        self.target_id = self.request.get('iid', None)
+        self.target_id = self.request.get('site', None)
         self.access_token = self.request.headers.get('Authorization', None)
 
         # CORS header
@@ -109,7 +109,7 @@ class NIMSRequestHandler(webapp2.RequestHandler):
             # adjust params
             self.params = self.request.params.mixed()
             if self.params.get('user'): del self.params['user']
-            del self.params['iid']
+            del self.params['site']
 
             # assemble msg, hash, and signature
             msg = self.request.method + self.request.path + str(self.params) + self.request.body + self.headers.get('Date')
@@ -149,7 +149,7 @@ class NIMSRequestHandler(webapp2.RequestHandler):
                 return self.options()
             r = requests.request(self.request.method, self.target_uri, params=self.params, data=self.request.body, headers=self.headers, verify=False)
             if r.status_code != 200:
-                self.abort(r.status_code, 'internims p2p err: ' + r.reason)
+                self.abort(r.status_code, 'InterNIMS p2p err: ' + r.reason)
             self.response.write(r.content)
 
     def abort(self, code, *args, **kwargs):