From e613dbc2a6dd9001abebb22ac8c0c51154c89425 Mon Sep 17 00:00:00 2001
From: "Kevin S. Hahn" <kshahn@stanford.edu>
Date: Thu, 23 Jan 2014 14:48:05 -0800
Subject: [PATCH] internims reports user w/ permissions at remotes
- internimsclient separated from nimsapi.wsgi
- args and config have more consistent naming
- using api_url instead of hostname
- improve internims error handling
- bootstrap.py ensures remotes collection index
---
internimsclient.py | 99 ++++++++++++++++++++
nimsapi.py | 37 ++------
nimsapi.wsgi | 227 ++++++++++++---------------------------------
nimsapiutil.py | 9 +-
4 files changed, 170 insertions(+), 202 deletions(-)
create mode 100644 internimsclient.py
diff --git a/internimsclient.py b/internimsclient.py
new file mode 100644
index 00000000..10e03c3a
--- /dev/null
+++ b/internimsclient.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+#
+# @author: Gunnar Schaefer, Kevin S. Hahn
+
+import json
+import base64
+import pymongo
+import datetime
+import requests
+import Crypto.Hash.SHA
+import Crypto.PublicKey.RSA
+import Crypto.Signature.PKCS1_v1_5
+
+import logging
+import logging.config
+log = logging.getLogger('nimsapi')
+logging.getLogger('requests').setLevel(logging.WARNING)
+
+
+def update(db, api_uri, site_id, privkey, internims_url):
+ """sends is-alive signal to internims central."""
+ exp_userlist = [exp['permissions'].viewkeys() for exp in db.experiments.find({}, {'_id': False, 'permissions': True})]
+ col_userlist = [col['permissions'].viewkeys() for col in db.collections.find({}, {'_id': False, 'permissions': True})]
+ userlists = exp_userlist + col_userlist
+ all_users = set([user for experiment in userlists for user in experiment])
+ remote_users = filter(lambda u: '#' in u, all_users)
+
+ payload = json.dumps({'iid': site_id, 'api_uri': api_uri, 'users': remote_users})
+ h = Crypto.Hash.SHA.new(payload)
+ signature = Crypto.Signature.PKCS1_v1_5.new(privkey).sign(h)
+ headers = {'Authorization': base64.b64encode(signature)}
+
+ r = requests.post(url=internims_url, data=payload, headers=headers, verify=True)
+ if r.status_code == 200:
+ response = (json.loads(r.content))
+ # update remotes entries
+ for site in response['sites']:
+ site['UTC'] = datetime.datetime.strptime(site['timestamp'], '%Y-%m-%dT%H:%M:%S.%f')
+ db.remotes.find_and_modify({'_id': site['_id']}, update=site, upsert=True)
+ log.debug('upserting remote: ' + site['_id'])
+
+ # update, add remotes to users
+ new_remotes = response['users']
+ log.debug('users w/ remotes: ' + str(new_remotes))
+ for user in response['users']:
+ db.users.update({'_id': user}, {'$set': {'remotes': new_remotes.get(user, [])}})
+
+ # cannot use new_remotes.viewkeys(). leads to 'bson.errors.InvalidDocument: Cannot encode object: dict_keys([])'
+ db.users.update({'remotes': {'$exists':True}, '_id': {'$nin': new_remotes.keys()}}, {'$unset': {'remotes': ''}}, multi=True)
+ else:
+ log.info((r.status_code, r.reason))
+
+
+if __name__ == '__main__':
+ import os
+ import sys
+ import time
+ import argparse
+ import ConfigParser
+
+ arg_parser = argparse.ArgumentParser()
+ arg_parser.add_argument('configfile', help='path to configuration file')
+ arg_parser.add_argument('--internims_url', help='https://internims.appspot.com')
+ arg_parser.add_argument('--db_uri', help='DB URI')
+ arg_parser.add_argument('--api_uri', help='API URL, without http:// or https://')
+ arg_parser.add_argument('--site_id', help='instance ID')
+ arg_parser.add_argument('--sleeptime', default=60, type=int, help='time to sleep between is alive signals')
+ arg_parser.add_argument('-k', '--ssl_key', help='path to privkey file')
+ args = arg_parser.parse_args()
+
+ config = ConfigParser.ConfigParser({'here': os.path.dirname(os.path.abspath(args.configfile))})
+ config.read(args.configfile)
+
+ logging.config.fileConfig(args.configfile, disable_existing_loggers=False)
+ log = logging.getLogger('nimsapi')
+
+ privkey_file = args.ssl_key or config.get('nims', 'ssl_key')
+ if privkey_file:
+ try:
+ privkey = Crypto.PublicKey.RSA.importKey(open(privkey_file).read())
+ except:
+ log.warn(privkey_file + ' is not a valid private SSL key file, bailing out.')
+ sys.exit(1)
+ else:
+ log.info('successfully loaded private SSL key from ' + privkey_file)
+ else:
+ log.warn('private SSL key not specified, bailing out.')
+ sys.exit(1)
+
+ db_uri = args.db_uri or config.get('nims', 'db_uri')
+ db = (pymongo.MongoReplicaSetClient(db_uri) if 'replicaSet' in db_uri else pymongo.MongoClient(db_uri)).get_default_database()
+
+ site_id = args.site_id or config.get('nims', 'site_id')
+ api_uri = args.api_uri or config.get('nims', 'api_uri')
+ internims_url = args.internims_url or config.get('nims', 'internims_url')
+
+ while True:
+ update(db, api_uri, site_id, privkey, internims_url)
+ time.sleep(args.sleeptime)
diff --git a/nimsapi.py b/nimsapi.py
index 1d2ea574..6d18beb3 100755
--- a/nimsapi.py
+++ b/nimsapi.py
@@ -18,14 +18,13 @@ import Crypto.PublicKey.RSA
import logging
import logging.config
+log = logging.getLogger('nimsapi')
import experiments
import nimsapiutil
import collections_
import tempdir as tempfile
-log = logging.getLogger('nimsapi')
-
class NIMSAPI(nimsapiutil.NIMSRequestHandler):
@@ -129,6 +128,11 @@ class NIMSAPI(nimsapiutil.NIMSRequestHandler):
paths += _idpaths
symlinks += _idsymlinks
+ def remotes(self):
+ """Return the list of remotes where user has membership"""
+ remotes = [remote['_id'] for remote in list(self.app.db.remotes.find({}, []))]
+ self.response.write(json.dumps(remotes))
+
class Users(nimsapiutil.NIMSRequestHandler):
@@ -362,38 +366,12 @@ class Group(nimsapiutil.NIMSRequestHandler):
"""Delete an Group."""
-class Remotes(nimsapiutil.NIMSRequestHandler):
-
- """/nimsapi/remotes """
-
- def get(self):
- """Return the list of remotes where user has membership"""
- # TODO: implement special 'all' case - report ALL available instances, regardless of user permissions
- # applies to adding new remote users, need to be able to select from ALL available remote sites
- # query, user in userlist, _id does not match this site _id
- query = {'users': {'$in': [self.user['_id']]}, '_id': {'$ne': self.app.config['site_id']}}
- projection = ['_id']
- # if app has no site-id or pubkey, cannot fetch peer registry, and db.remotes will be empty
- remotes = list(self.app.db.remotes.find(query, projection))
- data_remotes = [] # for list buildup
- for remote in remotes:
- # use own API to dispatch requests (hacky)
- response = self.app.get_response('/nimsapi/experiments?user=' + self.user['_id'] + '&iid=' + remote['_id'], headers=[('User-Agent', 'remotes_requestor')])
- xpcount = len(json.loads(response.body))
- if xpcount > 0:
- log.debug('%s has access to %s expirements on %s' % (self.user['_id'], xpcount, remote['_id']))
- data_remotes.append(remote['_id'])
-
- # return json encoded list of remote site '_id's
- self.response.write(json.dumps(data_remotes, indent=4, separators=(',', ': ')))
-
-
routes = [
webapp2.Route(r'/nimsapi', NIMSAPI),
webapp2_extras.routes.PathPrefixRoute(r'/nimsapi', [
webapp2.Route(r'/download', NIMSAPI, handler_method='download', methods=['GET']),
webapp2.Route(r'/upload', NIMSAPI, handler_method='upload', methods=['PUT']),
- webapp2.Route(r'/remotes', Remotes),
+ webapp2.Route(r'/remotes', NIMSAPI, handler_method='remotes', methods=['GET']),
webapp2.Route(r'/users', Users),
webapp2.Route(r'/users/count', Users, handler_method='count', methods=['GET']),
webapp2.Route(r'/users/listschema', Users, handler_method='schema', methods=['GET']),
@@ -450,6 +428,7 @@ if __name__ == '__main__':
config = ConfigParser.ConfigParser({'here': os.path.dirname(os.path.abspath(args.config_file))})
config.read(args.config_file)
+
logging.config.fileConfig(args.config_file, disable_existing_loggers=False)
if args.ssl_key:
diff --git a/nimsapi.wsgi b/nimsapi.wsgi
index ba24f7f6..6915565d 100644
--- a/nimsapi.wsgi
+++ b/nimsapi.wsgi
@@ -1,177 +1,68 @@
-#!/usr/bin/env python
-#
# @author: Gunnar Schaefer, Kevin S. Hahn
import os
import sys
import site
import ConfigParser
-import logging
-import logging.config
-
-
-def apply_config(configfile):
- """Return a ConfigParser object"""
- config = ConfigParser.ConfigParser()
- config.read(configfile)
- site.addsitedir(os.path.join(config.get('nims', 'virtualenv'), 'lib', 'python2.7', 'site-packages'))
- sys.path.append(config.get('nims', 'here'))
- os.environ['PYTHON_EGG_CACHE'] = config.get('nims', 'python_egg_cache')
- return config
-
-
-def configure_logger(configfile):
- """return a nimsapi configured logger"""
- logging.config.fileConfig(configfile, disable_existing_loggers=False)
- return logging.getLogger('nimsapi')
-
-
-def read_privkey(privkey_file):
- """reads SSL private key. returns key content as RSA.key object"""
- try:
- privkey = Crypto.PublicKey.RSA.importKey(open(privkey_file).read())
- except:
- log.warn(privkey_file + ' is not a valid private SSL key file')
- privkey = None
- else:
- log.info('successfully loaded private SSL key from ' + privkey_file)
- return privkey
-
-
-def connect_db(db_uri):
- """return mongodb default database"""
- kwargs = dict(tz_aware=True)
- db_client = pymongo.MongoReplicaSetClient(db_uri, **kwargs) if 'replicaSet' in db_uri else pymongo.MongoClient(db_uri, **kwargs)
- db = db_client.get_default_database()
- db.remotes.ensure_index('UTC', expireAfterSeconds=120)
- return db
-
-
-def internimsclient(db, hostname, site_id, privkey, internims_url):
- """sends is alive to internims central. no return"""
- # TODO: create a list of non-local users, who have permissions on experiments
- users = [users['_id'] for users in list(db.users.find({}, {'_id': True}))]
-
- payload = json.dumps({'iid': site_id, 'hostname': hostname, 'users': users})
- h = Crypto.Hash.SHA.new(payload)
- signature = Crypto.Signature.PKCS1_v1_5.new(privkey).sign(h)
- headers = {'Authorization': base64.b64encode(signature)}
-
- r = requests.post(url=internims_url, data=payload, headers=headers, verify=True)
- if r.status_code == 200:
- sites = json.loads(r.content)
- for site in sites:
- site['UTC'] = datetime.datetime.strptime(site['timestamp'], '%Y-%m-%dT%H:%M:%S.%f')
- db.remotes.find_and_modify(query={'_id': site['_id']}, update=site, upsert=True)
- log.debug('upserting remote site ' + site['_id'])
- else:
- log.info((r.status_code, r.reason))
-
-
-if __name__ == '__main__':
- import argparse
-
- arg_parser = argparse.ArgumentParser()
- arg_parser.add_argument('-c', '--configfile', help='path to configuration file')
- arg_parser.add_argument('--internims_url', help='https://internims.appspot.com')
- arg_parser.add_argument('--db_uri', help='DB URI')
- arg_parser.add_argument('--hostname', help='fqdn, without protocol')
- arg_parser.add_argument('--site_id', help='instance ID')
- arg_parser.add_argument('--sleeptime', default=60, type=int, help='time to sleep between is alive signals')
- arg_parser.add_argument('-k', '--privkey', help='path to privkey file')
- args = arg_parser.parse_args()
-
- # read config
- config = apply_config(args.configfile)
- # import everything else
- import json
- import time
- import base64
- import pymongo
- import webapp2
- import nimsapi
- import nimsutil
- import requests
- import datetime
- import Crypto.Random
- import Crypto.Hash.SHA
- import Crypto.PublicKey.RSA
- import Crypto.Signature.PKCS1_v1_5
- import signal # not in uwsgi execution
-
- # configure logger
- log = configure_logger(args.configfile)
-
- # args override configfile
- db_uri = args.db_uri or config.get('nims', 'db_uri')
- hostname = args.hostname or config.get('nims', 'hostname')
- site_id = args.site_id or config.get('nims', 'site_id')
- internims_url = args.internims_url or config.get('nims', 'internims_url')
- privkey_file = args.privkey or config.get('nims', 'privkey_file')
-
- # load in the privkey
- privkey = read_privkey(privkey_file)
-
- # connect to db
- db = connect_db(db_uri)
-
- def term_handler(signum, stack):
- alive = False
- log.debug('Recieved SIGTERM - shuttin down')
-
- signal.signal(signal.SIGTERM, term_handler)
-
- alive = True
- while alive:
- internimsclient(db, hostname, site_id, privkey, internims_url)
- time.sleep(args.sleeptime)
+configfile = '../production.ini'
+config = ConfigParser.ConfigParser()
+config.read(configfile)
+
+site.addsitedir(os.path.join(config.get('nims', 'virtualenv'), 'lib', 'python2.7', 'site-packages'))
+sys.path.append(config.get('nims', 'here'))
+os.environ['PYTHON_EGG_CACHE'] = config.get('nims', 'python_egg_cache')
+
+import json
+import time
+import base64
+import pymongo
+import webapp2
+import datetime
+import requests
+import Crypto.Random
+import Crypto.Hash.SHA
+import uwsgidecorators
+import Crypto.PublicKey.RSA
+import Crypto.Signature.PKCS1_v1_5
+import logging
+import logging.config
+logging.config.fileConfig(configfile, disable_existing_loggers=False)
+log = logging.getLogger('nimsapi')
+
+import nimsapi
+import internimsclient
+
+# read in private key
+privkey_file = config.get('nims', 'ssl_key')
+try:
+ privkey = Crypto.PublicKey.RSA.importKey(open(privkey_file).read())
+except:
+ log.warn(privkey_file + 'is not a valid private SSL key file')
+ privkey = None
else:
- # read config
- configfile = '../production.ini'
- config = apply_config(configfile)
-
- # import everything else
- import json
- import time
- import base64
- import pymongo
- import webapp2
- import nimsapi
- import argparse
- import datetime
- import nimsutil
- import requests
- import Crypto.Random
- import Crypto.Hash.SHA
- import Crypto.PublicKey.RSA
- import Crypto.Signature.PKCS1_v1_5
- import uwsgidecorators # only in uwsgi execution
-
- # configure logger
- log = configure_logger(configfile)
-
- db_uri = config.get('nims', 'db_uri')
- stage_path = config.get('nims', 'stage_path')
- site_id = config.get('nims', 'site_id')
-
- # load in privkey
- privkey = read_privkey(config.get('nims', 'privkey_file'))
-
- # config uwsgi application
- application = nimsapi.app
- application.config['stage_path'] = stage_path
- application.config['site_id'] = site_id
- application.config['privkey'] = privkey
-
- # connect db
- application.db = connect_db(db_uri)
-
- @uwsgidecorators.postfork
- def random_atfork():
- Crypto.Random.atfork()
-
- @uwsgidecorators.timer(60)
- def internimsclient_timer(signum):
- internimsclient(application.db, config.get('nims', 'hostname'), config.get('nims', 'site_id'), privkey, config.get('nims', 'internims_url'))
+ log.info('successfully loaded private SSL key from ' + privkey_file)
+
+# configure uwsgi application
+site_id = config.get('nims', 'site_id')
+application = nimsapi.app
+application.config['stage_path'] = config.get('nims', 'stage_path')
+application.config['site_id'] = site_id
+application.config['ssl_key'] = privkey
+
+# connect to db
+db_uri = config.get('nims', 'db_uri')
+application.db = (pymongo.MongoReplicaSetClient(db_uri) if 'replicaSet' in db_uri else pymongo.MongoClient(db_uri)).get_default_database()
+
+# send is-alive signals
+api_uri = config.get('nims', 'api_uri')
+internims_url = config.get('nims', 'internims_url')
+
+@uwsgidecorators.timer(60)
+def internimsclient_timer(signum):
+ internimsclient.update(application.db, api_uri, site_id, privkey, internims_url)
+
+@uwsgidecorators.postfork
+def random_atfork():
+ Crypto.Random.atfork()
diff --git a/nimsapiutil.py b/nimsapiutil.py
index 76f737c4..ab6044c8 100644
--- a/nimsapiutil.py
+++ b/nimsapiutil.py
@@ -2,8 +2,7 @@
import json
import base64
-import socket # socket.gethostname()
-import logging
+import socket
import webapp2
import datetime
import requests
@@ -12,6 +11,7 @@ import Crypto.Hash.SHA
import Crypto.PublicKey.RSA
import Crypto.Signature.PKCS1_v1_5
+import logging
log = logging.getLogger('nimsapi')
logging.getLogger('requests').setLevel(logging.WARNING) # silence Requests library logging
@@ -93,7 +93,7 @@ class NIMSRequestHandler(webapp2.RequestHandler):
elif self.ssl_key is not None and self.site_id is not None:
log.debug(socket.gethostname() + ' dispatching to remote ' + self.target_id)
# is target registered?
- target = self.app.db.remotes.find_one({'_id': self.target_id}, {'_id':False, 'hostname':True})
+ target = self.app.db.remotes.find_one({'_id': self.target_id}, {'_id':False, 'api_uri':True})
if not target:
log.debug('remote host ' + self.target_id + ' not in auth list. DENIED')
self.abort(403, self.target_id + 'is not authorized')
@@ -111,8 +111,7 @@ class NIMSRequestHandler(webapp2.RequestHandler):
reqheaders['Authorization'] = base64.b64encode(signature)
# construct outgoing request
- target_api = 'http://' + target['hostname'] + self.request.path # TODO: switch to https
- # target_api = 'https://' + target['hostname'] + self.request.path)
+ target_api = 'https://' + target['api_uri'] + self.request.path.split('/nimsapi')[1]
r = requests.request(method=self.request.method, data=reqpayload, url=target_api, params=reqparams, headers=reqheaders, verify=False)
# return response content
--
GitLab