Skip to content
Snippets Groups Projects
Commit e613dbc2 authored by Kevin S. Hahn's avatar Kevin S. Hahn
Browse files

internims reports user w/ permissions at remotes 

- internimsclient separated from nimsapi.wsgi
- args and config have more consistent naming
- using api_url instead of hostname
- improve internims error handling
- bootstrap.py ensures remotes collection index
parent 173b5320
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
internimsclient.py 0 → 100644
+ 99
− 0
View file @ e613dbc2
#!/usr/bin/env python
#
# @author: Gunnar Schaefer, Kevin S. Hahn
import json
import base64
import pymongo
import datetime
import requests
import Crypto.Hash.SHA
import Crypto.PublicKey.RSA
import Crypto.Signature.PKCS1_v1_5
import logging
import logging.config
log = logging.getLogger('nimsapi')
logging.getLogger('requests').setLevel(logging.WARNING)
def update(db, api_uri, site_id, privkey, internims_url):
"""sends is-alive signal to internims central."""
exp_userlist = [exp['permissions'].viewkeys() for exp in db.experiments.find({}, {'_id': False, 'permissions': True})]
col_userlist = [col['permissions'].viewkeys() for col in db.collections.find({}, {'_id': False, 'permissions': True})]
userlists = exp_userlist + col_userlist
all_users = set([user for experiment in userlists for user in experiment])
remote_users = filter(lambda u: '#' in u, all_users)
payload = json.dumps({'iid': site_id, 'api_uri': api_uri, 'users': remote_users})
h = Crypto.Hash.SHA.new(payload)
signature = Crypto.Signature.PKCS1_v1_5.new(privkey).sign(h)
headers = {'Authorization': base64.b64encode(signature)}
r = requests.post(url=internims_url, data=payload, headers=headers, verify=True)
if r.status_code == 200:
response = (json.loads(r.content))
# update remotes entries
for site in response['sites']:
site['UTC'] = datetime.datetime.strptime(site['timestamp'], '%Y-%m-%dT%H:%M:%S.%f')
db.remotes.find_and_modify({'_id': site['_id']}, update=site, upsert=True)
log.debug('upserting remote: ' + site['_id'])
# update, add remotes to users
new_remotes = response['users']
log.debug('users w/ remotes: ' + str(new_remotes))
for user in response['users']:
db.users.update({'_id': user}, {'$set': {'remotes': new_remotes.get(user, [])}})
# cannot use new_remotes.viewkeys(). leads to 'bson.errors.InvalidDocument: Cannot encode object: dict_keys([])'
db.users.update({'remotes': {'$exists':True}, '_id': {'$nin': new_remotes.keys()}}, {'$unset': {'remotes': ''}}, multi=True)
else:
log.info((r.status_code, r.reason))
if __name__ == '__main__':
import os
import sys
import time
import argparse
import ConfigParser
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('configfile', help='path to configuration file')
arg_parser.add_argument('--internims_url', help='https://internims.appspot.com')
arg_parser.add_argument('--db_uri', help='DB URI')
arg_parser.add_argument('--api_uri', help='API URL, without http:// or https://')
arg_parser.add_argument('--site_id', help='instance ID')
arg_parser.add_argument('--sleeptime', default=60, type=int, help='time to sleep between is alive signals')
arg_parser.add_argument('-k', '--ssl_key', help='path to privkey file')
args = arg_parser.parse_args()
config = ConfigParser.ConfigParser({'here': os.path.dirname(os.path.abspath(args.configfile))})
config.read(args.configfile)
logging.config.fileConfig(args.configfile, disable_existing_loggers=False)
log = logging.getLogger('nimsapi')
privkey_file = args.ssl_key or config.get('nims', 'ssl_key')
if privkey_file:
try:
privkey = Crypto.PublicKey.RSA.importKey(open(privkey_file).read())
except:
log.warn(privkey_file + ' is not a valid private SSL key file, bailing out.')
sys.exit(1)
else:
log.info('successfully loaded private SSL key from ' + privkey_file)
else:
log.warn('private SSL key not specified, bailing out.')
sys.exit(1)
db_uri = args.db_uri or config.get('nims', 'db_uri')
db = (pymongo.MongoReplicaSetClient(db_uri) if 'replicaSet' in db_uri else pymongo.MongoClient(db_uri)).get_default_database()
site_id = args.site_id or config.get('nims', 'site_id')
api_uri = args.api_uri or config.get('nims', 'api_uri')
internims_url = args.internims_url or config.get('nims', 'internims_url')
while True:
update(db, api_uri, site_id, privkey, internims_url)
time.sleep(args.sleeptime)
nimsapi.py
+ 8
− 29
View file @ e613dbc2
......@@ -18,14 +18,13 @@ import Crypto.PublicKey.RSA
import logging
import logging.config
log = logging.getLogger('nimsapi')
import experiments
import nimsapiutil
import collections_
import tempdir as tempfile
log = logging.getLogger('nimsapi')
class NIMSAPI(nimsapiutil.NIMSRequestHandler):
......@@ -129,6 +128,11 @@ class NIMSAPI(nimsapiutil.NIMSRequestHandler):
paths += _idpaths
symlinks += _idsymlinks
def remotes(self):
"""Return the list of remotes where user has membership"""
remotes = [remote['_id'] for remote in list(self.app.db.remotes.find({}, []))]
self.response.write(json.dumps(remotes))
class Users(nimsapiutil.NIMSRequestHandler):
......@@ -362,38 +366,12 @@ class Group(nimsapiutil.NIMSRequestHandler):
"""Delete an Group."""
class Remotes(nimsapiutil.NIMSRequestHandler):
"""/nimsapi/remotes """
def get(self):
"""Return the list of remotes where user has membership"""
# TODO: implement special 'all' case - report ALL available instances, regardless of user permissions
# applies to adding new remote users, need to be able to select from ALL available remote sites
# query, user in userlist, _id does not match this site _id
query = {'users': {'$in': [self.user['_id']]}, '_id': {'$ne': self.app.config['site_id']}}
projection = ['_id']
# if app has no site-id or pubkey, cannot fetch peer registry, and db.remotes will be empty
remotes = list(self.app.db.remotes.find(query, projection))
data_remotes = [] # for list buildup
for remote in remotes:
# use own API to dispatch requests (hacky)
response = self.app.get_response('/nimsapi/experiments?user=' + self.user['_id'] + '&iid=' + remote['_id'], headers=[('User-Agent', 'remotes_requestor')])
xpcount = len(json.loads(response.body))
if xpcount > 0:
log.debug('%s has access to %s expirements on %s' % (self.user['_id'], xpcount, remote['_id']))
data_remotes.append(remote['_id'])
# return json encoded list of remote site '_id's
self.response.write(json.dumps(data_remotes, indent=4, separators=(',', ': ')))
routes = [
webapp2.Route(r'/nimsapi', NIMSAPI),
webapp2_extras.routes.PathPrefixRoute(r'/nimsapi', [
webapp2.Route(r'/download', NIMSAPI, handler_method='download', methods=['GET']),
webapp2.Route(r'/upload', NIMSAPI, handler_method='upload', methods=['PUT']),
webapp2.Route(r'/remotes', Remotes),
webapp2.Route(r'/remotes', NIMSAPI, handler_method='remotes', methods=['GET']),
webapp2.Route(r'/users', Users),
webapp2.Route(r'/users/count', Users, handler_method='count', methods=['GET']),
webapp2.Route(r'/users/listschema', Users, handler_method='schema', methods=['GET']),
......@@ -450,6 +428,7 @@ if __name__ == '__main__':
config = ConfigParser.ConfigParser({'here': os.path.dirname(os.path.abspath(args.config_file))})
config.read(args.config_file)
logging.config.fileConfig(args.config_file, disable_existing_loggers=False)
if args.ssl_key:
......
nimsapi.wsgi
+ 59
− 168
View file @ e613dbc2
#!/usr/bin/env python
#
# @author: Gunnar Schaefer, Kevin S. Hahn
import os
import sys
import site
import ConfigParser
import logging
import logging.config
def apply_config(configfile):
"""Return a ConfigParser object"""
config = ConfigParser.ConfigParser()
config.read(configfile)
site.addsitedir(os.path.join(config.get('nims', 'virtualenv'), 'lib', 'python2.7', 'site-packages'))
sys.path.append(config.get('nims', 'here'))
os.environ['PYTHON_EGG_CACHE'] = config.get('nims', 'python_egg_cache')
return config
def configure_logger(configfile):
"""return a nimsapi configured logger"""
logging.config.fileConfig(configfile, disable_existing_loggers=False)
return logging.getLogger('nimsapi')
def read_privkey(privkey_file):
"""reads SSL private key. returns key content as RSA.key object"""
try:
privkey = Crypto.PublicKey.RSA.importKey(open(privkey_file).read())
except:
log.warn(privkey_file + ' is not a valid private SSL key file')
privkey = None
else:
log.info('successfully loaded private SSL key from ' + privkey_file)
return privkey
def connect_db(db_uri):
"""return mongodb default database"""
kwargs = dict(tz_aware=True)
db_client = pymongo.MongoReplicaSetClient(db_uri, **kwargs) if 'replicaSet' in db_uri else pymongo.MongoClient(db_uri, **kwargs)
db = db_client.get_default_database()
db.remotes.ensure_index('UTC', expireAfterSeconds=120)
return db
def internimsclient(db, hostname, site_id, privkey, internims_url):
"""sends is alive to internims central. no return"""
# TODO: create a list of non-local users, who have permissions on experiments
users = [users['_id'] for users in list(db.users.find({}, {'_id': True}))]
payload = json.dumps({'iid': site_id, 'hostname': hostname, 'users': users})
h = Crypto.Hash.SHA.new(payload)
signature = Crypto.Signature.PKCS1_v1_5.new(privkey).sign(h)
headers = {'Authorization': base64.b64encode(signature)}
r = requests.post(url=internims_url, data=payload, headers=headers, verify=True)
if r.status_code == 200:
sites = json.loads(r.content)
for site in sites:
site['UTC'] = datetime.datetime.strptime(site['timestamp'], '%Y-%m-%dT%H:%M:%S.%f')
db.remotes.find_and_modify(query={'_id': site['_id']}, update=site, upsert=True)
log.debug('upserting remote site ' + site['_id'])
else:
log.info((r.status_code, r.reason))
if __name__ == '__main__':
import argparse
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('-c', '--configfile', help='path to configuration file')
arg_parser.add_argument('--internims_url', help='https://internims.appspot.com')
arg_parser.add_argument('--db_uri', help='DB URI')
arg_parser.add_argument('--hostname', help='fqdn, without protocol')
arg_parser.add_argument('--site_id', help='instance ID')
arg_parser.add_argument('--sleeptime', default=60, type=int, help='time to sleep between is alive signals')
arg_parser.add_argument('-k', '--privkey', help='path to privkey file')
args = arg_parser.parse_args()
# read config
config = apply_config(args.configfile)
# import everything else
import json
import time
import base64
import pymongo
import webapp2
import nimsapi
import nimsutil
import requests
import datetime
import Crypto.Random
import Crypto.Hash.SHA
import Crypto.PublicKey.RSA
import Crypto.Signature.PKCS1_v1_5
import signal # not in uwsgi execution
# configure logger
log = configure_logger(args.configfile)
# args override configfile
db_uri = args.db_uri or config.get('nims', 'db_uri')
hostname = args.hostname or config.get('nims', 'hostname')
site_id = args.site_id or config.get('nims', 'site_id')
internims_url = args.internims_url or config.get('nims', 'internims_url')
privkey_file = args.privkey or config.get('nims', 'privkey_file')
# load in the privkey
privkey = read_privkey(privkey_file)
# connect to db
db = connect_db(db_uri)
def term_handler(signum, stack):
alive = False
log.debug('Recieved SIGTERM - shuttin down')
signal.signal(signal.SIGTERM, term_handler)
alive = True
while alive:
internimsclient(db, hostname, site_id, privkey, internims_url)
time.sleep(args.sleeptime)
configfile = '../production.ini'
config = ConfigParser.ConfigParser()
config.read(configfile)
site.addsitedir(os.path.join(config.get('nims', 'virtualenv'), 'lib', 'python2.7', 'site-packages'))
sys.path.append(config.get('nims', 'here'))
os.environ['PYTHON_EGG_CACHE'] = config.get('nims', 'python_egg_cache')
import json
import time
import base64
import pymongo
import webapp2
import datetime
import requests
import Crypto.Random
import Crypto.Hash.SHA
import uwsgidecorators
import Crypto.PublicKey.RSA
import Crypto.Signature.PKCS1_v1_5
import logging
import logging.config
logging.config.fileConfig(configfile, disable_existing_loggers=False)
log = logging.getLogger('nimsapi')
import nimsapi
import internimsclient
# read in private key
privkey_file = config.get('nims', 'ssl_key')
try:
privkey = Crypto.PublicKey.RSA.importKey(open(privkey_file).read())
except:
log.warn(privkey_file + 'is not a valid private SSL key file')
privkey = None
else:
# read config
configfile = '../production.ini'
config = apply_config(configfile)
# import everything else
import json
import time
import base64
import pymongo
import webapp2
import nimsapi
import argparse
import datetime
import nimsutil
import requests
import Crypto.Random
import Crypto.Hash.SHA
import Crypto.PublicKey.RSA
import Crypto.Signature.PKCS1_v1_5
import uwsgidecorators # only in uwsgi execution
# configure logger
log = configure_logger(configfile)
db_uri = config.get('nims', 'db_uri')
stage_path = config.get('nims', 'stage_path')
site_id = config.get('nims', 'site_id')
# load in privkey
privkey = read_privkey(config.get('nims', 'privkey_file'))
# config uwsgi application
application = nimsapi.app
application.config['stage_path'] = stage_path
application.config['site_id'] = site_id
application.config['privkey'] = privkey
# connect db
application.db = connect_db(db_uri)
@uwsgidecorators.postfork
def random_atfork():
Crypto.Random.atfork()
@uwsgidecorators.timer(60)
def internimsclient_timer(signum):
internimsclient(application.db, config.get('nims', 'hostname'), config.get('nims', 'site_id'), privkey, config.get('nims', 'internims_url'))
log.info('successfully loaded private SSL key from ' + privkey_file)
# configure uwsgi application
site_id = config.get('nims', 'site_id')
application = nimsapi.app
application.config['stage_path'] = config.get('nims', 'stage_path')
application.config['site_id'] = site_id
application.config['ssl_key'] = privkey
# connect to db
db_uri = config.get('nims', 'db_uri')
application.db = (pymongo.MongoReplicaSetClient(db_uri) if 'replicaSet' in db_uri else pymongo.MongoClient(db_uri)).get_default_database()
# send is-alive signals
api_uri = config.get('nims', 'api_uri')
internims_url = config.get('nims', 'internims_url')
@uwsgidecorators.timer(60)
def internimsclient_timer(signum):
internimsclient.update(application.db, api_uri, site_id, privkey, internims_url)
@uwsgidecorators.postfork
def random_atfork():
Crypto.Random.atfork()
nimsapiutil.py
+ 4
− 5
View file @ e613dbc2
......@@ -2,8 +2,7 @@
import json
import base64
import socket # socket.gethostname()
import logging
import socket
import webapp2
import datetime
import requests
......@@ -12,6 +11,7 @@ import Crypto.Hash.SHA
import Crypto.PublicKey.RSA
import Crypto.Signature.PKCS1_v1_5
import logging
log = logging.getLogger('nimsapi')
logging.getLogger('requests').setLevel(logging.WARNING) # silence Requests library logging
......@@ -93,7 +93,7 @@ class NIMSRequestHandler(webapp2.RequestHandler):
elif self.ssl_key is not None and self.site_id is not None:
log.debug(socket.gethostname() + ' dispatching to remote ' + self.target_id)
# is target registered?
target = self.app.db.remotes.find_one({'_id': self.target_id}, {'_id':False, 'hostname':True})
target = self.app.db.remotes.find_one({'_id': self.target_id}, {'_id':False, 'api_uri':True})
if not target:
log.debug('remote host ' + self.target_id + ' not in auth list. DENIED')
self.abort(403, self.target_id + 'is not authorized')
......@@ -111,8 +111,7 @@ class NIMSRequestHandler(webapp2.RequestHandler):
reqheaders['Authorization'] = base64.b64encode(signature)
# construct outgoing request
target_api = 'http://' + target['hostname'] + self.request.path # TODO: switch to https
# target_api = 'https://' + target['hostname'] + self.request.path)
target_api = 'https://' + target['api_uri'] + self.request.path.split('/nimsapi')[1]
r = requests.request(method=self.request.method, data=reqpayload, url=target_api, params=reqparams, headers=reqheaders, verify=False)
# return response content
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment