import atexit import json import os import sys import traceback import webapp2 import webapp2_extras.routes from . import base from .jobs.handlers import JobsHandler, JobHandler, GearsHandler, GearHandler, RulesHandler from . import encoder from . import root from . import util from . import config from . import centralclient from . import download from . import upload from .handlers import listhandler from .handlers import userhandler from .handlers import grouphandler from .handlers import containerhandler from .handlers import collectionshandler from .handlers import searchhandler from .handlers import schemahandler from .handlers import reporthandler from .request import SciTranRequest log = config.log try: import uwsgi except ImportError: uwsgi = None class Config(base.RequestHandler): def get(self): """Return public Scitran configuration information.""" return config.get_public_config() def get_js(self): """Return scitran config in javascript format.""" self.response.write( 'config = ' + json.dumps( self.get(), sort_keys=True, indent=4, separators=(',', ': '), default=encoder.custom_json_serializer,) + ';' ) class Version(base.RequestHandler): def get(self): """Return database schema version""" return config.get_version() #regexes used in routing table: routing_regexes = { # group id regex # length between 2 and 32 characters # allowed characters are [0-9a-z.@_-] (start and ends only with [0-9a-z]) 'group_id_re': '[0-9a-z][0-9a-z.@_-]{0,30}[0-9a-z]', # container id regex # hexadecimal string exactly of length 24 'cid_re': '[0-9a-f]{24}', # site id regex # length less than 24 characters # allowed characters are [0-9a-z] 'site_id_re': '[0-9a-z_]{0,24}', # user id regex # any length, allowed chars are [0-9a-z.@_-] 'user_id_re': '[0-9a-z.@_-]*', # container name regex # possible values are projects, sessions, acquisitions or collections 'cont_name_re': 'projects|sessions|acquisitions|collections', # tag regex # length between 3 and 24 characters # any character allowed except '/'' 'tag_re': '[^/]{1,32}', # filename regex # any character allowed except '/' 'filename_re': '[^/]+', # note id regex # hexadecimal string exactly of length 24 'note_id_re': '[0-9a-f]{24}', # schema regex # example: schema_path/schema.json 'schema_re': r'[^/.]{3,60}/[^/.]{3,60}\.json' } def _format(route): return route.format(**routing_regexes) routes = [ webapp2.Route(r'/api', root.Root), webapp2_extras.routes.PathPrefixRoute(r'/api', [ webapp2.Route(r'/download', download.Download, handler_method='download', methods=['GET', 'POST'], name='download'), webapp2.Route(r'/upload/<strategy:label|uid|uid-match>', upload.Upload, handler_method='upload', methods=['POST']), webapp2.Route(r'/clean-packfiles', upload.Upload, handler_method='clean_packfile_tokens', methods=['POST']), webapp2.Route(r'/engine', upload.Upload, handler_method='engine', methods=['POST']), webapp2.Route(r'/sites', centralclient.CentralClient, handler_method='sites', methods=['GET']), webapp2.Route(r'/register', centralclient.CentralClient, handler_method='register', methods=['POST']), webapp2.Route(r'/config', Config, methods=['GET']), webapp2.Route(r'/config.js', Config, handler_method='get_js', methods=['GET']), webapp2.Route(r'/version', Version, methods=['GET']), ]), webapp2.Route(r'/api/users', userhandler.UserHandler, handler_method='get_all', methods=['GET']), webapp2.Route(r'/api/users', userhandler.UserHandler, methods=['POST']), webapp2_extras.routes.PathPrefixRoute(r'/api/users', [ webapp2.Route(r'/self', userhandler.UserHandler, handler_method='self', methods=['GET']), webapp2.Route(r'/self/avatar', userhandler.UserHandler, handler_method='self_avatar', methods=['GET']), webapp2.Route(r'/self/key', userhandler.UserHandler, handler_method='generate_api_key',methods=['POST']), webapp2.Route(_format(r'/<_id:{user_id_re}>'), userhandler.UserHandler, name='user'), webapp2.Route(_format(r'/<uid:{user_id_re}>/groups'), grouphandler.GroupHandler, handler_method='get_all', methods=['GET'], name='groups'), webapp2.Route(_format(r'/<uid:{user_id_re}>/avatar'), userhandler.UserHandler, handler_method='avatar', methods=['GET'], name='avatar'), ]), webapp2.Route(r'/api/jobs', JobsHandler), webapp2_extras.routes.PathPrefixRoute(r'/api/jobs', [ webapp2.Route(r'/next', JobsHandler, handler_method='next', methods=['GET']), webapp2.Route(r'/stats', JobsHandler, handler_method='stats', methods=['GET']), webapp2.Route(r'/reap', JobsHandler, handler_method='reap_stale', methods=['POST']), webapp2.Route(r'/add', JobsHandler, handler_method='add', methods=['POST']), webapp2.Route(r'/<:[^/]+>', JobHandler, name='job'), webapp2.Route(r'/<:[^/]+>/config.json', JobHandler, name='job', handler_method='get_config'), webapp2.Route(r'/<:[^/]+>/retry', JobHandler, name='job', handler_method='retry', methods=['POST']), ]), webapp2.Route(r'/api/gears', GearsHandler), webapp2_extras.routes.PathPrefixRoute(r'/api/gears', [ webapp2.Route(r'/<:[^/]+>', GearHandler), webapp2.Route(r'/<:[^/]+>/invocation', GearHandler, handler_method='get_invocation'), webapp2.Route(r'/<:[^/]+>/suggest/<:[^/]+>/<:[^/]+>', GearHandler, handler_method='suggest'), ]), webapp2.Route(r'/api/rules', RulesHandler), webapp2.Route(r'/api/groups', grouphandler.GroupHandler, handler_method='get_all', methods=['GET']), webapp2.Route(r'/api/groups', grouphandler.GroupHandler, methods=['POST']), webapp2.Route(_format(r'/api/groups/<_id:{group_id_re}>'), grouphandler.GroupHandler, name='group_details', methods=['GET', 'DELETE', 'PUT']), webapp2.Route(r'/api/collections/curators', collectionshandler.CollectionsHandler, handler_method='curators', methods=['GET']), webapp2.Route(r'/api/collections', collectionshandler.CollectionsHandler, name='colls', handler_method='get_all', methods=['GET']), webapp2.Route(r'/api/collections', collectionshandler.CollectionsHandler, methods=['POST']), webapp2.Route(_format(r'/api/collections/<cid:{cid_re}>'), collectionshandler.CollectionsHandler, name='coll_details', methods=['GET', 'PUT', 'DELETE']), webapp2.Route(_format(r'/api/collections/<cid:{cid_re}>/sessions'), collectionshandler.CollectionsHandler, name='coll_ses', handler_method='get_sessions', methods=['GET']), webapp2.Route(_format(r'/api/collections/<cid:{cid_re}>/acquisitions'), collectionshandler.CollectionsHandler, name='coll_acq', handler_method='get_acquisitions', methods=['GET']), webapp2.Route(_format(r'/api/<cont_name:{cont_name_re}>'), containerhandler.ContainerHandler, name='cont_list', handler_method='get_all', methods=['GET']), webapp2.Route(_format(r'/api/<cont_name:{cont_name_re}>'), containerhandler.ContainerHandler, methods=['POST']), webapp2.Route(_format(r'/api/<cont_name:{cont_name_re}>/<cid:{cid_re}>'), containerhandler.ContainerHandler, name='cont_details', methods=['GET','PUT','DELETE']), webapp2.Route(_format(r'/api/<cont_name:{cont_name_re}>/<cid:{cid_re}>/jobs'), containerhandler.ContainerHandler, name='cont_jobs', handler_method='get_jobs', methods=['GET']), webapp2.Route(_format(r'/api/<cont_name:groups>/<cid:{group_id_re}>/<list_name:roles>'), listhandler.ListHandler, name='group_roles_post', methods=['POST']), webapp2.Route(_format(r'/api/<cont_name:groups>/<cid:{group_id_re}>/<list_name:roles>/<site:{site_id_re}>/<_id:{user_id_re}>'), listhandler.ListHandler, name='group_roles', methods=['GET', 'PUT', 'DELETE']), webapp2.Route(_format(r'/api/<cont_name:groups>/<cid:{group_id_re}>/<list_name:tags>'), listhandler.TagsListHandler, methods=['POST'], name='tags_post'), webapp2.Route(_format(r'/api/<cont_name:groups>/<cid:{group_id_re}>/<list_name:tags>/<value:{tag_re}>'), listhandler.TagsListHandler, name='tags', methods=['GET', 'PUT', 'DELETE']), webapp2.Route(_format(r'/api/<cont_name:{cont_name_re}>/<cid:{cid_re}>/<list_name:tags>'), listhandler.TagsListHandler, methods=['POST'], name='tags_post'), webapp2.Route(_format(r'/api/<cont_name:{cont_name_re}>/<cid:{cid_re}>/<list_name:tags>/<value:{tag_re}>'), listhandler.TagsListHandler, name='tags', methods=['GET', 'PUT', 'DELETE']), webapp2.Route(_format(r'/api/<cont_name:{cont_name_re}>/<cid:{cid_re}>/packfile-start'), listhandler.FileListHandler, name='packfile-start', handler_method='packfile_start', methods=['POST']), webapp2.Route(_format(r'/api/<cont_name:{cont_name_re}>/<cid:{cid_re}>/packfile'), listhandler.FileListHandler, name='packfile', handler_method='packfile', methods=['POST']), webapp2.Route(_format(r'/api/<cont_name:{cont_name_re}>/<cid:{cid_re}>/packfile-end'), listhandler.FileListHandler, name='packfile-end', handler_method='packfile_end'), webapp2.Route(_format(r'/api/<cont_name:{cont_name_re}>/<cid:{cid_re}>/<list_name:files>'), listhandler.FileListHandler, name='files_post', methods=['POST']), webapp2.Route(_format(r'/api/<cont_name:{cont_name_re}>/<cid:{cid_re}>/<list_name:files>/<name:{filename_re}>'), listhandler.FileListHandler, name='files', methods=['GET', 'DELETE']), webapp2.Route(_format(r'/api/<cont_name:collections|projects>/<cid:{cid_re}>/<list_name:permissions>'), listhandler.PermissionsListHandler, name='perms_post', methods=['POST']), webapp2.Route(_format(r'/api/<cont_name:collections|projects>/<cid:{cid_re}>/<list_name:permissions>/<site:{site_id_re}>/<_id:{user_id_re}>'), listhandler.PermissionsListHandler, name='perms', methods=['GET', 'PUT', 'DELETE']), webapp2.Route(_format(r'/api/<cont_name:{cont_name_re}>/<cid:{cid_re}>/<list_name:notes>'), listhandler.NotesListHandler, name='notes_post', methods=['POST']), webapp2.Route(_format(r'/api/<cont_name:{cont_name_re}>/<cid:{cid_re}>/<list_name:notes>/<_id:{note_id_re}>'), listhandler.NotesListHandler, name='notes', methods=['GET', 'PUT', 'DELETE']), webapp2.Route(_format(r'/api/users/<uid:{user_id_re}>/<cont_name:{cont_name_re}>'), containerhandler.ContainerHandler, name='user_conts', handler_method='get_all_for_user', methods=['GET']), webapp2.Route(r'/api/projects/groups', containerhandler.ContainerHandler, handler_method='get_groups_with_project', methods=['GET']), webapp2.Route(r'/api/projects/recalc', containerhandler.ContainerHandler, handler_method='calculate_project_compliance', methods=['POST']), webapp2.Route(_format(r'/api/projects/<cid:{cid_re}>/template'), containerhandler.ContainerHandler, handler_method='set_project_template', methods=['POST']), webapp2.Route(_format(r'/api/projects/<cid:{cid_re}>/template'), containerhandler.ContainerHandler, handler_method='delete_project_template', methods=['DELETE']), webapp2.Route(_format(r'/api/projects/<cid:{cid_re}>/recalc'), containerhandler.ContainerHandler, handler_method='calculate_project_compliance', methods=['POST']), webapp2.Route(_format(r'/api/<par_cont_name:groups>/<par_id:{group_id_re}>/<cont_name:projects>'), containerhandler.ContainerHandler, name='cont_sublist_groups', handler_method='get_all', methods=['GET']), webapp2.Route(_format(r'/api/<par_cont_name:{cont_name_re}>/<par_id:{cid_re}>/<cont_name:{cont_name_re}>'), containerhandler.ContainerHandler, name='cont_sublist', handler_method='get_all', methods=['GET']), webapp2.Route(_format(r'/api/search'), searchhandler.SearchHandler, handler_method='advanced_search', name='es_proxy', methods=['POST']), webapp2.Route(_format(r'/api/search/files'), searchhandler.SearchHandler, handler_method='get_datatree', name='es_data', methods=['GET']), webapp2.Route(_format(r'/api/search/<cont_name:{cont_name_re}>'), searchhandler.SearchHandler, name='es_proxy', methods=['GET']), webapp2.Route(_format(r'/api/schemas/<schema:{schema_re}>'), schemahandler.SchemaHandler, name='schemas', methods=['GET']), webapp2.Route(r'/api/report/<report_type:site|project>', reporthandler.ReportHandler, methods=['GET']), webapp2.Route(_format(r'/api/<cont_name:{cont_name_re}>/<cid:{cid_re}>/<list_name:analyses>'), listhandler.AnalysesHandler, name='analysis_post', methods=['POST']), webapp2.Route(_format(r'/api/<cont_name:{cont_name_re}>/<cid:{cid_re}>/<list_name:analyses>/<_id:{cid_re}>'), listhandler.AnalysesHandler, name='analysis', methods=['GET', 'DELETE']), webapp2.Route(_format(r'/api/<cont_name:{cont_name_re}>/<cid:{cid_re}>/<list_name:analyses>/<_id:{cid_re}>/files'), listhandler.AnalysesHandler, handler_method='download', methods=['GET'], name='analysis_files'), webapp2.Route(_format(r'/api/<cont_name:{cont_name_re}>/<cid:{cid_re}>/<list_name:analyses>/<_id:{cid_re}>/files/<name:{filename_re}>'), listhandler.AnalysesHandler, handler_method='download', methods=['GET'], name='analysis_single_file'), webapp2.Route(_format(r'/api/<cont_name:{cont_name_re}>/<cid:{cid_re}>/<list_name:analyses>/<_id:{cid_re}>/notes'), listhandler.AnalysesHandler, handler_method='add_note', methods=['POST'], name='analysis_add_note'), webapp2.Route(_format(r'/api/<cont_name:{cont_name_re}>/<cid:{cid_re}>/<list_name:analyses>/<_id:{cid_re}>/notes/<note_id:{cid_re}>'), listhandler.AnalysesHandler, handler_method='delete_note', methods=['DELETE'], name='analysis_delete_note'), ] def dispatcher(router, request, response): try: if uwsgi is not None: uwsgi.set_logvar('request_id', request.id) except: # pylint: disable=bare-except request.logger.error("Error setting request_id log var", exc_info=True) try: rv = router.default_dispatcher(request, response) if rv is not None: response.write(json.dumps(rv, default=encoder.custom_json_serializer)) response.headers['Content-Type'] = 'application/json; charset=utf-8' except webapp2.HTTPException as e: util.send_json_http_exception(response, str(e), e.code) except Exception as e: # pylint: disable=broad-except request.logger.error("Error dispatching request", exc_info=True) if config.get_item('core', 'debug'): message = traceback.format_exc() else: message = 'Internal Server Error' util.send_json_http_exception(response, message, 500) def app_factory(*_, **__): # pylint: disable=protected-access,unused-argument # don't use config.get_item() as we don't want to require the database at startup application = webapp2.WSGIApplication(routes, debug=config.__config['core']['debug']) application.router.set_dispatcher(dispatcher) application.request_class = SciTranRequest if os.environ.get("SCITRAN_RUNTIME_COVERAGE") == "true": start_coverage() # configure new relic if config.__config['core']['newrelic']: try: import newrelic.agent, newrelic.api.exceptions newrelic.agent.initialize(config.__config['core']['newrelic']) application = newrelic.agent.WSGIApplicationWrapper(application) log.info('New Relic detected and loaded. Monitoring enabled.') except ImportError: log.critical('New Relic libraries not found.') sys.exit(1) except newrelic.api.exceptions.ConfigurationError: log.critical('New Relic detected, but configuration invalid.') sys.exit(1) return application # Functions to enable code coverage when API is started for testing def start_coverage(): import coverage config.log.info("Enabling code coverage") cov = coverage.coverage(source=["api"], data_suffix="integration-tests") cov.start() atexit.register(save_coverage, cov) def save_coverage(cov): config.log.info("Saving coverage") cov.stop() cov.save()