diff --git a/api/api.py b/api/api.py index 881cb03e1cf90e0301e00f8faf10f34e3c39ca3e..98d74f11327242c5171b68ff6f259f188aa5e46a 100644 --- a/api/api.py +++ b/api/api.py @@ -172,6 +172,9 @@ routes = [ webapp2.Route(_format(r'/api/users/<uid:{user_id_re}>/<cont_name:{cont_name_re}>'), containerhandler.ContainerHandler, name='user_conts', handler_method='get_all_for_user', methods=['GET']), webapp2.Route(r'/api/projects/groups', containerhandler.ContainerHandler, handler_method='get_groups_with_project', methods=['GET']), + webapp2.Route(r'/api/projects/recalc', containerhandler.ContainerHandler, handler_method='calculate_project_compliance', methods=['POST']), + webapp2.Route(_format(r'/api/projects/<cid:{cid_re}>/template'), containerhandler.ContainerHandler, handler_method='set_project_template', methods=['POST']), + webapp2.Route(_format(r'/api/projects/<cid:{cid_re}>/recalc'), containerhandler.ContainerHandler, handler_method='calculate_project_compliance', methods=['POST']), webapp2.Route(_format(r'/api/<par_cont_name:groups>/<par_id:{group_id_re}>/<cont_name:projects>'), containerhandler.ContainerHandler, name='cont_sublist_groups', handler_method='get_all', methods=['GET']), webapp2.Route(_format(r'/api/<par_cont_name:{cont_name_re}>/<par_id:{cid_re}>/<cont_name:{cont_name_re}>'), containerhandler.ContainerHandler, name='cont_sublist', handler_method='get_all', methods=['GET']), diff --git a/api/config.py b/api/config.py index 124c30ff39d9495b6e94e1b1529fb005353bcac1..38f2102c24c80c4868915806e15934dbfeb74956 100644 --- a/api/config.py +++ b/api/config.py @@ -128,6 +128,7 @@ expected_input_schemas = set([ 'packfile.json', 'permission.json', 'project.json', + 'project-template.json', 'session.json', 'subject.json', 'user-new.json', diff --git a/api/dao/containerstorage.py b/api/dao/containerstorage.py index 9b444df0b15e772ed546ba0e8d305679aa05e5c4..1110927b065e166a9083669d6512d033d11c98f6 100644 --- a/api/dao/containerstorage.py +++ b/api/dao/containerstorage.py @@ -1,15 +1,22 @@ import bson.errors import bson.objectid +import json import pymongo.errors from .. import util from .. import config from . import consistencychecker -from . import APIStorageException, APIConflictException +from . import APIStorageException, APIConflictException, APINotFoundException from . import hierarchy log = config.log +# TODO: Find a better place to put this until OOP where we can just call cont.children +CHILD_MAP = { + 'groups': 'projects', + 'projects': 'sessions', + 'sessions': 'acquisitions' +} class ContainerStorage(object): """ @@ -23,58 +30,77 @@ class ContainerStorage(object): self.use_object_id = use_object_id self.dbc = config.db[cont_name] - def get_container(self, _id, projection=None, get_children=False): - cont = self._get_el(_id, projection=projection) + @staticmethod + def factory(cont_name, use_object_id = False): + """ + Factory method to aid in the creation of a ContainerStorage instance + when cont_name is dynamic. + """ + if cont_name == 'groups': + return GroupStorage() + elif cont_name == 'projects': + return ProjectStorage() + elif cont_name == 'sessions': + return SessionStorage() + elif cont_name == 'acquisitions': + return AcquisitionStorage() + else: + return ContainerStorage(cont_name, use_object_id) + + def get_container(self, _id, projection=None, get_children=False): + cont = self.get_el(_id, projection=projection) if get_children: - child_map = { - 'groups': 'projects', - 'projects': 'sessions', - 'sessions': 'acquisitions' - } - - child_name = child_map.get(self.cont_name) - if not child_name: - raise ValueError('Children can only be listed from group, project or session level') - else: - query = {self.cont_name[:-1]: bson.objectid.ObjectId(_id)} - cont[child_name] = ContainerStorage(child_name, True).exec_op('GET', query=query, projection=projection) + children = self.get_children(_id, projection=projection) + cont[CHILD_MAP[self.cont_name]] = children + return cont + def get_children(self, _id, projection=None): + try: + child_name = CHILD_MAP[self.cont_name] + except KeyError: + raise APINotFoundException('Children cannot be listed from the {0} level'.format(self.cont_name)) + query = {self.cont_name[:-1]: bson.objectid.ObjectId(_id)} + return self.factory(child_name, use_object_id=True).get_all_el(query, None, projection) + + def _from_mongo(self, cont): return cont + def _to_mongo(self, payload): + return payload def exec_op(self, action, _id=None, payload=None, query=None, user=None, public=False, projection=None, recursive=False, r_payload=None, # pylint: disable=unused-argument replace_metadata=False): """ - Generic method to exec an operation. - The request is dispatched to the corresponding private methods. + Generic method to exec a CRUD operation from a REST verb. """ check = consistencychecker.get_container_storage_checker(action, self.cont_name) data_op = payload or {'_id': _id} check(data_op) if action == 'GET' and _id: - return self._get_el(_id, projection) + return self.get_el(_id, projection=projection) if action == 'GET': - return self._get_all_el(query, user, projection) + return self.get_all_el(query, user, projection) if action == 'DELETE': - return self._delete_el(_id) + return self.delete_el(_id) if action == 'PUT': - return self._update_el(_id, payload, recursive, r_payload, replace_metadata) + return self.update_el(_id, payload, recursive=recursive, r_payload=r_payload, replace_metadata=replace_metadata) if action == 'POST': - return self._create_el(payload) + return self.create_el(payload) raise ValueError('action should be one of GET, POST, PUT, DELETE') - def _create_el(self, payload): + def create_el(self, payload): log.debug(payload) + payload = self._to_mongo(payload) try: result = self.dbc.insert_one(payload) except pymongo.errors.DuplicateKeyError: raise APIConflictException('Object with id {} already exists.'.format(payload['_id'])) return result - def _update_el(self, _id, payload, recursive=False, r_payload=None, replace_metadata=False): + def update_el(self, _id, payload, recursive=False, r_payload=None, replace_metadata=False): replace = None if replace_metadata: replace = {} @@ -83,6 +109,7 @@ class ContainerStorage(object): if payload.get('subject') is not None and payload['subject'].get('metadata') is not None: replace['subject.metadata'] = util.mongo_sanitize_fields(payload['subject'].pop('metadata')) + payload = self._to_mongo(payload) update = { '$set': util.mongo_dict(payload) } @@ -98,7 +125,7 @@ class ContainerStorage(object): hierarchy.propagate_changes(self.cont_name, _id, {}, {'$set': util.mongo_dict(r_payload)}) return self.dbc.update_one({'_id': _id}, update) - def _delete_el(self, _id): + def delete_el(self, _id): if self.use_object_id: try: _id = bson.objectid.ObjectId(_id) @@ -106,15 +133,15 @@ class ContainerStorage(object): raise APIStorageException(e.message) return self.dbc.delete_one({'_id':_id}) - def _get_el(self, _id, projection=None): + def get_el(self, _id, projection=None): if self.use_object_id: try: _id = bson.objectid.ObjectId(_id) except bson.errors.InvalidId as e: raise APIStorageException(e.message) - return self.dbc.find_one(_id, projection) + return self._from_mongo(self.dbc.find_one(_id, projection)) - def _get_all_el(self, query, user, projection): + def get_all_el(self, query, user, projection): if user: if query.get('permissions'): query['$and'] = [{'permissions': {'$elemMatch': user}}, {'permissions': query.pop('permissions')}] @@ -122,12 +149,17 @@ class ContainerStorage(object): query['permissions'] = {'$elemMatch': user} log.debug(query) log.debug(projection) - result = self.dbc.find(query, projection) - return list(result) + results = list(self.dbc.find(query, projection)) + for cont in results: + cont = self._from_mongo(cont) + return results class GroupStorage(ContainerStorage): - def _create_el(self, payload): + def __init__(self): + super(GroupStorage,self).__init__('groups', use_object_id=False) + + def create_el(self, payload): log.debug(payload) roles = payload.pop('roles') return self.dbc.update_one( @@ -137,3 +169,117 @@ class GroupStorage(ContainerStorage): '$setOnInsert': {'roles': roles} }, upsert=True) + + +class ProjectStorage(ContainerStorage): + + def __init__(self): + super(ProjectStorage,self).__init__('projects', use_object_id=True) + + def _from_mongo(self, cont): + template = cont.get('template') + if template: + cont['template'] = json.loads(template) + return super(ProjectStorage,self)._from_mongo(cont) + + def _to_mongo(self, payload): + template = payload.get('template') + if template: + payload['template'] = json.dumps(template) + return super(ProjectStorage,self)._to_mongo(payload) + + + def recalc_sessions_compliance(self, project_id=None): + if project_id is None: + # Recalc all projects + projects = self.get_all_el({'template': {'$exists': True}}, None, None) + else: + project = self.get_container(project_id) + if project: + projects = [project] + else: + raise APINotFoundException('Could not find project {}'.format(project_id)) + changed_sessions = [] + + for project in projects: + template = project.get('template',{}) + if not template: + continue + else: + session_storage = SessionStorage() + sessions = session_storage.get_all_el({'project': project['_id']}, None, None) + for s in sessions: + changed = session_storage.recalc_session_compliance(s['_id'], session=s, template=template) + if changed: + changed_sessions.append(s['_id']) + return changed_sessions + + +class SessionStorage(ContainerStorage): + + def __init__(self): + super(SessionStorage,self).__init__('sessions', use_object_id=True) + + def create_el(self, payload): + project = ProjectStorage().get_container(payload['project']) + if project.get('template'): + payload['project_has_template'] = True + payload['satisfies_template'] = hierarchy.is_session_compliant(payload, project.get('template')) + return super(SessionStorage, self).create_el(payload) + + def update_el(self, _id, payload, recursive=False, r_payload=None, replace_metadata=False): + session = self.get_container(_id) + if session is None: + raise APINotFoundException('Could not find session {}'.format(_id)) + if session.get('project_has_template') or payload.get('project_has_template'): + project = ProjectStorage().get_container(session['project']) + session.update(payload) + payload['satisfies_template'] = hierarchy.is_session_compliant(session, project.get('template')) + return super(SessionStorage, self).update_el(_id, payload, recursive=recursive, r_payload=r_payload, replace_metadata=r_payload) + + def recalc_session_compliance(self, session_id, session=None, template=None): + """ + Calculates a session's compliance with the project's session template. + Returns True if the status changed, False otherwise + """ + if session is None: + session = self.get_container(session_id) + if session is None: + raise APINotFoundException('Could not find session {}'.format(session_id)) + if session.get('project_has_template'): + if template is None: + template = ProjectStorage().get_container(session['project']).get('template') + satisfies_template = hierarchy.is_session_compliant(session, template) + if session.get('satisfies_template') != satisfies_template: + update = {'satisfies_template': satisfies_template} + super(SessionStorage, self).update_el(session_id, update) + return True + return False + + +class AcquisitionStorage(ContainerStorage): + + def __init__(self): + super(AcquisitionStorage,self).__init__('acquisitions', use_object_id=True) + + def create_el(self, payload): + result = super(AcquisitionStorage, self).create_el(payload) + SessionStorage().recalc_session_compliance(payload['session']) + return result + + def update_el(self, _id, payload, recursive=False, r_payload=None, replace_metadata=False): + result = super(AcquisitionStorage, self).update_el(_id, payload, recursive, r_payload, replace_metadata) + acquisition = self.get_container(_id) + if acquisition is None: + raise APINotFoundException('Could not find acquisition {}'.format(_id)) + SessionStorage().recalc_session_compliance(acquisition['session']) + return result + + def delete_el(self, _id): + acquisition = self.get_container(_id) + if acquisition is None: + raise APINotFoundException('Could not find acquisition {}'.format(_id)) + result = super(AcquisitionStorage, self).delete_el(_id) + SessionStorage().recalc_session_compliance(acquisition['session']) + return result + diff --git a/api/dao/containerutil.py b/api/dao/containerutil.py index c79fc0f6ed2343d3666d2ad4a4e52e928da4633c..f0bcbddb4064ee5f7de7b9f7d187d9e26693f8fe 100644 --- a/api/dao/containerutil.py +++ b/api/dao/containerutil.py @@ -39,22 +39,46 @@ def add_id_to_subject(subject, pid): def get_stats(cont, cont_type): """ - Add a session and attachment count to a project or collection + Add a session, subject, non-compliant session and attachment count to a project or collection """ if cont_type not in ['projects', 'collections']: return cont - session_ids = [] + # Get attachment count from file array length + cont['attachment_count'] = len(cont.get('files', [])) + + # Get session and non-compliant session count + match_q = {} if cont_type == 'projects': - result = list(config.db.sessions.find({'project': cont['_id']}, {'_id': 1})) - session_ids = [s['_id'] for s in result] + match_q = {'project': cont['_id']} elif cont_type == 'collections': result = config.db.acquisitions.find({'collections': cont['_id']}, {'session': 1}) session_ids = list(set([s['session'] for s in result])) + match_q = {'_id': {'$in': session_ids}} + + pipeline = [ + {'$match': match_q}, + {'$project': {'_id': 1, 'non_compliant': {'$cond': [{'$eq': ['$satisfies_template', False]}, 1, 0]}}}, + {'$group': {'_id': 1, 'noncompliant_count': {'$sum': '$non_compliant'}, 'total': {'$sum': 1}}} + ] + + result = config.db.command('aggregate', 'sessions', pipeline=pipeline).get('result', []) + if len(result) > 0: + cont['session_count'] = result[0].get('total', 0) + cont['noncompliant_session_count'] = result[0].get('noncompliant_count', 0) + else: + # If there are no sessions, return zero'd out stats + cont['session_count'] = 0 + cont['noncompliant_session_count'] = 0 + cont['subject_count'] = 0 + return cont + + # Get subject count + match_q['subject._id'] = {'$ne': None} pipeline = [ - {'$match': {'_id': {'$in': session_ids}, 'subject._id': {'$ne': None}}}, + {'$match': match_q}, {'$group': {'_id': '$subject._id'}}, {'$group': {'_id': 1, 'count': { '$sum': 1 }}} ] @@ -66,9 +90,6 @@ def get_stats(cont, cont_type): else: cont['subject_count'] = 0 - cont['attachment_count'] = len(cont.get('files', [])) - cont['session_count'] = len(session_ids) - return cont diff --git a/api/dao/hierarchy.py b/api/dao/hierarchy.py index a13eb93d8bde776951faba9d052edd337a11e0ee..2f0ccb30c163811179a7dee25a171bb8269bef16 100644 --- a/api/dao/hierarchy.py +++ b/api/dao/hierarchy.py @@ -3,6 +3,7 @@ import copy import datetime import dateutil.parser import difflib +from jsonschema import Draft4Validator, ValidationError import pymongo import re @@ -112,6 +113,63 @@ def propagate_changes(cont_name, _id, query, update): else: raise ValueError('changes can only be propagated from group, project or session level') +def is_session_compliant(session, template): + """ + Given a project-level session template and a session, + returns True/False if the session is in compliance with the template + """ + s_requirements = template.get('session') + a_requirements = template.get('acquisitions') + f_requirements = template.get('files') + + acquisitions = [] + if a_requirements or f_requirements: + acquisitions = list(config.db.acquisitions.find({'session': session['_id']})) + + if s_requirements: + validator = Draft4Validator(s_requirements.get('schema')) + try: + validator.validate(session) + except ValidationError: + return False + + if a_requirements: + for req in a_requirements: + validator = Draft4Validator(req.get('schema')) + min_count = req.get('minimum') + count = 0 + for a in acquisitions: + try: + validator.validate(a) + except ValidationError: + continue + else: + count += 1 + if count >= min_count: + break + if count < min_count: + return False + + if f_requirements: + files_ = [f for a in acquisitions for f in a.get('files', [])] + for req in f_requirements: + validator = Draft4Validator(req.get('schema')) + min_count = req.get('minimum') + count = 0 + for f in files_: + try: + validator.validate(a) + except ValidationError: + continue + else: + count += 1 + if count >= min_count: + break + if count < min_count: + return False + + return True + def upsert_fileinfo(cont_name, _id, fileinfo): # TODO: make all functions take singular noun cont_name += 's' diff --git a/api/dao/liststorage.py b/api/dao/liststorage.py index 17bb33787978cd02a7382c43da9ff71329314593..7206395c6fe7cc07a7dc49c7f5d3f6eaed07c1a6 100644 --- a/api/dao/liststorage.py +++ b/api/dao/liststorage.py @@ -4,6 +4,7 @@ import bson.objectid from .. import config from . import consistencychecker, containerutil from . import APIStorageException, APIConflictException +from .containerstorage import SessionStorage, AcquisitionStorage from ..jobs.jobs import Job log = config.log @@ -105,7 +106,14 @@ class ListStorage(object): update = {'$pull': {self.list_name: query_params} } log.debug('query {}'.format(query)) log.debug('update {}'.format(update)) - return self.dbc.update_one(query, update) + result = self.dbc.update_one(query, update) + if self.list_name is 'files' and self.cont_name in ['sessions', 'acquisitions']: + if self.cont_name == 'sessions': + session_id = _id + else: + session_id = AcquisitionStorage().get_container(_id).get('session') + SessionStorage().recalc_session_compliance(session_id) + return result def _get_el(self, _id, query_params): log.debug('query_params {}'.format(query_params)) diff --git a/api/handlers/containerhandler.py b/api/handlers/containerhandler.py index f4f7008b50911d6c9a37b180d2318723a01e11ff..cdcb48664c262a137d8c17bf605367e5b4e9f85e 100644 --- a/api/handlers/containerhandler.py +++ b/api/handlers/containerhandler.py @@ -49,9 +49,9 @@ class ContainerHandler(base.RequestHandler): # "use_object_id" implies that the container ids are converted to ObjectId container_handler_configurations = { 'projects': { - 'storage': containerstorage.ContainerStorage('projects', use_object_id=use_object_id['projects']), + 'storage': containerstorage.ProjectStorage(), 'permchecker': containerauth.default_container, - 'parent_storage': containerstorage.ContainerStorage('groups', use_object_id=use_object_id['groups']), + 'parent_storage': containerstorage.GroupStorage(), 'storage_schema_file': 'project.json', 'payload_schema_file': 'project.json', 'list_projection': {'metadata': 0}, @@ -59,9 +59,9 @@ class ContainerHandler(base.RequestHandler): 'children_cont': 'sessions' }, 'sessions': { - 'storage': containerstorage.ContainerStorage('sessions', use_object_id=use_object_id['sessions']), + 'storage': containerstorage.SessionStorage(), 'permchecker': containerauth.default_container, - 'parent_storage': containerstorage.ContainerStorage('projects', use_object_id=use_object_id['projects']), + 'parent_storage': containerstorage.ProjectStorage(), 'storage_schema_file': 'session.json', 'payload_schema_file': 'session.json', 'list_projection': {'metadata': 0}, @@ -69,9 +69,9 @@ class ContainerHandler(base.RequestHandler): 'children_cont': 'acquisitions' }, 'acquisitions': { - 'storage': containerstorage.ContainerStorage('acquisitions', use_object_id=use_object_id['acquisitions']), + 'storage': containerstorage.AcquisitionStorage(), 'permchecker': containerauth.default_container, - 'parent_storage': containerstorage.ContainerStorage('sessions', use_object_id=use_object_id['sessions']), + 'parent_storage': containerstorage.SessionStorage(), 'storage_schema_file': 'acquisition.json', 'payload_schema_file': 'acquisition.json', 'list_projection': {'metadata': 0} @@ -482,6 +482,36 @@ class ContainerHandler(base.RequestHandler): group_ids = list(set((p['group'] for p in self.get_all('projects')))) return list(config.db.groups.find({'_id': {'$in': group_ids}}, ['name'])) + def set_project_template(self, **kwargs): + project_id = kwargs.pop('cid') + self.config = self.container_handler_configurations['projects'] + self.storage = self.config['storage'] + container = self._get_container(project_id) + + template = self.request.json_body + validators.validate_data(template, 'project-template.json', 'input', 'POST') + payload = {'template': template} + payload['modified'] = datetime.datetime.utcnow() + + permchecker = self._get_permchecker(container) + result = permchecker(self.storage.exec_op)('PUT', _id=project_id, payload=payload) + + if result.modified_count == 1: + sessions = self.storage.get_children(project_id, projection={'_id':1}) + session_storage = self.container_handler_configurations['sessions']['storage'] + for s in sessions: + session_storage.exec_op('PUT', s['_id'], payload={'project_has_template': True}) + return {'modified': result.modified_count} + else: + self.abort(404, 'Could not find project {}'.format(project_id)) + + def calculate_project_compliance(self, **kwargs): + project_id = kwargs.pop('cid', None) + log.debug("project_id is {}".format(project_id)) + self.config = self.container_handler_configurations['projects'] + self.storage = self.config['storage'] + return {'sessions_changed': self.storage.recalc_sessions_compliance(project_id=project_id)} + def _get_validators(self): mongo_schema_uri = validators.schema_uri('mongo', self.config.get('storage_schema_file')) mongo_validator = validators.decorator_from_schema_path(mongo_schema_uri) diff --git a/api/handlers/grouphandler.py b/api/handlers/grouphandler.py index cb23723e4fa1d074f8aceae9cc0682777845d442..b7b7a805f9471d92fbe3278395dd845e756583e3 100644 --- a/api/handlers/grouphandler.py +++ b/api/handlers/grouphandler.py @@ -11,7 +11,7 @@ class GroupHandler(base.RequestHandler): def __init__(self, request=None, response=None): super(GroupHandler, self).__init__(request, response) - self.storage = containerstorage.GroupStorage('groups', use_object_id=False) + self.storage = containerstorage.GroupStorage() def get(self, _id): group = self._get_group(_id) diff --git a/api/jobs/gears.py b/api/jobs/gears.py index 9471cabbaf19d35a28203d32c0500e9383a51550..48097e1defbdc32080c5925e0c4b3be3d40f70cc 100644 --- a/api/jobs/gears.py +++ b/api/jobs/gears.py @@ -62,7 +62,7 @@ def suggest_container(gear, cont_name, cid): Given a container reference, suggest files that would work well for each input on a gear. """ - root = ContainerStorage(cont_name, True).get_container(cid, projection={'permissions':0}, get_children=True) + root = ContainerStorage.factory(cont_name, True).get_container(cid, projection={'permissions':0}, get_children=True) invocation_schema = get_invocation_schema(gear) schemas = {} diff --git a/api/placer.py b/api/placer.py index 2298558539fe495d1bd3e04fba0dfd00fa801482..2caef4954ef5808cbb548832eb18160fd2aadc73 100644 --- a/api/placer.py +++ b/api/placer.py @@ -13,6 +13,7 @@ from . import files from . import tempdir as tempfile from . import util from . import validators +from .dao.containerstorage import SessionStorage, AcquisitionStorage from .dao import containerutil, hierarchy from .jobs import rules from .types import Origin @@ -94,6 +95,14 @@ class Placer(object): # Queue any jobs as a result of this upload rules.create_jobs(config.db, self.container, self.container_type, info) + def recalc_session_compliance(self): + if self.container_type in ['session', 'acquisition'] and self.id_: + if self.container_type == 'session': + session_id = self.id_ + else: + session_id = AcquisitionStorage().get_container(str(self.id_)).get('session') + SessionStorage().recalc_session_compliance(session_id) + class TargetedPlacer(Placer): """ @@ -114,6 +123,7 @@ class TargetedPlacer(Placer): self.saved.append(info) def finalize(self): + self.recalc_session_compliance() return self.saved @@ -129,6 +139,7 @@ class UIDPlacer(Placer): def __init__(self, container_type, container, id_, metadata, timestamp, origin, context): super(UIDPlacer, self).__init__(container_type, container, id_, metadata, timestamp, origin, context) self.metadata_for_file = {} + self.session_id = None def check(self): @@ -144,6 +155,8 @@ class UIDPlacer(Placer): self.metadata_for_file = {} for target in targets: + if target[0].level is 'session': + self.session_id = target[0].id_ for name in target[1]: self.metadata_for_file[name] = { 'container': target[0], @@ -181,6 +194,10 @@ class UIDPlacer(Placer): self.saved.append(info) def finalize(self): + if self.session_id: + self.container_type = 'session' + self.id_ = self.session_id + self.recalc_session_compliance() return self.saved @@ -238,6 +255,7 @@ class EnginePlacer(Placer): self.metadata[k].pop('files', {}) hierarchy.update_container_hierarchy(self.metadata, bid, self.container_type) + self.recalc_session_compliance() return self.saved @@ -279,6 +297,7 @@ class TokenPlacer(Placer): dest = os.path.join(self.folder, os.path.basename(path)) shutil.move(path, dest) + self.recalc_session_compliance() return self.saved @@ -514,6 +533,8 @@ class PackfilePlacer(Placer): self.save_file(cgi_field, cgi_info) + self.recalc_session_compliance() + # Delete token config.db['tokens'].delete_one({ '_id': token }) diff --git a/raml/api.raml b/raml/api.raml index 23cb5716a3826b66330141475826d1255e59f1d3..ee180ea45f166d9d000bba03b27275a71048f94c 100644 --- a/raml/api.raml +++ b/raml/api.raml @@ -26,3 +26,4 @@ traits: /gears: !include resources/gears.raml /rules: !include resources/rules.raml /groups: !include resources/groups.raml +/projects: !include resources/projects.raml diff --git a/raml/examples/input/project-template.json b/raml/examples/input/project-template.json new file mode 100644 index 0000000000000000000000000000000000000000..6e5b25d4725362eeb665e50dc16d10553033a322 --- /dev/null +++ b/raml/examples/input/project-template.json @@ -0,0 +1,55 @@ +{ + "session": { + "schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "properties": { + "label": { + "type": "string", + "pattern": "^(?i)test_pattern$" } + } + } + }, + "acquisitions": [ + { + "schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "measurement": { + "type": "string", + "pattern": "^[aA]natomical$" } + }, + "required": ["measurement"] + }, + "minimum": 2 + }, + { + "schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "measurement": { + "type": "string", + "pattern": "^(?i)functional$" } + }, + "required": ["measurement"] + }, + "minimum": 1 + }, + { + "schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "measurement": { "enum": ["Localizer"] }, + "label": { + "type": "string", + "pattern": "t1" + } + }, + "required": ["label", "measurement"] + }, + "minimum": 1 + } + ] +} diff --git a/raml/resources/projects.raml b/raml/resources/projects.raml new file mode 100644 index 0000000000000000000000000000000000000000..63c913313c04e420ce857e9c1baa376921f590c0 --- /dev/null +++ b/raml/resources/projects.raml @@ -0,0 +1,50 @@ +/{ProjectId}/template: + uriParameters: + ProjectId: + type: string + required: true + post: + description: Set the session template for a project + body: + application/json: + example: !include ../examples/input/project-template.json + schema: !include ../schemas/input/project-template.json + responses: + 200: + description: Template was saved + body: + application/json: + example: | + {"modified": 1} + 404: + description: Project was not found + +/recalc: + post: + description: | + Iterates all projects that have a session template. + Recalculate if projects' sessions satisfy the template. + Returns list of modified session ids. + responses: + 200: + description: | + Projects' sessions' compliance was recalculated. + Returns list of session that were modified. + +/{ProjectId}/recalc: + uriParameters: + ProjectId: + type: string + required: true + post: + description: | + Recalculate if sessions in the project satisfy the template. + Returns list of modified session ids. + responses: + 200: + description: | + Project's sessions' compliance was recalculated. + Returns list of session that were modified. + 404: + description: Project was not found + diff --git a/raml/schemas/definitions/project-template.json b/raml/schemas/definitions/project-template.json new file mode 100644 index 0000000000000000000000000000000000000000..d70bfb700010e0aedc308b19479107aff0576a86 --- /dev/null +++ b/raml/schemas/definitions/project-template.json @@ -0,0 +1,38 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "A project's session template", + "type": "object", + "definitions": { + "requirement": { + "anyOf": [ + {"required": ["minimum"]}, + {"required": ["maximum"]} + ], + "properties": { + "schema": {"$ref": "http://json-schema.org/draft-04/schema"}, + "minimum": {"type": "integer", "minimum": 0}, + "maximum": {"type": "integer", "minimum": 0} + }, + "required": ["schema"] + } + }, + "properties": { + "session": { + "properties": { + "schema": {"$ref": "http://json-schema.org/draft-04/schema"} + }, + "required": ["schema"] + }, + "acquisitions": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/definitions/requirement"} + }, + "files": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/definitions/requirement"} + } + }, + "additionalProperties": false +} diff --git a/raml/schemas/input/project-template.json b/raml/schemas/input/project-template.json new file mode 100644 index 0000000000000000000000000000000000000000..30b70e694ce29bba82ad7cc554db40b8f8c46567 --- /dev/null +++ b/raml/schemas/input/project-template.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref":"../definitions/project-template.json#" +} diff --git a/raml/schemas/mongo/project.json b/raml/schemas/mongo/project.json index 945ce55cf04a811f25689b223e926806fa644933..03ae0e63e95bb941ea78589378ffd2fa883ea1e8 100644 --- a/raml/schemas/mongo/project.json +++ b/raml/schemas/mongo/project.json @@ -12,6 +12,7 @@ "public": {}, "archived": {}, "label": {}, + "template": {"$ref": "../definitions/project-template.json"}, "tags": {}, "metadata": {}, diff --git a/raml/schemas/mongo/session.json b/raml/schemas/mongo/session.json index 4ea5ab9a806f4efcc6bacacf014ac1729eab19d9..191bbf5f83a488e0f59d05c0d82b1c1539b02cdf 100644 --- a/raml/schemas/mongo/session.json +++ b/raml/schemas/mongo/session.json @@ -12,6 +12,8 @@ "public": {}, "archived": {}, "label": {}, + "satisfies_template": {"type": "boolean"}, + "project_has_template": {"type": "boolean"}, "tags": {}, "metadata": {}, diff --git a/test/integration_tests/abao/abao_test_hooks.js b/test/integration_tests/abao/abao_test_hooks.js index 995471e66be8ac720780c3bddbe94c65754b7d86..471244e13c40023f868b74c235a71330ad544538 100644 --- a/test/integration_tests/abao/abao_test_hooks.js +++ b/test/integration_tests/abao/abao_test_hooks.js @@ -43,6 +43,11 @@ hooks.skip("POST /upload/uid-match -> 200"); hooks.skip("POST /upload/uid-match -> 404"); hooks.skip("POST /engine -> 200"); +// Skipping until merge with rest of project raml (So we have a ProjectId) +hooks.skip("POST /projects/{ProjectId}/template -> 200") +hooks.skip("POST /projects/{ProjectId}/recalc -> 200") + + hooks.beforeEach(function (test, done) { test.request.query.root = "true" test.request.headers.Authorization = "scitran-user XZpXI40Uk85eozjQkU1zHJ6yZHpix+j0mo1TMeGZ4dPzIqVPVGPmyfeK"; diff --git a/test/integration_tests/postman/integration_tests.postman_collection b/test/integration_tests/postman/integration_tests.postman_collection index a98bc4b52dd3e502bf2d306d57adb6f516a2172e..cb09a52f2ac76106dd496a7ba62d89496adc2cca 100644 --- a/test/integration_tests/postman/integration_tests.postman_collection +++ b/test/integration_tests/postman/integration_tests.postman_collection @@ -2,7 +2,7 @@ "variables": [], "info": { "name": "test", - "_postman_id": "fc8f9020-aa70-327d-4e99-9c8a98db1009", + "_postman_id": "0cd62962-97a4-0bda-dd64-7c3a79d47345", "description": "", "schema": "https://schema.getpostman.com/json/collection/v2.0.0/collection.json" }, @@ -36,8 +36,7 @@ "body": { "mode": "raw", "raw": "{\"_id\":\"jane.doe@gmail.com\",\"firstname\":\"Jane\",\"lastname\":\"Doe\",\"email\":\"jane.doe@gmail.com\"}" - }, - "description": "List users\n\n" + } }, "response": [] }, @@ -70,8 +69,7 @@ "body": { "mode": "raw", "raw": "{\"_id\":\"jane.doe@gmail.com\", \"lastname\":\"Doe\",\"email\":\"jane.doe@gmail.com\"}" - }, - "description": "\n\n" + } }, "response": [] }, @@ -113,8 +111,7 @@ "enabled": true } ] - }, - "description": "" + } }, "response": [] }, @@ -142,8 +139,7 @@ "body": { "mode": "raw", "raw": "{\n\"_id\":\"test-group\"\n}" - }, - "description": "" + } }, "response": [] }, @@ -171,8 +167,7 @@ "body": { "mode": "raw", "raw": " {\n \"group\": \"test-group\",\n \"label\": \"test-project\",\n \"public\": false\n }" - }, - "description": "" + } }, "response": [] }, @@ -200,8 +195,7 @@ "body": { "mode": "raw", "raw": " {\n \"subject\": {\n \"code\": \"test-subject-1\"\n },\n \"label\": \"test-session-1\",\n \"project\": \"{{test-project-id}}\",\n \"public\": false\n }" - }, - "description": "" + } }, "response": [] }, @@ -229,8 +223,7 @@ "body": { "mode": "raw", "raw": "{\n \"label\": \"test-acquisition-1\",\n \"session\":\"{{test-session-id}}\",\n \"public\": false\n }" - }, - "description": "" + } }, "response": [] }, @@ -268,12 +261,10 @@ "key": "file", "src": "test/integration_tests/postman/test_files/test-1.dcm", "type": "file", - "enabled": true, - "value": "" + "enabled": true } ] - }, - "description": "" + } }, "response": [] }, @@ -363,6 +354,383 @@ "description": "" }, "response": [] + }, + { + "name": "[Session Template] Create project", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": "tests[\"Status code is 200\"] = responseCode.code === 200;\n\ntests[\"Response time is less than 200ms\"] = responseTime < 200;\n\nvar jsonData = JSON.parse(responseBody);\n\npostman.setGlobalVariable(\"ST-project-id\", jsonData._id);" + } + } + ], + "request": { + "url": "{{baseUri}}/projects", + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "scitran-user {{test_user_api_key}}", + "description": "" + } + ], + "body": { + "mode": "raw", + "raw": " {\n \"group\": \"test-group\",\n \"label\": \"Project with template\",\n \"public\": false\n }" + }, + "description": "Create the project for testing session templates" + }, + "response": [] + }, + { + "name": "[Session Template] Create compliant session", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": "tests[\"Status code is 200\"] = responseCode.code === 200;\n\ntests[\"Response time is less than 200ms\"] = responseTime < 200;\n\nvar jsonData = JSON.parse(responseBody);\n\npostman.setGlobalVariable(\"ST-compliant-session-id\", jsonData._id);" + } + } + ], + "request": { + "url": "{{baseUri}}/sessions?root=true", + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "scitran-user {{test_user_api_key}}", + "description": "" + } + ], + "body": { + "mode": "raw", + "raw": " {\n \"subject\": {\n \"code\": \"ex8945\"\n },\n \"label\": \"Compliant Session\",\n \"project\": \"{{ST-project-id}}\",\n \"public\": false\n }" + }, + "description": "Create a session that will be compliant with the project-level session template" + }, + "response": [] + }, + { + "name": "[Session Template] Create non-compliant session", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": "tests[\"Status code is 200\"] = responseCode.code === 200;\n\ntests[\"Response time is less than 200ms\"] = responseTime < 200;\n\nvar jsonData = JSON.parse(responseBody);\n\npostman.setGlobalVariable(\"ST-noncompliant-session-id\", jsonData._id);" + } + } + ], + "request": { + "url": "{{baseUri}}/sessions?root=true", + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "scitran-user {{test_user_api_key}}", + "description": "" + } + ], + "body": { + "mode": "raw", + "raw": " {\n \"subject\": {\n \"code\": \"ex9849\"\n },\n \"label\": \"Non-compliant Session\",\n \"project\": \"{{ST-project-id}}\",\n \"public\": false\n }" + }, + "description": "Create a session that will NOT be compliant with the project-level session template" + }, + "response": [] + }, + { + "name": "[Session Template] Create acquisition-1 for compliant session", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": "tests[\"Status code is 200\"] = responseCode.code === 200;\n\ntests[\"Response time is less than 200ms\"] = responseTime < 200;\n\nvar jsonData = JSON.parse(responseBody);\n\npostman.setGlobalVariable(\"ST-cs-acquisition-1\", jsonData._id);" + } + } + ], + "request": { + "url": "{{baseUri}}/acquisitions?root=true", + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "scitran-user {{test_user_api_key}}", + "description": "" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"label\": \"c-acquisition-1-t1\",\n \"session\":\"{{ST-compliant-session-id}}\",\n \"public\": false,\n \"measurement\": \"localizer\"\n}" + }, + "description": "Create an acquisition for the compliant session" + }, + "response": [] + }, + { + "name": "[Session Template] Create acquisition-2 for compliant session", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": "tests[\"Status code is 200\"] = responseCode.code === 200;\n\ntests[\"Response time is less than 200ms\"] = responseTime < 200;\n\nvar jsonData = JSON.parse(responseBody);\n\npostman.setGlobalVariable(\"ST-cs-acquisition-2\", jsonData._id);" + } + } + ], + "request": { + "url": "{{baseUri}}/acquisitions?root=true", + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "scitran-user {{test_user_api_key}}", + "description": "" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"label\": \"c-acquisition-2\",\n \"session\":\"{{ST-compliant-session-id}}\",\n \"public\": false,\n \"measurement\": \"localizer\"\n}" + }, + "description": "Create an acquisition for the compliant session" + }, + "response": [] + }, + { + "name": "[Session Template] Create acquisition-1 for noncompliant session", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": "tests[\"Status code is 200\"] = responseCode.code === 200;\n\ntests[\"Response time is less than 200ms\"] = responseTime < 200;\n\nvar jsonData = JSON.parse(responseBody);\n\npostman.setGlobalVariable(\"ST-ncs-acquisition-1\", jsonData._id);" + } + } + ], + "request": { + "url": "{{baseUri}}/acquisitions?root=true", + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "scitran-user {{test_user_api_key}}", + "description": "" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"label\": \"nc-acquisition-1\",\n \"session\":\"{{ST-noncompliant-session-id}}\",\n \"public\": false,\n \"measurement\": \"localizer\"\n}" + }, + "description": "Create an acquisition for the noncompliant session" + }, + "response": [] + }, + { + "name": "[Session Template] Add project template", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": "tests[\"Status code is 200\"] = responseCode.code === 200;\n\ntests[\"Response time is less than 200ms\"] = responseTime < 200;\n\nvar jsonData = JSON.parse(responseBody);\n\ntests[\"Project updated\"] = jsonData.modified == 1;" + } + } + ], + "request": { + "url": "{{baseUri}}/projects/{{ST-project-id}}/template", + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "scitran-user {{test_user_api_key}}", + "description": "" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"session\": {\n \"schema\": {\n \"$schema\": \"http://json-schema.org/draft-04/schema#\",\n \"properties\": {\n \"subject\": {\n \"type\": \"object\",\n \"properties\": {\n \"code\": {\n \"type\": \"string\",\n \"pattern\": \"^ex\" \n }\n },\n \"required\": [\"code\"]\n }\n },\n \"required\": [\"subject\"]\n }\n },\n \"acquisitions\": [\n {\n \"schema\": {\n \"$schema\": \"http://json-schema.org/draft-04/schema#\",\n \"type\": \"object\",\n \"properties\": {\n \"measurement\": {\n \"type\": \"string\",\n \"pattern\": \"^(?i)localizer$\" \n }\n },\n \"required\": [\"measurement\"]\n },\n \"minimum\": 2\n },\n {\n \"schema\": {\n \"$schema\": \"http://json-schema.org/draft-04/schema#\",\n \"type\": \"object\",\n \"properties\": {\n \"measurement\": {\n \"type\": \"string\",\n \"pattern\": \"^(?i)localizer$\" \n },\n \"label\": {\n \"type\": \"string\",\n \"pattern\": \"t1\"\n }\n },\n \"required\": [\"label\", \"measurement\"]\n },\n \"minimum\": 1\n }\n ]\n}" + }, + "description": "Should modify project and it's sessions. Both sessions should have a \"project-has-template\" flag as \"true\" and the session setup to be compliant should have \"compliant\" flag set to True." + }, + "response": [] + }, + { + "name": "[Session Template] Test session 1 is marked as compilant", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": "tests[\"Status code is 200\"] = responseCode.code === 200;\n\ntests[\"Response time is less than 200ms\"] = responseTime < 200;\n\nvar jsonData = JSON.parse(responseBody);\n\ntests[\"Project has template marked\"] = jsonData.project_has_template === true;\n\ntests[\"Session marked as compliant\"] = jsonData.satisfies_template === true;" + } + } + ], + "request": { + "url": "{{baseUri}}/sessions/{{ST-compliant-session-id}}", + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "scitran-user {{test_user_api_key}}", + "description": "" + } + ], + "body": { + "mode": "raw", + "raw": "" + }, + "description": "" + }, + "response": [] + }, + { + "name": "[Session Template] Test session 2 is marked as non-compilant copy", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": "tests[\"Status code is 200\"] = responseCode.code === 200;\n\ntests[\"Response time is less than 200ms\"] = responseTime < 200;\n\nvar jsonData = JSON.parse(responseBody);\n\ntests[\"Project has template marked\"] = jsonData.project_has_template === true;\n\ntests[\"Session marked as compliant\"] = jsonData.satisfies_template === false;" + } + } + ], + "request": { + "url": "{{baseUri}}/sessions/{{ST-noncompliant-session-id}}", + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "scitran-user {{test_user_api_key}}", + "description": "" + } + ], + "body": { + "mode": "raw", + "raw": "" + }, + "description": "" + }, + "response": [] + }, + { + "name": "[Session Template] Create acquisition-2 for noncompliant session", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": "tests[\"Status code is 200\"] = responseCode.code === 200;\n\ntests[\"Response time is less than 200ms\"] = responseTime < 200;\n\nvar jsonData = JSON.parse(responseBody);\n\npostman.setGlobalVariable(\"ST-ncs-acquisition-2\", jsonData._id);" + } + } + ], + "request": { + "url": "{{baseUri}}/acquisitions?root=true", + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "scitran-user {{test_user_api_key}}", + "description": "" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"label\": \"nc-acquisition-2-t1\",\n \"session\":\"{{ST-noncompliant-session-id}}\",\n \"public\": false,\n \"measurement\": \"localizer\"\n}" + }, + "description": "This should make the session compliant" + }, + "response": [] + }, + { + "name": "[Session Template] Test session 2 is now marked as compliant", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": "tests[\"Status code is 200\"] = responseCode.code === 200;\n\ntests[\"Response time is less than 200ms\"] = responseTime < 200;\n\nvar jsonData = JSON.parse(responseBody);\n\ntests[\"Project has template marked\"] = jsonData.project_has_template === true;\n\ntests[\"Session marked as compliant\"] = jsonData.satisfies_template === true;" + } + } + ], + "request": { + "url": "{{baseUri}}/sessions/{{ST-noncompliant-session-id}}", + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "scitran-user {{test_user_api_key}}", + "description": "" + } + ], + "body": { + "mode": "raw", + "raw": "" + }, + "description": "" + }, + "response": [] + }, + { + "name": "[Session Template] Update session 2 to be non-compliant", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": "tests[\"Status code is 200\"] = responseCode.code === 200;\n\ntests[\"Response time is less than 200ms\"] = responseTime < 200;" + } + } + ], + "request": { + "url": "{{baseUri}}/sessions/{{ST-noncompliant-session-id}}?root=true", + "method": "PUT", + "header": [ + { + "key": "Authorization", + "value": "scitran-user {{test_user_api_key}}", + "description": "" + } + ], + "body": { + "mode": "raw", + "raw": " {\n \"subject\": {\n \"code\": \"bad-subject-code\"\n }\n }" + }, + "description": "Create a session that will NOT be compliant with the project-level session template" + }, + "response": [] + }, + { + "name": "[Session Template] Test session 2 is now marked as non-compliant", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": "tests[\"Status code is 200\"] = responseCode.code === 200;\n\ntests[\"Response time is less than 200ms\"] = responseTime < 200;\n\nvar jsonData = JSON.parse(responseBody);\n\ntests[\"Project has template marked\"] = jsonData.project_has_template === true;\n\ntests[\"Session marked as compliant\"] = jsonData.satisfies_template === false;" + } + } + ], + "request": { + "url": "{{baseUri}}/sessions/{{ST-noncompliant-session-id}}", + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "scitran-user {{test_user_api_key}}", + "description": "" + } + ], + "body": { + "mode": "raw", + "raw": "" + }, + "description": "" + }, + "response": [] } ] -} \ No newline at end of file +}