Merge pull request #593 from scitran/resolver

Server-side resolver

api/api.py

@@ -21,6 +21,7 @@ from .handlers import userhandler
 from .handlers import grouphandler
 from .handlers import containerhandler
 from .handlers import collectionshandler
+from .handlers import resolvehandler
 from .handlers import searchhandler
 from .handlers import schemahandler
 from .handlers import reporthandler
@@ -99,6 +100,7 @@ routes = [
         webapp2.Route(r'/engine',           upload.Upload, handler_method='engine', methods=['POST']),
         webapp2.Route(r'/sites',            centralclient.CentralClient, handler_method='sites', methods=['GET']),
         webapp2.Route(r'/register',         centralclient.CentralClient, handler_method='register', methods=['POST']),
+        webapp2.Route(r'/resolve',         resolvehandler.ResolveHandler, handler_method='resolve', methods=['POST']),
         webapp2.Route(r'/config',           Config, methods=['GET']),
         webapp2.Route(r'/config.js',        Config, handler_method='get_js', methods=['GET']),
         webapp2.Route(r'/version',          Version, methods=['GET']),

api/handlers/resolvehandler.py

+"""
+API request handlers for the jobs module
+"""
+
+from .. import base
+from ..resolver import Resolver
+
+class ResolveHandler(base.RequestHandler):
+
+    """Provide /resolve API route."""
+
+    def resolve(self):
+        """Resolve a path through the hierarchy."""
+
+        if self.public_request:
+            self.abort(403, 'Request requires login')
+
+        doc = self.request.json
+        result = Resolver.resolve(doc['path'])
+
+        # Cancel the request if anything in the path is unauthorized; remove any children that are unauthorized.
+        if not self.superuser_request:
+            for x in result["path"]:
+                ok = False
+                if x['node_type'] in ['acquisition', 'session', 'project', 'group']:
+                    perms = x.get('roles', []) + x.get('permissions', [])
+                    for y in perms:
+                        if y.get('_id') == self.uid:
+                            ok = True
+                            break
+
+                    if not ok:
+                        self.abort(403, "Not authorized")
+
+            filtered_children = []
+            for x in result["children"]:
+                ok = False
+                if x['node_type'] in ['acquisition', 'session', 'project', 'group']:
+                    perms = x.get('roles', []) + x.get('permissions', [])
+                    for y in perms:
+                        if y.get('_id') == self.uid:
+                            ok = True
+                            break
+                else:
+                    ok = True
+
+                if ok:
+                    filtered_children.append(x)
+
+            result["children"] = filtered_children
+
+        return result

api/resolver.py

+"""
+Resolve an ambiguous path through the data hierarchy.
+"""
+
+from . import config
+
+class Node(object):
+
+    # All lists obtained by the Resolver are sorted by the created timestamp, then the database ID as a fallback.
+    # As neither property should ever change, this sort should be consistent
+    sorting = [('created', 1), ('_id', 1)]
+
+    # Globally disable extraneous properties of unbounded length.
+    projection = {'files': 0}
+    # Version of same for debugging purposes.
+    # projection = {'roles': 0, 'permissions': 0, 'files': 0}
+
+    @staticmethod
+    def get_children(parent):
+        raise NotImplementedError()
+
+    @staticmethod
+    def filter(children, criterion):
+        raise NotImplementedError()
+
+
+def _pipeline(table, pipeline):
+    """
+    Temporary philosophical dupe with reporthandler.py.
+    Execute a mongo pipeline, check status, return results.
+    A workaround for wonky pymongo aggregation behavior.
+    """
+
+    output = config.db.command('aggregate', table, pipeline=pipeline)
+    result = output.get('result')
+
+    if output.get('ok') != 1.0 or result is None:
+        raise Exception()
+
+    return result
+
+def _get_files(table, match):
+    """
+    Return a consistently-ordered set of files for a given container query.
+    """
+
+    pipeline = [
+        {'$match': match },
+        {'$unwind': '$files'},
+        {'$sort': {'files.name': 1}},
+        {'$group': {'_id':'$_id', 'files': {'$push':'$files'}}}
+    ]
+
+    result = _pipeline(table, pipeline)
+    if len(result) == 0:
+        return []
+
+    files = result[0]['files']
+    for x in files:
+        x.update({'node_type': 'file'})
+    return files
+
+def _get_docs(table, label, match):
+    results = list(config.db[table].find(match, Node.projection, sort=Node.sorting))
+    for y in results:
+        y.update({'node_type': label})
+    return results
+
+
+class FileNode(Node):
+    @staticmethod
+    def get_children(parent):
+        return []
+
+    @staticmethod
+    def filter(children, criterion):
+        raise Exception("Files have no children")
+
+class AcquisitionNode(Node):
+    @staticmethod
+    def get_children(parent):
+        files    = _get_files('acquisitions', {'_id' : parent['_id'] })
+
+        return files
+
+    @staticmethod
+    def filter(children, criterion):
+        for x in children:
+            if x['node_type'] == "file" and x.get('name') == criterion:
+                return x, FileNode
+        raise Exception('No ' + criterion + ' acquisition or file found.')
+
+class SessionNode(Node):
+
+    @staticmethod
+    def get_children(parent):
+        acqs = _get_docs('acquisitions', 'acquisition', {'session' : parent['_id']})
+        files    = _get_files('sessions', {'_id' : parent['_id'] })
+
+        return list(acqs) + files
+
+    @staticmethod
+    def filter(children, criterion):
+        for x in children:
+            if x['node_type'] == "acquisition" and x.get('label') == criterion:
+                return x, AcquisitionNode
+            if x['node_type'] == "file" and x.get('name') == criterion:
+                return x, FileNode
+        raise Exception('No ' + criterion + ' acquisition or file found.')
+
+class ProjectNode(Node):
+
+    @staticmethod
+    def get_children(parent):
+        sessions = _get_docs('sessions', 'session', {'project' : parent['_id']})
+        files    = _get_files('projects', {'_id' : parent['_id'] })
+
+        return list(sessions) + files
+
+    @staticmethod
+    def filter(children, criterion):
+        for x in children:
+            if x['node_type'] == "session" and x.get('label') == criterion:
+                return x, SessionNode
+            if x['node_type'] == "file" and x.get('name') == criterion:
+                return x, FileNode
+        raise Exception('No ' + criterion + ' session or file found.')
+
+class GroupNode(Node):
+
+    @staticmethod
+    def get_children(parent):
+        projects = _get_docs('projects', 'project', {'group' : parent['_id']})
+        return projects
+
+    @staticmethod
+    def filter(children, criterion):
+        for x in children:
+            if x.get('label') == criterion:
+                return x, ProjectNode
+        raise Exception('No ' + criterion + ' project found.')
+
+class RootNode(Node):
+
+    @staticmethod
+    def get_children(parent):
+        groups = _get_docs('groups', 'group', {})
+        return groups
+
+    @staticmethod
+    def filter(children, criterion):
+        for x in children:
+            if x.get('_id') == criterion:
+                return x, GroupNode
+        raise Exception('No ' + criterion + ' group found.')
+
+
+class Resolver(object):
+    """
+    Given an array of human-meaningful, possibly-ambiguous strings, resolve it as a path through the hierarchy.
+
+    Does not tolerate ambiguity at any level of the path except the final node.
+    """
+
+    @staticmethod
+    def resolve(path):
+
+        if not isinstance(path, list):
+            raise Exception("Path must be an array of strings")
+
+        node, resolved, last = Resolver._resolve(path, RootNode)
+        children = node.get_children(last)
+
+        return {
+            'path': resolved,
+            'children': children
+        }
+
+    @staticmethod
+    def _resolve(path, node, parents=None):
+
+        if parents is None:
+            parents = []
+
+        last = None
+        if len(parents) > 0:
+            last = parents[len(parents) - 1]
+
+        if len(path) == 0:
+            return node, parents, last
+
+        current  = path[0]
+        children = node.get_children(last)
+        selected, next_ = node.filter(children, current)
+
+        path = path[1:]
+        parents.append(selected)
+
+        return Resolver._resolve(path, next_, parents)

test/bin/lint.sh

@@ -13,7 +13,7 @@ echo "Checking for files with windows-style newlines:"
 
 echo "Running pylint ..."
 # TODO: Enable Refactor and Convention reports
-pylint --reports=no --disable=C,R api
+pylint --reports=no --disable=C,R,W0312 api
 
 #echo
 #

test/integration_tests/python/basics.py

+# New test fixtures!
+#
+# The intention is to slowly build these up until we like them, then port the tests to them and replace parts of conftest.py
+
+import pytest
+import requests
+
+# The request Session object has no support for a base URL; this is a subclass to embed one.
+# Dynamically generated via a pytest fixture so that we can use the upstream fixtures.
+@pytest.fixture(scope="module")
+def base_url_session(base_url):
+    class BaseUrlSession(requests.Session):
+        def request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=None, allow_redirects=True, proxies=None, hooks=None, stream=None, verify=None, cert=None, json=None):
+
+            url = base_url + url
+
+            return super(BaseUrlSession, self).request(method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
+
+    return BaseUrlSession
+
+
+_apiAsAdmin = None
+
+# A replacement for the RequestsAccessor class. Less boilerplate, no kwarg fiddling.
+# This has the added benefit of re-using HTTP connections, which might speed up our testing considerably.
+#
+# Ref: http://stackoverflow.com/a/34491383
+@pytest.fixture(scope="module")
+def as_admin(base_url_session):
+
+    global _apiAsAdmin
+
+    # Create one session and reuse it.
+    if _apiAsAdmin is None:
+
+        s = base_url_session()
+
+        s.headers.update({
+            "Authorization":"scitran-user XZpXI40Uk85eozjQkU1zHJ6yZHpix+j0mo1TMeGZ4dPzIqVPVGPmyfeK"
+        })
+        s.params.update({
+            "root": "true"
+        })
+
+        _apiAsAdmin = s
+
+    return _apiAsAdmin

test/integration_tests/python/conftest.py

@@ -7,6 +7,14 @@ import pymongo
 import requests
 
 
+# Pytest considers fixtures to be provided by "plugins", which are generally provided by
+# files called conftest.py. This prevents us from placing module-level fixture logic in
+# well-organized files. To fix this, we simply star-import from files that we need.
+#
+# Ref: http://pytest.org/2.2.4/plugins.html
+from basics import *
+from states import *
+
 @pytest.fixture(scope="session")
 def bunch():
     class BunchFactory:

test/integration_tests/python/states.py

+# Various wholesale app states that might be useful in your tests
+
+import time
+import pytest
+
+# Currently a dupe from test_uploads.py.
+# Could not understand why this doesn't work if I remove the original; future work needed here.
+@pytest.fixture(scope="module")
+def with_hierarchy_and_file_data(api_as_admin, bunch, request, data_builder):
+    group =         data_builder.create_group('test_upload_' + str(int(time.time() * 1000)))
+    project =       data_builder.create_project(group)
+    session =       data_builder.create_session(project)
+    acquisition =   data_builder.create_acquisition(session)
+
+    file_names = ['one.csv', 'two.csv']
+    files = {}
+    for i, name in enumerate(file_names):
+        files['file' + str(i+1)] = (name, 'some,data,to,send\nanother,row,to,send\n')
+
+    def teardown_db():
+        data_builder.delete_acquisition(acquisition)
+        data_builder.delete_session(session)
+        data_builder.delete_project(project)
+        data_builder.delete_group(group)
+
+    request.addfinalizer(teardown_db)
+
+    fixture_data = bunch.create()
+    fixture_data.group = group
+    fixture_data.project = project
+    fixture_data.session = session
+    fixture_data.acquisition = acquisition
+    fixture_data.files = files
+    return fixture_data

test/integration_tests/python/test_resolver.py

+import json
+import logging
+
+log = logging.getLogger(__name__)
+sh = logging.StreamHandler()
+log.addHandler(sh)
+
+
+def test_resolver_root(as_admin, with_hierarchy_and_file_data):
+    r = as_admin.post('/resolve', json={'path': []})
+    assert r.ok
+
+    result   = r.json()
+    path     = result['path']
+    children = result['children']
+
+    # root node should not walk
+    assert len(path) == 0
+
+    # should be 3 groups
+    assert len(children) == 3
+
+    for node in children:
+        assert node['node_type'] == 'group'
+
+
+def test_resolver_group(as_admin, with_hierarchy_and_file_data):
+    r = as_admin.post('/resolve', json={'path': [ 'scitran' ]})
+    assert r.ok
+
+    result   = r.json()
+    path     = result['path']
+    children = result['children']
+
+    # group node is one down from root
+    assert len(path) == 1
+
+    # should be no children
+    assert len(children) == 0