diff --git a/Makefile b/Makefile new file mode 100755 index 0000000000000000000000000000000000000000..6bda5ce8951f57b67f8ad6cce7cd3993d7d752ca --- /dev/null +++ b/Makefile @@ -0,0 +1,18 @@ +clean-pyc: + find . -name "*.pyc" | xargs rm -f + +clean-so: + find . -name "*.so" | xargs rm -f + find . -name "*.pyd" | xargs rm -f + +clean-build: + rm -rf _build + +clean-ctags: + rm -f tags + +clean-cache: + find . -name "__pycache__" | xargs rm -rf + +clean: clean-build clean-pyc clean-so clean-ctags clean-cache + diff --git a/__init__.py b/__init__.py deleted file mode 100644 index 1cb20092166f7a716554074a6fa785697b98b5b7..0000000000000000000000000000000000000000 --- a/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# @author: Gunnar Schaefer - -import api - -app = api.app diff --git a/api.wsgi b/api.wsgi index 6a4a35c34f979e6cec79b0dc4e0133083a37d322..5c6aa61609ae864cdb4d76e7932d65960030f0c2 100644 --- a/api.wsgi +++ b/api.wsgi @@ -23,9 +23,7 @@ import time import pymongo import argparse -import api -import centralclient -import jobs +from api import api, centralclient, jobs os.environ['PYTHON_EGG_CACHE'] = '/tmp/python_egg_cache' diff --git a/api/__init__.py b/api/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..83c202fd5a120a1e802c9a3b0b15e7bd632c9620 --- /dev/null +++ b/api/__init__.py @@ -0,0 +1,3 @@ +# @author: Gunnar Schaefer + +from .api import app, dispatcher # noqa diff --git a/acquisitions.py b/api/acquisitions.py similarity index 94% rename from acquisitions.py rename to api/acquisitions.py index 1b8e2d7a6ca8f35339e1a88c71fe29c6b73727d6..ebad23b26504c602887617ab8eb5a4366243547b 100644 --- a/acquisitions.py +++ b/api/acquisitions.py @@ -1,15 +1,11 @@ # @author: Gunnar Schaefer -import logging -log = logging.getLogger('scitran.api') - import bson -import scitran.data import scitran.data.medimg -import util -import containers +from . import util +from . import containers ACQUISITION_POST_SCHEMA = { '$schema': 'http://json-schema.org/draft-04/schema#', @@ -148,9 +144,9 @@ class Acquisition(containers.Container): def schema(self, *args, **kwargs): return super(Acquisition, self).schema(scitran.data.medimg.medimg.MedImgReader.acquisition_properties) - scitran.data.project_properties(ds_dict['project_type']) - scitran.data.session_properties(ds_dict['session_type']) - scitran.data.acquisition_properties(ds_dict['acquisition_type']) + # scitran.data.project_properties(ds_dict['project_type']) + # scitran.data.session_properties(ds_dict['session_type']) + # scitran.data.acquisition_properties(ds_dict['acquisition_type']) def get(self, aid): """Return one Acquisition, conditionally with details.""" diff --git a/api.py b/api/api.py similarity index 97% rename from api.py rename to api/api.py index 9fac4909b8bd270acc5571e88f680045d503e86c..915b3fd044af3945fe65061bac0ae3c8999f2600 100644 --- a/api.py +++ b/api/api.py @@ -5,14 +5,14 @@ import webapp2 import bson.json_util import webapp2_extras.routes -import apps -import core -import jobs -import users -import projects -import sessions -import acquisitions -import collections_ +from . import apps +from . import core +from . import jobs +from . import users +from . import projects +from . import sessions +from . import acquisitions +from . import collections_ routes = [ @@ -97,7 +97,7 @@ routes = [ ] -with open(os.path.join(os.path.dirname(__file__), 'schema.json')) as fp: +with open(os.path.join(os.path.dirname(__file__), 'data', 'schema.json')) as fp: schema_dict = json.load(fp) for cls in [ users.Group, diff --git a/apps.py b/api/apps.py similarity index 92% rename from apps.py rename to api/apps.py index 6c9548ef42a920400aeb9329ad22691347868c55..54fa2e48c3987de1b56a9b42cce1c83d3215d6ea 100644 --- a/apps.py +++ b/api/apps.py @@ -7,18 +7,14 @@ represents the /apps route """ import os -import json -import hashlib -import logging -import tarfile -import jsonschema +# import json +# import hashlib +# import tarfile +# import jsonschema -log = logging.getLogger('scitran.jobs') - -import tempdir as tempfile - -import base -import util +# from . import tempdir as tempfile +from . import base +# from .util import log, insert_app # TODO: create schemas to verify various json payloads APP_SCHEMA = { @@ -78,6 +74,7 @@ class Apps(base.RequestHandler): """Create a new App.""" # this handles receive and writing the file # but the the json validation and database is handled by util. + """ apps_path = self.app.config['apps_path'] if not apps_path: self.abort(503, 'POST api/apps unavailable. apps_path not defined') @@ -107,8 +104,11 @@ class Apps(base.RequestHandler): jsonschema.validate(app_meta, APP_SCHEMA) except (ValueError, jsonschema.ValidationError) as e: self.abort(400, str(e)) - util.insert_app(self.app.db, app_temp, apps_path, app_meta=app_meta) # pass meta info, prevent re-reading + insert_app(self.app.db, app_temp, apps_path, app_meta=app_meta) # pass meta info, prevent re-reading log.debug('Recieved App: %s' % app_meta.get('_id')) + """ + # XXX util.insert_app doesn't exist...? + raise NotImplementedError class App(base.RequestHandler): diff --git a/base.py b/api/base.py similarity index 98% rename from base.py rename to api/base.py index 9de76779096bd7d410d3e148f5d08205d9db28e5..0d6063c177b68ff0ac96cad3d203d24524167f71 100644 --- a/base.py +++ b/api/base.py @@ -1,16 +1,18 @@ # @author: Gunnar Schaefer, Kevin S. Hahn -import logging -log = logging.getLogger('scitran.api') -logging.getLogger('requests').setLevel(logging.WARNING) # silence Requests library logging - import copy import json +import logging import webapp2 import datetime import requests import jsonschema +from .util import log + +# silence Requests library logging +logging.getLogger('requests').setLevel(logging.WARNING) + class RequestHandler(webapp2.RequestHandler): diff --git a/centralclient.py b/api/centralclient.py similarity index 69% rename from centralclient.py rename to api/centralclient.py index 7025f2c8338d1a1e7f1e3c2801ab0c085e047431..2414f26a760e9e268b3e71d4c10f5c2784ca4ada 100755 --- a/centralclient.py +++ b/api/centralclient.py @@ -9,16 +9,16 @@ recieve information about other registered instances, and which of it's local users are permitted to access data in other instances. """ +import re +import json +import requests import logging import logging.config + logging.basicConfig() -log = logging.getLogger('centralclient') +log = logging.getLogger('scitran.api.centralclient') logging.getLogger('urllib3').setLevel(logging.WARNING) # silence Requests library logging -import re -import json -import requests - def update(db, api_uri, site_name, site_id, ssl_cert, central_url): """Send is-alive signal to central peer registry.""" @@ -80,38 +80,3 @@ def clean_remotes(db, site_id): log.debug('removing remotes from users, and remotes collection') db.sites.remove({'_id': {'$ne': [site_id]}}) db.users.update({'remotes': {'$exists': True}}, {'$unset': {'remotes': ''}}, multi=True) - - -if __name__ == '__main__': - import time - import pymongo - import argparse - - arg_parser = argparse.ArgumentParser() - arg_parser.add_argument('--central_url', help='Scitran Central API URL', default='https://sdmc.scitran.io') - arg_parser.add_argument('--db_uri', help='DB URI', required=True) - arg_parser.add_argument('--api_uri', help='API URL, with https:// prefix', required=True) - arg_parser.add_argument('--site_id', help='instance hostname (used as unique ID)', required=True) - arg_parser.add_argument('--site_name', help='instance name', nargs='+', required=True) - arg_parser.add_argument('--ssl_cert', help='path to server ssl certificate file', required=True) - arg_parser.add_argument('--sleeptime', default=60, type=int, help='time to sleep between is alive signals') - arg_parser.add_argument('--debug', help='enable default mode', action='store_true', default=False) - arg_parser.add_argument('--log_level', help='log level [info]', default='info') - args = arg_parser.parse_args() - args.site_name = ' '.join(args.site_name) if args.site_name else None # site_name as string - - logging.basicConfig() - log.setLevel(getattr(logging, args.log_level.upper())) - - db = (pymongo.MongoReplicaSetClient(args.db_uri) if 'replicaSet' in args.db_uri else pymongo.MongoClient(args.db_uri)).get_default_database() - - fail_count = 0 - while True: - if not update(db, args.api_uri, args.site_name, args.site_id, args.ssl_cert, args.central_url): - fail_count += 1 - else: - fail_count = 0 - if fail_count == 3: - log.debug('scitran central unreachable, purging all remotes info') - clean_remotes(db) - time.sleep(args.sleeptime) diff --git a/collections_.py b/api/collections_.py similarity index 98% rename from collections_.py rename to api/collections_.py index cb4003b11b69a885a98db306a2ef380ee1fced3a..d861d9f823cc845d7f0130ec6382c12d1e3445b3 100644 --- a/collections_.py +++ b/api/collections_.py @@ -1,17 +1,14 @@ # @author: Gunnar Schaefer -import logging -log = logging.getLogger('scitran.api') - import bson import datetime import jsonschema -import users -import util -import containers -import sessions -import acquisitions +from . import users +from . import util +from . import containers +from . import sessions +from . import acquisitions COLLECTION_POST_SCHEMA = { '$schema': 'http://json-schema.org/draft-04/schema#', diff --git a/containers.py b/api/containers.py similarity index 99% rename from containers.py rename to api/containers.py index a144ca6f1266d7f7829faae612a350c4bdc3c5a1..d8509d9c65886142e6935a443700cc339b85d28c 100644 --- a/containers.py +++ b/api/containers.py @@ -1,8 +1,5 @@ # @author: Gunnar Schaefer, Kevin S. Hahn -import logging -log = logging.getLogger('scitran.api') - import os import cgi import bson @@ -14,9 +11,10 @@ import jsonschema import tempdir as tempfile -import base -import util -import users +from . import base +from . import util +from .util import log +from . import users FILE_SCHEMA = { diff --git a/core.py b/api/core.py similarity index 99% rename from core.py rename to api/core.py index 41ba2bd1658d2fba323f13876f783deee92e7309..563f454981e11d42e6f8d3bcf2e3daa077d3879f 100644 --- a/core.py +++ b/api/core.py @@ -1,9 +1,6 @@ # @author: Gunnar Schaefer, Kevin S. Hahn import logging -log = logging.getLogger('scitran.api') -logging.getLogger('MARKDOWN').setLevel(logging.WARNING) # silence Markdown library logging - import os import re import cgi @@ -18,10 +15,14 @@ import markdown import cStringIO import jsonschema -import base -import util -import users -import tempdir as tempfile +from . import base +from . import util +from .util import log +from . import users +from . import tempdir as tempfile + +# silence Markdown library logging +logging.getLogger('MARKDOWN').setLevel(logging.WARNING) UPLOAD_SCHEMA = { '$schema': 'http://json-schema.org/draft-04/schema#', diff --git a/schema.json b/api/data/schema.json similarity index 100% rename from schema.json rename to api/data/schema.json diff --git a/jobs.py b/api/jobs.py similarity index 99% rename from jobs.py rename to api/jobs.py index 6015abde5c1760bd3af17587e737644096662de8..e88d730476dd357f0133244f8116b89ebf8b1f94 100644 --- a/jobs.py +++ b/api/jobs.py @@ -5,14 +5,14 @@ API request handlers for process-job-handling. """ import logging -log = logging.getLogger('scitran.jobs') +log = logging.getLogger('scitran.api.jobs') import bson import pymongo import datetime -import base -import util +from . import base +from . import util JOB_STATES = [ 'pending', # Job is queued diff --git a/projects.py b/api/projects.py similarity index 98% rename from projects.py rename to api/projects.py index 3915909c015148d87901740e73e1e97c798801b5..b54b5a07d1f44e6edec8fe55fe8558ab51b7f843 100644 --- a/projects.py +++ b/api/projects.py @@ -1,16 +1,13 @@ # @author: Gunnar Schaefer -import logging -log = logging.getLogger('scitran.api') - import bson import datetime import scitran.data.medimg -import util -import users -import containers +from . import util +from . import users +from . import containers PROJECT_POST_SCHEMA = { '$schema': 'http://json-schema.org/draft-04/schema#', diff --git a/sessions.py b/api/sessions.py similarity index 98% rename from sessions.py rename to api/sessions.py index 7cfecdaf636f6b20ff1e27f47ebd0278f40e855d..e8bd51e07b99c456d0084da40801e99fb808f438 100644 --- a/sessions.py +++ b/api/sessions.py @@ -1,14 +1,11 @@ # @author: Gunnar Schaefer -import logging -log = logging.getLogger('scitran.api') - import bson import scitran.data.medimg -import util -import containers +from . import util +from . import containers SESSION_POST_SCHEMA = { '$schema': 'http://json-schema.org/draft-04/schema#', diff --git a/tempdir.py b/api/tempdir.py similarity index 100% rename from tempdir.py rename to api/tempdir.py diff --git a/users.py b/api/users.py similarity index 99% rename from users.py rename to api/users.py index b0bf691601e63bc2aa6c585ef6380c1025ea2157..201a4a6d1488c18c8c31462a2f744a6e9d89176c 100644 --- a/users.py +++ b/api/users.py @@ -1,16 +1,12 @@ # @author: Gunnar Schaefer -import logging -log = logging.getLogger('scitran.api') - -import copy import hashlib import pymongo import datetime import jsonschema -import base -import util +from . import base +from . import util ROLES = [ { diff --git a/util.py b/api/util.py similarity index 91% rename from util.py rename to api/util.py index 6be85a04fd63fe97fa60fb9554d5bb0e0a803d4e..908c361dc7a7e0f5cb7635d715c0a94f9957f1dc 100644 --- a/util.py +++ b/api/util.py @@ -1,8 +1,5 @@ # @author: Gunnar Schaefer -import logging -log = logging.getLogger('scitran.api') - import os import copy import pytz @@ -15,9 +12,15 @@ import datetime import mimetypes import dateutil.parser import tempdir as tempfile +import logging import scitran.data +log_fmt = '%(asctime)s %(message)s' +datefmt = '%Y-%m-%d %H:%M:%S' +log = logging.getLogger('scitran.api') +logging.basicConfig(format=log_fmt, datefmt=datefmt, level=logging.INFO) + MIMETYPES = [ ('.bvec', 'text', 'bvec'), ('.bval', 'text', 'bval'), @@ -80,6 +83,17 @@ def commit_file(dbc, _id, datainfo, filepath, data_path, force=False): """Insert a file as an attachment or as a file.""" filename = os.path.basename(filepath) fileinfo = datainfo['fileinfo'] + log_path = os.path.join(os.path.split(data_path)[0], 'logs') + # XXX Eventually it would be good to find a more principled/constant + # way to do this + if os.path.isdir(log_path): + hdlr = logging.FileHandler(os.path.join(log_path, 'commit_file.log')) + fmt = logging.Formatter(fmt=log_fmt, datefmt=datefmt) + hdlr.setFormatter(fmt) + hdlr.setLevel(logging.DEBUG) + log.addHandler(hdlr) + else: + hdlr = None log.info('Sorting %s' % filename) if _id is None: _id = _update_db(dbc.database, datainfo) @@ -110,6 +124,8 @@ def commit_file(dbc, _id, datainfo, filepath, data_path, force=False): dbc.update_one({'_id': _id}, {'$push': {'files': fileinfo}}) updated = True log.debug('Done %s' % filename) + if hdlr is not None: + log.removeHandler(hdlr) return updated @@ -121,6 +137,7 @@ def _update_db(db, datainfo): session = db.sessions.find_one(session_spec, ['project']) if session: # skip project creation, if session exists project = db.projects.find_one({'_id': session['project']}, projection=PROJECTION_FIELDS + ['name']) + group = db.groups.find_one({'_id': project['group']}, projection=PROJECTION_FIELDS + ['name']) else: existing_group_ids = [g['_id'] for g in db.groups.find(None, ['_id'])] group_id_matches = difflib.get_close_matches(datainfo['group_id'], existing_group_ids, cutoff=0.8) @@ -150,6 +167,12 @@ def _update_db(db, datainfo): new=True, projection=PROJECTION_FIELDS, ) + try: + session_label = session['label'] + except KeyError: # this can happen if nims_metadata_status == None + session_label = '(unknown)' + log.info('Using group_id="%s", project_name="%s", and session_label="%s"' + % (project['group'], project['name'], session_label)) acquisition_spec = {'uid': datainfo['acquisition_id']} acquisition = db.acquisitions.find_and_modify( acquisition_spec, diff --git a/bootstrap.py b/bin/bootstrap.py similarity index 92% rename from bootstrap.py rename to bin/bootstrap.py index 589740d7a15b410bf24e9cc818f5aa23bb6c63c8..5e2c2b9b3f6d8b3ab65de48f819c3aced84af821 100755 --- a/bootstrap.py +++ b/bin/bootstrap.py @@ -1,24 +1,19 @@ #!/usr/bin/env python # # @author: Gunnar Schaefer - -import logging -logging.basicConfig( - format='%(asctime)s %(message)s', - datefmt='%Y-%m-%d %H:%M:%S', - level=logging.INFO, -) -log = logging.getLogger('scitran.bootstrap') +"""This script helps bootstrap data""" import os import json -import time import hashlib +import logging import pymongo import argparse import datetime -import util +from api import util # from scitran.api import util + +log = logging.getLogger('scitran.api.bootstrap') def dbinit(args): @@ -67,7 +62,7 @@ def dbinit(args): dbinit_desc = """ example: -./scripts/bootstrap.py dbinit mongodb://cnifs.stanford.edu/nims?replicaSet=cni -j nims_users_and_groups.json +./bin/bootstrap.py dbinit mongodb://cnifs.stanford.edu/nims?replicaSet=cni -j nims_users_and_groups.json """ @@ -104,7 +99,7 @@ def sort(args): sort_desc = """ example: -./scripts/bootstrap.py sort mongodb://localhost/nims /tmp/data /tmp/sorted +./bin/bootstrap.py sort mongodb://localhost/nims /tmp/data /tmp/sorted """ diff --git a/bin/centralclient.py b/bin/centralclient.py new file mode 100644 index 0000000000000000000000000000000000000000..42aab241acc9ff68630eed903b1f71e645a634f4 --- /dev/null +++ b/bin/centralclient.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +import logging +import time +import pymongo +import argparse + +from api.centralclient import log, update, clean_remotes + +arg_parser = argparse.ArgumentParser() +arg_parser.add_argument('--central_url', help='Scitran Central API URL', default='https://sdmc.scitran.io') +arg_parser.add_argument('--db_uri', help='DB URI', required=True) +arg_parser.add_argument('--api_uri', help='API URL, with https:// prefix', required=True) +arg_parser.add_argument('--site_id', help='instance hostname (used as unique ID)', required=True) +arg_parser.add_argument('--site_name', help='instance name', nargs='+', required=True) +arg_parser.add_argument('--ssl_cert', help='path to server ssl certificate file', required=True) +arg_parser.add_argument('--sleeptime', default=60, type=int, help='time to sleep between is alive signals') +arg_parser.add_argument('--debug', help='enable default mode', action='store_true', default=False) +arg_parser.add_argument('--log_level', help='log level [info]', default='info') +args = arg_parser.parse_args() +args.site_name = ' '.join(args.site_name) if args.site_name else None # site_name as string + +logging.basicConfig() +log.setLevel(getattr(logging, args.log_level.upper())) + +db = (pymongo.MongoReplicaSetClient(args.db_uri) + if 'replicaSet' in args.db_uri else + pymongo.MongoClient(args.db_uri)).get_default_database() + +fail_count = 0 +while True: + if not update(db, args.api_uri, args.site_name, args.site_id, + args.ssl_cert, args.central_url): + fail_count += 1 + else: + fail_count = 0 + if fail_count == 3: + log.debug('scitran central unreachable, purging all remotes info') + clean_remotes(db) + time.sleep(args.sleeptime)