From 451dfc1224f4031b76ff3375ec09badeb480ac83 Mon Sep 17 00:00:00 2001
From: "Kevin S. Hahn" <kevinshahn@gmail.com>
Date: Mon, 6 Apr 2015 18:03:11 -0700
Subject: [PATCH] create jobs based on available compatible app
---
util.py | 139 ++++++++++++++++++++++++++++----------------------------
1 file changed, 69 insertions(+), 70 deletions(-)
diff --git a/util.py b/util.py
index 6f0b53d2..bb903536 100644
--- a/util.py
+++ b/util.py
@@ -4,10 +4,12 @@ import logging
log = logging.getLogger('scitran.api')
import os
+import json
import bson
import copy
import shutil
import difflib
+import tarfile
import datetime
import mimetypes
import tempdir as tempfile
@@ -151,81 +153,78 @@ def _update_db(db, dataset):
# TODO: create job should be use-able from bootstrap.py with only database information
def create_job(dbc, dataset):
- db = dbc.database
- type_ = dataset.nims_file_type
- kinds_ = dataset.nims_file_kinds
- state_ = dataset.nims_file_state
- app = None
- # TODO: check if there are 'default apps' set for this project/session/acquisition
- acquisition = db.acquisitions.find_one({'uid': dataset.nims_acquisition_id})
- session = db.sessions.find_one({'_id': bson.ObjectId(acquisition.get('session'))})
- project = db.projects.find_one({'_id': bson.ObjectId(session.get('project'))})
- aid = acquisition.get('_id')
+ db = dbc.database
+ type_ = dataset.nims_file_type
+ kinds_ = dataset.nims_file_kinds
+ state_ = dataset.nims_file_state
+ app = None
+ # TODO: check if there are 'default apps' set for this project/session/acquisition
+ acquisition = db.acquisitions.find_one({'uid': dataset.nims_acquisition_id})
+ session = db.sessions.find_one({'_id': bson.ObjectId(acquisition.get('session'))})
+ project = db.projects.find_one({'_id': bson.ObjectId(session.get('project'))})
+ aid = acquisition.get('_id')
- # job descriptions have a few special cases
- # XXX: outputs can specify to __INHERIT__ a value from the parent input file, for ex: kinds
- # XXX: if an input kinds = None, then that job is meant to work on any file kinds
- # otherwise, kinds will restrict the app to work on specific kinds only
- app = db.apps.find_one({
- '$or': [
- {'inputs': {'$elemMatch': {'type': type_, 'state': state_, 'kinds': kinds_}}, 'default': True},
- {'inputs': {'$elemMatch': {'type': type_, 'state': state_, 'kinds': None}}, 'default': True},
- ],
- })
- # TODO: this has to move...
- # force acquisition dicom file to be marked as 'optional = True'
- db.acquisitions.find_and_modify(
- {'uid': dataset.nims_acquisition_id, 'files.type': 'dicom'},
- {'$set': {'files.$.optional': True}},
- )
+ # XXX: if an input kinds = None, then that job is meant to work on any file kinds
+ app = db.apps.find_one({
+ '$or': [
+ {'inputs': {'$elemMatch': {'type': type_, 'state': state_, 'kinds': kinds_}}, 'default': True},
+ {'inputs': {'$elemMatch': {'type': type_, 'state': state_, 'kinds': None}}, 'default': True},
+ ],
+ })
+ # TODO: this has to move...
+ # force acquisition dicom file to be marked as 'optional = True'
+ db.acquisitions.find_and_modify(
+ {'uid': dataset.nims_acquisition_id, 'files.type': 'dicom'},
+ {'$set': {'files.$.optional': True}},
+ )
- if not app:
- log.info('no app for type=%s, state=%s, kinds=%s, default=True. no job created.' % (type_, state_, kinds_))
- else:
- # replace any special values in the app description
- for output in app['outputs']:
- if output['kinds'] == '__INHERIT__':
- output['kinds'] = kinds_
+ if not app:
+ log.info('no app for type=%s, state=%s, kinds=%s, default=True. no job created.' % (type_, state_, kinds_))
+ else:
+ # XXX: outputs can specify to __INHERIT__ a value from the parent input file, for ex: kinds
+ for output in app['outputs']:
+ if output['kinds'] == '__INHERIT__':
+ output['kinds'] = kinds_
- # TODO: job description needs more metadata to be searchable in a useful way
- output_url = '%s/%s/%s' % ('acquisitions', aid, 'file')
- job = db.jobs.find_and_modify(
- {
- '_id': db.jobs.count() + 1,
+ # TODO: job description needs more metadata to be searchable in a useful way
+ output_url = '%s/%s/%s' % ('acquisitions', aid, 'file')
+ job = db.jobs.find_and_modify(
+ {
+ '_id': db.jobs.count() + 1,
+ },
+ {
+ '_id': db.jobs.count() + 1,
+ 'group': project.get('group_id'),
+ 'project': {
+ '_id': project.get('_id'),
+ 'name': project.get('name'),
},
- {
- '_id': db.jobs.count() + 1,
- 'group': project.get('group_id'),
- 'project': {
- '_id': project.get('_id'),
- 'name': project.get('name'),
- },
- 'exam': session.get('exam'),
- 'app': {
- '_id': app['_id'],
- 'type': 'docker',
- },
- 'inputs': [
- {
- 'filename': dataset.nims_file_name + dataset.nims_file_ext,
- 'url': '%s/%s/%s' % ('acquisitions', aid, 'file'),
- 'payload': {
- 'type': dataset.nims_file_type,
- 'state': dataset.nims_file_state,
- 'kinds': dataset.nims_file_kinds,
- },
- }
- ],
- 'outputs': [{'url': output_url, 'payload': i} for i in app['outputs']],
- 'status': 'pending',
- 'activity': None,
- 'added': datetime.datetime.now(),
- 'timestamp': datetime.datetime.now(),
+ 'exam': session.get('exam'),
+ 'app': {
+ '_id': app['_id'],
+ 'type': 'docker',
},
- upsert=True,
- new=True,
- )
- log.info('created job %d, group: %s, project %s' % (job['_id'], job['group'], job['project']))
+ 'inputs': [
+ {
+ 'filename': dataset.nims_file_name + dataset.nims_file_ext,
+ 'url': '%s/%s/%s' % ('acquisitions', aid, 'file'),
+ 'payload': {
+ 'type': dataset.nims_file_type,
+ 'state': dataset.nims_file_state,
+ 'kinds': dataset.nims_file_kinds,
+ },
+ }
+ ],
+ 'outputs': [{'url': output_url, 'payload': i} for i in app['outputs']],
+ 'status': 'pending',
+ 'activity': None,
+ 'added': datetime.datetime.now(),
+ 'timestamp': datetime.datetime.now(),
+ },
+ upsert=True,
+ new=True,
+ )
+ log.info('created job %d, group: %s, project %s' % (job['_id'], job['group'], job['project']))
def _entity_metadata(dataset, properties, metadata={}, parent_key=''):
--
GitLab