Temp hacks

api/api.py

@@ -58,6 +58,7 @@ routes = [
         webapp2.Route(r'/reaper',           core.Core, handler_method='reaper', methods=['POST']),
         webapp2.Route(r'/sites',            core.Core, handler_method='sites', methods=['GET']),
         webapp2.Route(r'/config',           core.Config, methods=['GET']),
+        webapp2.Route(r'/hack',             core.Core, handler_method='hack',   methods=['GET']),
         webapp2.Route(r'/config.js',        core.Config, handler_method='get_js', methods=['GET'])
     ]),
     webapp2.Route(r'/api/users',            userhandler.UserHandler, handler_method='get_all', methods=['GET']),

api/core.py

@@ -14,6 +14,8 @@ from . import base
 from . import files
 from . import util
 from . import config
+from . import jobs
+from . import rules
 from .dao import reaperutil
 from . import tempdir as tempfile
 
@@ -41,6 +43,52 @@ class Core(base.RequestHandler):
         """Return 200 OK."""
         pass
 
+    def hack(self):
+        for c_type in ['projects', 'collections', 'sessions', 'acquisitions']:
+
+           # This projection needs every field required to know what type of container it is & navigate to its project
+           containers = config.db[c_type].find({'files.unprocessed': True}, ['files', 'session', 'project'])
+
+           for c in containers:
+                for f in c['files']:
+                    if f.get('unprocessed'):
+                        rules.create_jobs(config.db, c, c_type, f)
+                        r = config.db[c_type].update_one(
+                                {
+                                    '_id': c['_id'],
+                                    'files': {
+                                        '$elemMatch': {
+                                            'name': f['name'],
+                                            'hash': f['hash'],
+                                        },
+                                    },
+                                },
+                                {
+                                    '$set': {
+                                        'files.$.unprocessed': False,
+                                    },
+                                },
+                                )
+                        if not r.matched_count:
+                            log.info('file modified or removed, not marked as clean: %s %s, %s' % (c_type, c, f['name']))
+        while True:
+            j = config.db.jobs.find_one_and_update(
+                {
+                    'state': 'running',
+                    'modified': {'$lt': datetime.datetime.utcnow() - datetime.timedelta(seconds=100)},
+                },
+                {
+                    '$set': {
+                        'state': 'failed',
+                    },
+                },
+                )
+            if j is None:
+                break
+            else:
+                jobs.retry_job(config.db, j)
+
+
     def get(self):
         """Return API documentation"""
         resources = """

api/files.py

@@ -91,6 +91,7 @@ class FileStore(object):
 
     def _save_multipart_file(self, dest_path, hash_alg):
         form = getHashingFieldStorage(dest_path, hash_alg)(fp=self.body, environ=self.environ, keep_blank_values=True)
+
         self.received_file = form['file'].file
         self.filename = os.path.basename(form['file'].filename)
         self.tags = json.loads(form['tags'].file.getvalue()) if 'tags' in form else None

api/jobs.py

@@ -22,7 +22,6 @@ JOB_STATES = [
     'running',  # Job has been handed to an engine and is being processed
     'failed',   # Job has an expired heartbeat (orphaned) or has suffered an error
     'complete', # Job has successfully completed
-
 ]
 
 JOB_STATES_ALLOWED_MUTATE = [
@@ -169,7 +168,7 @@ def generate_formula(algorithm_id, i):
                 'location': '/input/' + i['filename'],
             }
         ],
-        'transform': {
+        'target': {
             'command': [ 'echo', 'No command specified for ' + algorithm_id],
             'env': { },
             'dir': "/",
@@ -186,11 +185,11 @@ def generate_formula(algorithm_id, i):
 
     if algorithm_id == 'dcm2nii':
         f['inputs'][0]['uri'] = '/opt/flywheel-temp/dcm_convert-0.1.1.tar'
-        f['transform']['command'] = ['bash', '-c', 'mkdir /output; /scripts/run /input/' + i['filename'] + ' /output/' + i['filename'].split('_')[0]]
+        f['target']['command'] = ['bash', '-c', 'mkdir /output; /scripts/run /input/' + i['filename'] + ' /output/' + i['filename'].split('_')[0]]
 
     elif algorithm_id == 'qa':
         f['inputs'][0]['uri'] = '/opt/flywheel-temp/qa-report-fmri-0.0.2.tar'
-        f['transform']['command'] = ['bash', '-c', 'mkdir /output; /scripts/run; exit 0']
+        f['target']['command'] = ['bash', '-c', 'mkdir /output; /scripts/run; exit 0']
     else:
         raise Exception('Command for algorithm ' + algorithm_id + ' not specified')