Add a menagerie of new gear features!

api/api.py

@@ -142,6 +142,7 @@ endpoints = [
             route('/<:[^/]+>/logs',        JobHandler,  h='add_logs',      m=['POST']),
         ]),
         route('/gears',                                  GearsHandler),
+        route('/gears/check',                            GearsHandler, h='check', m=['POST']),
         route('/gears/temp',                             GearHandler, h='upload', m=['POST']),
         route('/gears/temp/<cid:{cid}>',  GearHandler, h='download', m=['GET']),
         prefix('/gears', [

api/jobs/gears.py

@@ -169,6 +169,11 @@ def remove_gear(_id):
         raise Exception("Deleted failed " + str(result.raw_result))
 
 def upsert_gear(doc):
+    check_for_gear_insertion(doc)
+
+    return insert_gear(doc)
+
+def check_for_gear_insertion(doc):
     gear_tools.validate_manifest(doc['gear'])
 
     # Remove previous gear if name & version combo already exists
@@ -179,6 +184,4 @@ def upsert_gear(doc):
     })
 
     if conflict is not None:
-        raise Exception('Gear ' + doc['gear']['name'] + ' ' + doc['gear']['version'] + ' already exists')
-
-    return insert_gear(doc)
+        raise Exception('Gear "' + doc['gear']['name'] + '" version "' + doc['gear']['version'] + '" already exists, consider changing the version string.')

api/jobs/handlers.py

@@ -19,7 +19,7 @@ from .. import config
 from . import batch
 from ..validators import validate_data, verify_payload_exists
 
-from .gears import validate_gear_config, get_gears, get_gear, get_invocation_schema, remove_gear, upsert_gear, suggest_container, get_gear_by_name
+from .gears import validate_gear_config, get_gears, get_gear, get_invocation_schema, remove_gear, upsert_gear, suggest_container, get_gear_by_name, check_for_gear_insertion
 from .jobs import Job, Logs
 from .batch import check_state, update
 from .queue import Queue
@@ -43,6 +43,17 @@ class GearsHandler(base.RequestHandler):
 
         return gears
 
+    def check(self):
+        """
+        Check if a gear upload is likely to succeed.
+        """
+
+        if self.public_request:
+            self.abort(403, 'Request requires login')
+
+        check_for_gear_insertion(self.request.json)
+        return None
+
 class GearHandler(base.RequestHandler):
     """Provide /gears/x API routes."""
 
@@ -74,7 +85,6 @@ class GearHandler(base.RequestHandler):
         gear = get_gear(_id)
         return suggest_container(gear, cont_name+'s', cid)
 
-    # Temporary Function
     def upload(self): # pragma: no cover
         """Upload new gear tarball file"""
         if not self.user_is_admin:
@@ -88,7 +98,6 @@ class GearHandler(base.RequestHandler):
 
         return {'_id': str(gear_id)}
 
-    # Temporary Function
     def download(self, **kwargs): # pragma: no cover
         """Download gear tarball file"""
         dl_id = kwargs.pop('cid')
@@ -175,7 +184,6 @@ class RulesHandler(base.RequestHandler):
         result = config.db.project_rules.insert_one(doc)
         return { '_id': result.inserted_id }
 
-
 class RuleHandler(base.RequestHandler):
 
     def get(self, cid, rid):
@@ -247,7 +255,6 @@ class RuleHandler(base.RequestHandler):
             raise APINotFoundException('Rule not found.')
         return
 
-
 class JobsHandler(base.RequestHandler):
     """Provide /jobs API routes."""
     def get(self): # pragma: no cover (no route)
@@ -264,7 +271,7 @@ class JobsHandler(base.RequestHandler):
 
         # Translate maps to FileReferences
         inputs = {}
-        for x in submit['inputs'].keys():
+        for x in submit.get('inputs', {}).keys():
             input_map = submit['inputs'][x]
             inputs[x] = create_filereference_from_dictionary(input_map)
 
@@ -280,6 +287,9 @@ class JobsHandler(base.RequestHandler):
         if submit.get('destination', None) is not None:
             destination = create_containerreference_from_dictionary(submit['destination'])
         else:
+            if len(inputs.keys()) < 1:
+                raise Exception('No destination specified and no inputs to derive from')
+
             key = inputs.keys()[0]
             destination = create_containerreference_from_filereference(inputs[key])
 
@@ -319,9 +329,9 @@ class JobsHandler(base.RequestHandler):
         # You can count on neither occurring before a job starts, because the queue is not globally FIFO.
         # So option #2 is potentially more convenient, but unintuitive and prone to user confusion.
 
-
         for x in inputs:
-            if gear['gear']['inputs'][x]['base'] == 'file':
+            input_type = gear['gear']['inputs'][x]['base']
+            if input_type == 'file':
 
                 obj = inputs[x].get_file()
                 cr = create_containerreference_from_filereference(inputs[x])
@@ -339,6 +349,8 @@ class JobsHandler(base.RequestHandler):
                     },
                     'object': obj_projection,
                 }
+            elif input_type == 'api-key':
+                pass
             else:
                 self.abort(500, 'Non-file input base type')
 
@@ -399,7 +411,8 @@ class JobHandler(base.RequestHandler):
         if not self.superuser_request:
             self.abort(403, 'Request requires superuser')
 
-        c = Job.get(_id).config
+        j = Job.get(_id)
+        c = j.config
         if c is None:
             c = {}
 

api/jobs/jobs.py

@@ -11,6 +11,7 @@ from ..types import Origin
 from ..dao.containerutil import create_filereference_from_dictionary, create_containerreference_from_dictionary, create_containerreference_from_filereference
 
 from .. import config
+from ..util import render_template
 
 
 class Job(object):
@@ -253,7 +254,7 @@ class Job(object):
                 }
             ],
             'target': {
-                'command': ['bash', '-c', 'rm -rf output; mkdir -p output; ./run'],
+                'command': None,
                 'env': {
                     'PATH': '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
                 },
@@ -271,6 +272,20 @@ class Job(object):
         # Map destination to upload URI
         r['outputs'][0]['uri'] = '/engine?level=' + self.destination.type + '&id=' + self.destination.id
 
+        # Add environment, if any
+        for key in gear['gear'].get('environment', {}).keys():
+            r['target']['env'][key] = gear['gear']['environment'][key]
+
+        # Add command, if any
+        command_base = 'env; rm -rf output; mkdir -p output; '
+        if gear['gear'].get('command') is not None:
+
+            command = render_template(gear['gear']['command'], self.config['config'])
+
+            r['target']['command'] = ['bash', '-c', command_base + command ]
+        else:
+            r['target']['command'] = ['bash', '-c', command_base + './run' ]
+
         # Add config, if any
         if self.config is not None:
 
@@ -290,21 +305,24 @@ class Job(object):
                 bash_variable_letters = set(string.ascii_letters + string.digits + ' ' + '_')
 
                 for x in cf:
+                    # Strip non-whitelisted characters, set to underscore, and uppercase
+                    config_name = filter(lambda char: char in bash_variable_letters, x)
+                    config_name = config_name.replace(' ', '_').upper()
 
-                    if isinstance(cf[x], list) or isinstance(cf[x], dict):
-                        # Current gear spec only allows for scalars!
-                        raise Exception('Non-scalar config value ' + x + ' ' + str(cf[x]))
-                    else:
+                    # Don't set nonsensical environment variables
+                    if config_name == '':
+                        print 'The gear config name ' + x + ' has no whitelisted characters!'
+                        continue
 
-                        # Strip non-whitelisted characters, set to underscore, and uppercase
-                        config_name = filter(lambda char: char in bash_variable_letters, x)
-                        config_name = config_name.replace(' ', '_').upper()
+                    if isinstance(cf[x], list):
+                        # Stringify array or set
+                        # Might have same issue as scalars with "True" >:(
+                        r['target']['env']['FW_CONFIG_' + config_name] = str(cf[x])
 
-                        # Don't set nonsensical environment variables
-                        if config_name == '':
-                            print 'The gear config name ' + x + ' has no whitelisted characters!'
-                            continue
+                    elif isinstance(cf[x], dict):
+                        raise Exception('Disallowed object-type config value ' + x + ' ' + str(cf[x]))
 
+                    else:
                         # Stringify scalar
                         # Python strings true as "True"; fix
                         if not isinstance(cf[x], bool):

api/util.py

@@ -11,6 +11,27 @@ import requests
 import string
 import uuid
 
+import django
+from django.conf import settings
+from django.template import Template, Context
+
+# If this is not called before templating, django throws a hissy fit
+settings.configure(
+    TEMPLATES=[{'BACKEND': 'django.template.backends.django.DjangoTemplates'}],
+)
+django.setup()
+
+def render_template(template, context):
+    """
+    Dead-simple wrapper to call django text templating.
+    Set up your own Template and Context objects if re-using heavily.
+    """
+
+    t = Template(template)
+    c = Context(context)
+    return t.render(c)
+
+
 MIMETYPES = [
     ('.bvec', 'text', 'bvec'),
     ('.bval', 'text', 'bval'),

requirements.txt

+django >= 1.11.5
 elasticsearch==5.3.0
 enum==0.4.6
 jsonschema==2.6.0
@@ -9,8 +10,8 @@ pytz==2015.7
 requests==2.9.1
 rfc3987==1.3.4
 strict-rfc3339==0.7
+unicodecsv==0.9.0
 uwsgi==2.0.13.1
 webapp2==2.5.2
 WebOb==1.5.1
-git+https://github.com/flywheel-io/gears.git@v0.1.2#egg=gears
-unicodecsv==0.9.0
+git+https://github.com/flywheel-io/gears.git@d89f763#egg=gears