Merge pull request #236 from scitran/origin_labels

Improve drone name extraction for UI display

api/api.py

@@ -80,9 +80,6 @@ routes = [
     webapp2.Route(r'/api',                  root.Root),
     webapp2_extras.routes.PathPrefixRoute(r'/api', [
         webapp2.Route(r'/download',         download.Download, handler_method='download', methods=['GET', 'POST'], name='download'),
-        # webapp2.Route(r'/reaper',           upload.Upload, handler_method='reaper', methods=['POST']),
-        # upload take a parameter type with possible values ['label', 'uid'] default to 'label'
-        webapp2.Route(r'/uploader',           upload.Upload, handler_method='upload', methods=['POST'], defaults={'strategy': 'label'}),
         webapp2.Route(r'/upload/<strategy:label|uid>',           upload.Upload, handler_method='upload', methods=['POST']),
         webapp2.Route(r'/engine',           upload.Upload, handler_method='engine', methods=['POST']),
         webapp2.Route(r'/sites',            centralclient.CentralClient, handler_method='sites', methods=['GET']),

api/base.py

@@ -18,9 +18,6 @@ from .dao import APIConsistencyException
 
 log = config.log
 
-# When authenticating as a drone, the user agent must start with this prefix.
-DRONE_PREFIX = 'SciTran Drone '
-
 class RequestHandler(webapp2.RequestHandler):
 
     json_schema = None
@@ -34,11 +31,12 @@ class RequestHandler(webapp2.RequestHandler):
         self.uid = None
         self.source_site = None
         drone_request = False
-        drone_name = ''
 
         user_agent = self.request.headers.get('User-Agent', '')
         access_token = self.request.headers.get('Authorization', None)
         drone_secret = self.request.headers.get('X-SciTran-Auth', None)
+        drone_method = self.request.headers.get('X-SciTran-Method', None)
+        drone_name = self.request.headers.get('X-SciTran-Name', None)
 
         site_id = config.get_item('site', 'id')
         if site_id is None:
@@ -53,14 +51,14 @@ class RequestHandler(webapp2.RequestHandler):
             self.uid = self.get_param('user')
 
         # Drone shared secret authentication
-        elif drone_secret is not None and user_agent.startswith(DRONE_PREFIX):
+        elif drone_secret is not None:
+            if drone_method is None or drone_name is None:
+                self.abort(400, 'X-SciTran-Method or X-SciTran-Name header missing')
             if config.get_item('core', 'drone_secret') is None:
                 self.abort(401, 'drone secret not configured')
             if drone_secret != config.get_item('core', 'drone_secret'):
                 self.abort(401, 'invalid drone secret')
             drone_request = True
-            drone_name = user_agent.replace(DRONE_PREFIX, '')
-            log.info('drone "' + drone_name + '" request accepted')
 
         # Cross-site authentication
         elif user_agent.startswith('SciTran Instance '):
@@ -92,7 +90,7 @@ class RequestHandler(webapp2.RequestHandler):
             else:
                 self.superuser_request = False
 
-        self.set_origin(drone_request, drone_name)
+        self.set_origin(drone_request)
 
     def authenticate_user(self, access_token):
         """
@@ -167,7 +165,7 @@ class RequestHandler(webapp2.RequestHandler):
 
         return uid
 
-    def set_origin(self, drone_request, drone_name):
+    def set_origin(self, drone_request):
         """
         Add an origin to the request object. Used later in request handler logic.
 
@@ -182,9 +180,15 @@ class RequestHandler(webapp2.RequestHandler):
                 'id': self.uid
             }
         elif drone_request:
+
+            method = self.request.headers.get('X-SciTran-Method')
+            name = self.request.headers.get('X-SciTran-Name')
+
             self.origin = {
+                'id': (method + '_' + name).replace(' ', '_'),
                 'type': str(Origin.device),
-                'id': drone_name
+                'method': method,
+                'name': name
             }
 
             # Upsert device record, with last-contacted time.
@@ -194,7 +198,9 @@ class RequestHandler(webapp2.RequestHandler):
                 }, {
                     '$set': {
                         '_id': self.origin['id'],
-                        'last-seen': datetime.datetime.utcnow()
+                        'last-seen': datetime.datetime.utcnow(),
+                        'method': self.origin['method'],
+                        'name': self.origin['name']
                     }
                 },
                 upsert=True,

bin/bootstrap.py

@@ -55,7 +55,8 @@ if args.insecure:
     requests.packages.urllib3.disable_warnings()
 
 http_headers = {
-    'User-Agent': 'SciTran Drone Bootstrapper',
+    'X-SciTran-Method': 'bootstrapper',
+    'X-SciTran-Name': 'Bootstrapper',
 }
 if args.secret:
     http_headers['X-SciTran-Auth'] = args.secret

bin/database.py

 #!/usr/bin/env python
 
 import json
+import bson
 import sys
 import logging
 from api import config
 
-CURRENT_DATABASE_VERSION = 1 # An int that is bumped when a new schema change is made
+CURRENT_DATABASE_VERSION = 2 # An int that is bumped when a new schema change is made
 
 def get_db_version():
 
@@ -40,9 +41,45 @@ def confirm_schema_match():
         sys.exit(0)
 
 def upgrade_to_1():
-    # scitran/core issue #206
+    """
+    scitran/core issue #206
+
+    Initialize db version to 1
+    """
     config.db.version.insert_one({'_id': 'version', 'database': CURRENT_DATABASE_VERSION})
 
+def upgrade_to_2():
+    """
+    Scitran/core PR #236
+
+    Set file.origin.name to id if does not exist
+    Set file.origin.method to '' if does not exist
+    """
+
+    def update_file_origins(cont_list, cont_name):
+        for container in cont_list:
+            updated_files = []
+            for file in container.get('files', []):
+                origin = file.get('origin')
+                if origin is not None:
+                    if origin.get('name', None) is None:
+                        file['origin']['name'] = origin['id']
+                    if origin.get('method', None) is None:
+                        file['origin']['method'] = ''
+                updated_files.append(file)
+
+            query = {'_id': container['_id']}
+            update = {'$set': {'files': updated_files}}
+            result = config.db[cont_name].update_one(query, update)
+
+    query = {'$and':[{'files.origin.name': { '$exists': False}}, {'files.origin.id': { '$exists': True}}]}
+
+    update_file_origins(config.db.collections.find(query), 'collections')
+    update_file_origins(config.db.projects.find(query), 'projects')
+    update_file_origins(config.db.sessions.find(query), 'sessions')
+    update_file_origins(config.db.acquisitions.find(query), 'acquisitions')
+
+
 def upgrade_schema():
     """
     Upgrades db to the current schema version
@@ -54,6 +91,8 @@ def upgrade_schema():
     try:
         if db_version < 1:
             upgrade_to_1()
+        if db_version < 2:
+            upgrade_to_2()
     except Exception as e:
         logging.exception('Incremental upgrade of db failed')
         sys.exit(1)

bin/run.sh

@@ -31,6 +31,7 @@ SCITRAN_PERSISTENT_DATA_PATH=${SCITRAN_PERSISTENT_DATA_PATH:-"$SCITRAN_PERSISTEN
 SCITRAN_PERSISTENT_DB_PATH=${SCITRAN_PERSISTENT_DB_PATH:-"$SCITRAN_PERSISTENT_PATH/db"}
 SCITRAN_PERSISTENT_DB_PORT=${SCITRAN_PERSISTENT_DB_PORT:-"9001"}
 SCITRAN_PERSISTENT_DB_URI=${SCITRAN_PERSISTENT_DB_URI:-"mongodb://localhost:$SCITRAN_PERSISTENT_DB_PORT/scitran"}
+SCITRAN_CORE_DRONE_SECRET=${SCITRAN_CORE_DRONE_SECRET:-"change-me"}
 
 [ -z "$SCITRAN_RUNTIME_SSL_PEM" ] && SCITRAN_SITE_API_URL="http" || SCITRAN_SITE_API_URL="https"
 SCITRAN_SITE_API_URL="$SCITRAN_SITE_API_URL://$SCITRAN_RUNTIME_HOST:$SCITRAN_RUNTIME_PORT/api"
@@ -154,10 +155,7 @@ trap "{
 
 
 # Wait for everything to come up
-sleep 1
-
-
-
+sleep 2
 
 
 # Boostrap users and groups
@@ -184,7 +182,7 @@ if [ -f "$SCITRAN_PERSISTENT_DATA_PATH/.bootstrapped" ]; then
     echo "Persistence store exists at $SCITRAN_PERSISTENT_PATH/data. Not bootstrapping data. Remove to re-bootstrap."
 else
     echo "Bootstrapping testdata"
-    folder_reaper --insecure --secret "$SCITRAN_CORE_DRONE_SECRET" $SCITRAN_SITE_API_URL "$SCITRAN_PERSISTENT_PATH/testdata"
+    folder_uploader --insecure --secret "$SCITRAN_CORE_DRONE_SECRET" $SCITRAN_SITE_API_URL "$SCITRAN_PERSISTENT_PATH/testdata"
     echo "Bootstrapped testdata"
     touch "$SCITRAN_PERSISTENT_DATA_PATH/.bootstrapped"
 fi

docker/bootstrap-data.sh

@@ -25,7 +25,7 @@ bootstrap_data_label=7d5c3608ff360d6ae28aab0ef262e6781c4ae8d6
 
 
 # Same as bootstrap_data_label above, except for scitran/reaper.
-bootstrap_reaper_label=30215c66a33b18685e1608dbe952e78c370d8765
+bootstrap_reaper_label=0778a6e8294134db060e230b5bd5af4e09e53b56
 
 
 # Move to API folder for relative path assumptions later on
@@ -75,6 +75,6 @@ pip install "git+https://github.com/scitran/reaper.git@${bootstrap_reaper_label}
 API_URL="$SCITRAN_RUNTIME_PROTOCOL://$SCITRAN_RUNTIME_HOST:$SCITRAN_RUNTIME_PORT/api"
 
 ## load the test data in
-folder_reaper --insecure --secret "${SCITRAN_CORE_DRONE_SECRET}" "${API_URL}" "$TESTDATA_DIR/download"
+folder_uploader --insecure --secret "${SCITRAN_CORE_DRONE_SECRET}" "${API_URL}" "$TESTDATA_DIR/download"
 
 )