Add db upgrade for analysis input/output separation

bin/database.py

@@ -22,7 +22,7 @@ from api.jobs import gears
 from api.types import Origin
 from api.jobs import batch
 
-CURRENT_DATABASE_VERSION = 42 # An int that is bumped when a new schema change is made
+CURRENT_DATABASE_VERSION = 43 # An int that is bumped when a new schema change is made
 
 def get_db_version():
 
@@ -1360,6 +1360,25 @@ def upgrade_to_42():
         process_cursor(cursor, upgrade_to_42_closure, context=cont_name)
 
 
+def upgrade_to_43_closure(analysis):
+    inputs = [f for f in analysis['files'] if f.get('input')]
+    outputs = [f for f in analysis['files'] if f.get('output')]
+    for f in inputs + outputs:
+        f.pop('input', None)
+        f.pop('output', None)
+    config.db.analyses.update_one({'_id': analysis['_id']}, {'$set': {'inputs': inputs, 'files': outputs}})
+    return True
+
+def upgrade_to_43():
+    """
+    Remove analysis files' input/output tags and store them separately instead:
+       - inputs under `analysis.inputs`
+       - outputs under `analysis.files`
+    """
+    cursor = config.db.analyses.find({'files': {'$exists': True, '$ne': []}})
+    process_cursor(cursor, upgrade_to_43_closure)
+
+
 ###
 ### BEGIN RESERVED UPGRADE SECTION
 ###

tests/integration_tests/python/test_upgrades.py

@@ -32,3 +32,41 @@ def test_42(data_builder, api_db, as_admin, database):
     # Verify archived was removed when false as well
     session_data = as_admin.get('/sessions/' + session2).json()
     assert 'archived' not in session_data
+
+
+def test_43(data_builder, api_db, as_admin, file_form, database):
+    # Create session and upload file for later use as analysis input
+    session = data_builder.create_session()
+    r = as_admin.post('/sessions/' + session + '/files', files=file_form('input.txt'))
+    assert r.ok
+
+    # Create ad-hoc analysis with input ref, then upload output
+    r = as_admin.post('/sessions/' + session + '/analyses', json={
+        'label': 'offline',
+        'inputs': [{'type': 'session', 'id': session, 'name': 'input.txt'}]
+    })
+    assert r.ok
+    analysis_id = r.json()['_id']
+    r = as_admin.post('/analyses/' + analysis_id + '/files', files=file_form('output.txt', meta=[{'name': 'output.txt'}]))
+    assert r.ok
+
+    # Mimic old-style analysis input/output tags
+    analysis = api_db.analyses.find_one({'_id': bson.ObjectId(analysis_id)}, ['inputs', 'files'])
+    for f in analysis['inputs']:
+        f['input'] = True
+    for f in analysis['files']:
+        f['output'] = True
+    api_db.analyses.update_one({'_id': bson.ObjectId(analysis_id)},
+                               {'$set': {'files': analysis['inputs'] + analysis['files']},
+                                '$unset': {'inputs': ''}})
+
+    # Verify upgrade gets rid of tags and separates inputs/files
+    database.upgrade_to_43()
+    analysis = as_admin.get('/analyses/' + analysis_id).json()
+    assert 'inputs' in analysis
+    assert len(analysis['inputs']) == 1
+    assert 'input' not in analysis['inputs'][0]
+
+    assert 'files' in analysis
+    assert len(analysis['files']) == 1
+    assert 'output' not in analysis['files'][0]