Improve persistent path variables

api/config.py

@@ -68,6 +68,8 @@ for outer_key, scoped_config in __config.iteritems():
                 value = None
             __config[outer_key][inner_key] = value
 
+if not os.path.exists(__config['persistent']['data_path']):
+    os.makedirs(__config['persistent']['data_path'])
 
 db = pymongo.MongoClient(__config['persistent']['db_uri'], j=True, connectTimeoutMS=2000, serverSelectionTimeoutMS=3000).get_default_database()
 

bin/bootstrap.py

@@ -72,8 +72,6 @@ example:
 
 
 def data(args):
-    if not os.path.exists(args.storage_path):
-        os.makedirs(args.storage_path)
     log.info('inspecting %s' % args.path)
     files = []
     for dirpath, dirnames, filenames in os.walk(args.path):
@@ -97,7 +95,7 @@ def data(args):
             for chunk in iter(lambda: fd.read(2**20), ''):
                 hash_.update(chunk)
         computed_hash = hash_.hexdigest()
-        destpath = os.path.join(args.storage_path, util.path_from_hash(computed_hash))
+        destpath = os.path.join(config.get_item('persistent', 'data_path'), util.path_from_hash(computed_hash))
         dir_destpath = os.path.dirname(destpath)
         filename = os.path.basename(filepath)
         if not os.path.exists(dir_destpath):
@@ -152,7 +150,6 @@ data_parser = subparsers.add_parser(
         )
 data_parser.add_argument('-c', '--copy', action='store_true', help='copy data instead of moving it')
 data_parser.add_argument('path', help='filesystem path to data')
-data_parser.add_argument('storage_path', help='filesystem path to sorted data')
 data_parser.set_defaults(func=data)
 
 args = parser.parse_args()

bin/run.sh

@@ -8,12 +8,12 @@ cd "$( dirname "${BASH_SOURCE[0]}" )/.."
 echo() { builtin echo "[SCITRAN] $@"; }
 
 
+set -o allexport
+
 if [ "$#" -eq 1 ]; then
     EXISTING_ENV=$(env | grep "SCITRAN_" | cat)
-    set -o allexport
     source "$1"
     eval "$EXISTING_ENV"
-    set +o allexport
 fi
 if [ "$#" -gt 1 ]; then
     echo "Usage: $0 [config file]"
@@ -40,6 +40,12 @@ fi
 if [ -z "$SCITRAN_PERSISTENT_PATH" ]; then
     SCITRAN_PERSISTENT_PATH="./persistent"
 fi
+if [ -z "$SCITRAN_PERSISTENT_DATA_PATH" ]; then
+    SCITRAN_PERSISTENT_DATA_PATH="$SCITRAN_PERSISTENT_PATH/data"
+fi
+if [ -z "$SCITRAN_PERSISTENT_DB_PATH" ]; then
+    SCITRAN_PERSISTENT_DB_PATH="$SCITRAN_PERSISTENT_PATH/db"
+fi
 if [ -z "$SCITRAN_PERSISTENT_DB_PORT" ]; then
     SCITRAN_PERSISTENT_DB_PORT="9001"
 fi
@@ -47,12 +53,14 @@ if [ -z "$SCITRAN_PERSISTENT_DB_URI" ]; then
     SCITRAN_PERSISTENT_DB_URI="mongodb://localhost:$SCITRAN_PERSISTENT_DB_PORT/scitran"
 fi
 
+set +o allexport
+
 
-if [ -f "$SCITRAN_PERSISTENT_PATH/db/mongod.lock" ]; then
+if [ -f "$SCITRAN_PERSISTENT_DB_PATH/mongod.lock" ]; then
     BOOTSTRAP_USERS=0
 else
-    echo "Creating database location at $SCITRAN_PERSISTENT_PATH/db"
-    mkdir -p $SCITRAN_PERSISTENT_PATH/db
+    echo "Creating database location at $SCITRAN_PERSISTENT_DB_PATH"
+    mkdir -p $SCITRAN_PERSISTENT_DB_PATH
     if ! [ -f "$SCITRAN_SYSTEM_BOOTSTRAP" ]; then
         echo "Aborting. Please create $SCITRAN_SYSTEM_BOOTSTRAP from bootstrap.json.sample."
         exit 1
@@ -118,7 +126,7 @@ pip install -U -r requirements.txt
 
 
 # Launch mongod
-mongod --dbpath $SCITRAN_PERSISTENT_PATH/db --smallfiles --port $SCITRAN_PERSISTENT_DB_PORT &
+mongod --dbpath $SCITRAN_PERSISTENT_DB_PATH --smallfiles --port $SCITRAN_PERSISTENT_DB_PORT &
 MONGO_PID=$!
 
 # Set python path so scripts can work
@@ -147,12 +155,13 @@ else
 fi
 builtin echo "$TESTDATA_VERSION" > "$SCITRAN_PERSISTENT_PATH/.testdata_version"
 
-if [ -d "$SCITRAN_PERSISTENT_PATH/data" ]; then
+if [ -f "$SCITRAN_PERSISTENT_DATA_PATH/.bootstrapped" ]; then
     echo "Persistence store exists at $SCITRAN_PERSISTENT_PATH/data. Not bootstrapping data. Remove to re-bootstrap."
 else
     echo "Bootstrapping testdata"
-    bin/bootstrap.py data --copy $SCITRAN_PERSISTENT_PATH/testdata $SCITRAN_PERSISTENT_PATH/data
+    bin/bootstrap.py data --copy $SCITRAN_PERSISTENT_PATH/testdata
     echo "Bootstrapped testdata"
+    touch "$SCITRAN_PERSISTENT_DATA_PATH/.bootstrapped"
 fi
 
 

sample.config

@@ -17,6 +17,8 @@
 #SCITRAN_SITE_SSL_CERT=""
 
 #SCITRAN_PERSISTENT_PATH="./persistent"
+#SCITRAN_PERSISTENT_DATA_PATH="./persistent/data" # for fine-grain control
+#SCITRAN_PERSISTENT_DB_PATH="./persistent/db"     # for fine-grain control
 #SCITRAN_PERSISTENT_DB_PORT=9001
 #SCITRAN_PERSISTENT_DB_URI="mongodb://localhost:$SCITRAN_PERSISTENT_DB_PORT/scitran"