Could this be a fix for config key dupe?

api/auth/authproviders.py

@@ -81,7 +81,7 @@ class AuthProvider(object):
             'auth_type': self.auth_type,
             'uid': uid
         }
-        dbutil.fault_tolerant_replace_one('refreshtokens', query, refresh_doc, upsert=True)
+        dbutil.fault_tolerant_replace_one(config.db, 'refreshtokens', query, refresh_doc, upsert=True)
 
 class JWTAuthProvider(AuthProvider):
 

api/config.py

@@ -8,6 +8,7 @@ import datetime
 import elasticsearch
 
 from . import util
+from .dao.dbutil import try_replace_one
 
 logging.basicConfig(
     format='%(asctime)s %(name)16.16s %(filename)24.24s %(lineno)5d:%(levelname)4.4s %(message)s',
@@ -269,7 +270,16 @@ def get_config():
             __config['created'] = now
         __config['modified'] = now
 
-        db.singletons.replace_one({'_id': 'config'}, __config, upsert=True)
+        # Attempt to set the config object, ignoring duplicate key problems.
+        # This worker might have lost the race - in which case, be grateful about it.
+        #
+        # Ref:
+        # https://github.com/scitran/core/issues/212
+        # https://github.com/scitran/core/issues/844
+        _, success = try_replace_one(db, 'singletons', {'_id': 'config'}, __config, upsert=True)
+        if not success:
+            log.debug('Worker lost config upsert race; ignoring.')
+
         __config_persisted = True
         __last_update = now
     elif now - __last_update > datetime.timedelta(seconds=120):

api/dao/dbutil.py

@@ -2,30 +2,40 @@ import random
 import time
 
 from pymongo.errors import DuplicateKeyError
-from .. import config
 from . import APIStorageException
 
+def try_replace_one(db, coll_name, query, update, upsert=False):
+    """
+    Mongo does not see replace w/ upsert as an atomic action:
+    https://jira.mongodb.org/browse/SERVER-14322
+
+    This function will try a replace_one operation, returning the result and if the operation succeeded.
+    """
 
+    try:
+        result = db[coll_name].replace_one(query, update, upsert=upsert)
+    except DuplicateKeyError:
+        return result, False
+    else:
+        return result, True
 
 
-def fault_tolerant_replace_one(coll_name, query, update, upsert=False):
+def fault_tolerant_replace_one(db, coll_name, query, update, upsert=False):
     """
-    Mongo does not see replace w/ upsert as an atomic action:
-    https://jira.mongodb.org/browse/SERVER-14322
+    Like try_replace_one, but will retry several times, waiting a random short duration each time.
 
-    Attempt a retry if first try produces DuplicateKeyError
+    Raises an APIStorageException if the retry loop gives up.
     """
 
     attempts = 0
     while attempts < 10:
         attempts += 1
-        try:
-            result = config.db[coll_name].replace_one(query, update, upsert=upsert)
-        except DuplicateKeyError:
-            time.sleep(random.uniform(0.01,0.05))
-        else:
-            return result
 
-    raise APIStorageException('Unable to replace object.')
+        result, success = try_replace_one(db, coll_name, query, update, upsert)
 
+        if success:
+            return result
+        else:
+            time.sleep(random.uniform(0.01,0.05))
 
+    raise APIStorageException('Unable to replace object.')