MongoDB: Connection String Error when Upgrading Config Servers to Replica Set

Question

I'm following these instructions to upgrade my Mongo config servers to a replica set. Everything seems to work initially, but then when connecting to a Mongos and running sh.status, I get the following error message and stack.

Of importance: "Detected upgrade from mirrored (SCCC) config servers to replica set (CSRS) config servers. setShardVersion was given: csReplSet/mongodb-config-1:27019 for the config server connection string, but has stored: mongodb-config-1:27019,mongodb-config-2:27019,mongodb-config-3:27019

This causes the PRIMARY and one of the SECONDARY config servers to crash. I checked that all mongos have the right connection string and that all mongod and mongos are running the same version of MongoDB (3.2.11). I also tried firing up config servers from scratch (at the expense of losing data), but to no avail.

Any ideas?

2016-12-13T08:27:47.825+0000 I SHARDING [conn849] Detected upgrade from mirrored (SCCC) config servers to replica set (CSRS) config servers.  setShardVersion was given: csReplSet/mo\
ngodb-config-1:27019 for the config server connection string, but has stored: mongodb-config-1:27019,mongodb-config-2:27019,mongodb-config-3:27019
2016-12-13T08:27:47.825+0000 F -        [conn849] Invalid access at address: 0x108
2016-12-13T08:27:47.831+0000 F -        [conn849] Got signal: 11 (Segmentation fault).

 0x132f902 0x132ea59 0x132edd8 0x7f288d434330 0x7f288d42e404 0x1140949 0xfa1480 0xbd2653 0xbd34d9 0xb2c1f0 0xce12f5 0x9be81c 0x12dc0c5 0x7f288d42c184 0x7f288d15937d
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"F2F902","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"F2EA59"},{"b":"400000","o":"F2EDD8"},{"b":"7F288D424000","o":"10330"},{"b":"7F288D4\
24000","o":"A404","s":"pthread_mutex_lock"},{"b":"400000","o":"D40949","s":"_ZN5mongo24ForwardingCatalogManager37scheduleReplaceCatalogManagerIfNeededENS_14CatalogManager16ConfigSer\
verModeERKNS_16ConnectionStringE"},{"b":"400000","o":"BA1480"},{"b":"400000","o":"7D2653","s":"_ZN5mongo7Command3runEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS3_21ReplyB\
uilderInterfaceE"},{"b":"400000","o":"7D34D9","s":"_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_RKNS_3rpc16RequestInterfaceEPNS4_21ReplyBuilderInterfaceE"},{"b":"400000\
","o":"72C1F0","s":"_ZN5mongo11runCommandsEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS2_21ReplyBuilderInterfaceE"},{"b":"400000","o":"8E12F5","s":"_ZN5mongo16assembleResp\
onseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE"},{"b":"400000","o":"5BE81C","s":"_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessag\
ingPortE"},{"b":"400000","o":"EDC0C5","s":"_ZN5mongo17PortMessageServer17handleIncomingMsgEPv"},{"b":"7F288D424000","o":"8184"},{"b":"7F288D05F000","o":"FA37D","s":"clone"}],"proces\
sInfo":{ "mongodbVersion" : "3.2.11", "gitVersion" : "009580ad490190ba33d1c6253ebd8d91808923e4", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "4.4.0-53-gener\
ic", "version" : "#74~14.04.1-Ubuntu SMP Fri Dec 2 03:43:31 UTC 2016", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "55994D9F4E3C47C30959A86BC9F3\
3188CB6ADF21" }, { "b" : "7FFCEDF32000", "elfType" : 3, "buildId" : "AA5EEB9DFDD3AABAC9F493EC883C381016A1C312" }, { "b" : "7F288E346000", "path" : "/lib/x86_64-linux-gnu/libssl.so.1\
.0.0", "elfType" : 3, "buildId" : "15A6A69B862FAC0AA589921DB1B0DF8A673FEC81" }, { "b" : "7F288DF6A000", "path" : "/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId"\
 : "607C4A6F3428B6F065D55691AFDC4E343786A8A5" }, { "b" : "7F288DD62000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "E2A6DD5048A0A051FD61043BDB69D8CC681\
92AB7" }, { "b" : "7F288DB5E000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "DA9B8C234D0FE9FD8CAAC8970A7EC1B6C8F6623F" }, { "b" : "7F288D858000", "path\
" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "D144258E614900B255A31F3FD2283A878670D5BC" }, { "b" : "7F288D642000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1\
", "elfType" : 3, "buildId" : "36311B4457710AE5578C4BF00791DED7359DBB92" }, { "b" : "7F288D424000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "31E\
9F21AE8C10396171F1E13DA15780986FA696C" }, { "b" : "7F288D05F000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "CF699A15CAAE64F50311FC4655B86DC39A479789" }\
, { "b" : "7F288E5A5000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "D0F537904076D73F29E4A37341F8A449E2EF6CD0" } ] }}
 mongod(_ZN5mongo15printStackTraceERSo+0x32) [0x132f902]
 mongod(+0xF2EA59) [0x132ea59]
 mongod(+0xF2EDD8) [0x132edd8]
 libpthread.so.0(+0x10330) [0x7f288d434330]
 libpthread.so.0(pthread_mutex_lock+0x4) [0x7f288d42e404]
 mongod(_ZN5mongo24ForwardingCatalogManager37scheduleReplaceCatalogManagerIfNeededENS_14CatalogManager16ConfigServerModeERKNS_16ConnectionStringE+0x39) [0x1140949]
 mongod(+0xBA1480) [0xfa1480]
 mongod(_ZN5mongo7Command3runEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS3_21ReplyBuilderInterfaceE+0x473) [0xbd2653]
 mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_RKNS_3rpc16RequestInterfaceEPNS4_21ReplyBuilderInterfaceE+0x3F9) [0xbd34d9]
 mongod(_ZN5mongo11runCommandsEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS2_21ReplyBuilderInterfaceE+0x1F0) [0xb2c1f0]
 mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB45) [0xce12f5]
 mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortE+0xEC) [0x9be81c]
 mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x325) [0x12dc0c5]
 libpthread.so.0(+0x8184) [0x7f288d42c184]
 libc.so.6(clone+0x6D) [0x7f288d15937d]
-----  END BACKTRACE  -----
2016-12-13T08:27:47.831+0000 F -        [conn849] /proc/self/maps:
00400000-02008000 r-xp 00000000 08:01 2125                               /usr/bin/mongod
2016-12-13T08:27:47.831+0000 F -        [conn849] 02008000-020db000 rw-p 01c07000 08:01 2125                               /usr/bin/mongod
2016-12-13T08:27:47.831+0000 F -        [conn849] 020db000-0214b000 rw-p 00000000 00:00 0
2016-12-13T08:27:47.831+0000 F -        [conn849] 02885000-03486000 rw-p 00000000 00:00 0                                  [heap]
2016-12-13T08:27:47.831+0000 F -        [conn849] 03486000-08972000 rw-p 00000000 00:00 0                                  [heap]
2016-12-13T08:27:47.831+0000 F -        [conn849] 7f28689ba000-7f28689bb000 ---p 00000000 00:00 0
2016-12-13T08:27:47.831+0000 F -        [conn849] 7f28689bb000-7f2868abb000 rw-p 00000000 00:00 0
2016-12-13T08:27:47.831+0000 F -        [conn849] 7f2868abb000-7f2868abc000 ---p 00000000 00:00 0
2016-12-13T08:27:47.831+0000 F -        [conn849] 7f2868abc000-7f2868bbc000 rw-p 00000000 00:00 0
2016-12-13T08:27:47.831+0000 F -        [conn849] 7f2868bbc000-7f2868bbd000 ---p 00000000 00:00 0
2016-12-13T08:27:47.831+0000 F -        [conn849] 7f2868bbd000-7f2868cbd000 rw-p 00000000 00:00 0
2016-12-13T08:27:47.831+0000 F -        [conn849] 7f2868cbd000-7f2868cbe000 ---p 00000000 00:00 0
.
.
.
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288c85e000-7f288c85f000 ---p 00000000 00:00 0
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288c85f000-7f288d05f000 rw-p 00000000 00:00 0
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288d05f000-7f288d219000 r-xp 00000000 08:01 3079                       /lib/x86_64-linux-gnu/libc-2.19.so
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288d219000-7f288d419000 ---p 001ba000 08:01 3079                       /lib/x86_64-linux-gnu/libc-2.19.so
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288d419000-7f288d41d000 r--p 001ba000 08:01 3079                       /lib/x86_64-linux-gnu/libc-2.19.so
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288d41d000-7f288d41f000 rw-p 001be000 08:01 3079                       /lib/x86_64-linux-gnu/libc-2.19.so
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288d41f000-7f288d424000 rw-p 00000000 00:00 0
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288d424000-7f288d43d000 r-xp 00000000 08:01 3042                       /lib/x86_64-linux-gnu/libpthread-2.19.so
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288d43d000-7f288d63c000 ---p 00019000 08:01 3042                       /lib/x86_64-linux-gnu/libpthread-2.19.so
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288d63c000-7f288d63d000 r--p 00018000 08:01 3042                       /lib/x86_64-linux-gnu/libpthread-2.19.so
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288d63d000-7f288d63e000 rw-p 00019000 08:01 3042                       /lib/x86_64-linux-gnu/libpthread-2.19.so
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288d63e000-7f288d642000 rw-p 00000000 00:00 0
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288d642000-7f288d658000 r-xp 00000000 08:01 3111                       /lib/x86_64-linux-gnu/libgcc_s.so.1
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288d658000-7f288d857000 ---p 00016000 08:01 3111                       /lib/x86_64-linux-gnu/libgcc_s.so.1
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288d857000-7f288d858000 rw-p 00015000 08:01 3111                       /lib/x86_64-linux-gnu/libgcc_s.so.1
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288d858000-7f288d95d000 r-xp 00000000 08:01 3084                       /lib/x86_64-linux-gnu/libm-2.19.so
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288d95d000-7f288db5c000 ---p 00105000 08:01 3084                       /lib/x86_64-linux-gnu/libm-2.19.so
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288db5c000-7f288db5d000 r--p 00104000 08:01 3084                       /lib/x86_64-linux-gnu/libm-2.19.so
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288db5d000-7f288db5e000 rw-p 00105000 08:01 3084                       /lib/x86_64-linux-gnu/libm-2.19.so
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288db5e000-7f288db61000 r-xp 00000000 08:01 3076                       /lib/x86_64-linux-gnu/libdl-2.19.so
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288db61000-7f288dd60000 ---p 00003000 08:01 3076                       /lib/x86_64-linux-gnu/libdl-2.19.so
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288dd60000-7f288dd61000 r--p 00002000 08:01 3076                       /lib/x86_64-linux-gnu/libdl-2.19.so
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288dd61000-7f288dd62000 rw-p 00003000 08:01 3076                       /lib/x86_64-linux-gnu/libdl-2.19.so
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288dd62000-7f288dd69000 r-xp 00000000 08:01 3002                       /lib/x86_64-linux-gnu/librt-2.19.so
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288dd69000-7f288df68000 ---p 00007000 08:01 3002                       /lib/x86_64-linux-gnu/librt-2.19.so
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288df68000-7f288df69000 r--p 00006000 08:01 3002                       /lib/x86_64-linux-gnu/librt-2.19.so
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288df69000-7f288df6a000 rw-p 00007000 08:01 3002                       /lib/x86_64-linux-gnu/librt-2.19.so
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288df6a000-7f288e11d000 r-xp 00000000 08:01 3012                       /lib/x86_64-linux-gnu/libcrypto.so.1.0.0
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288e11d000-7f288e31c000 ---p 001b3000 08:01 3012                       /lib/x86_64-linux-gnu/libcrypto.so.1.0.0
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288e31c000-7f288e337000 r--p 001b2000 08:01 3012                       /lib/x86_64-linux-gnu/libcrypto.so.1.0.0
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288e337000-7f288e342000 rw-p 001cd000 08:01 3012                       /lib/x86_64-linux-gnu/libcrypto.so.1.0.0
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288e342000-7f288e346000 rw-p 00000000 00:00 0
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288e346000-7f288e39b000 r-xp 00000000 08:01 3059                       /lib/x86_64-linux-gnu/libssl.so.1.0.0
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288e39b000-7f288e59b000 ---p 00055000 08:01 3059                       /lib/x86_64-linux-gnu/libssl.so.1.0.0
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288e59b000-7f288e59e000 r--p 00055000 08:01 3059                       /lib/x86_64-linux-gnu/libssl.so.1.0.0
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288e59e000-7f288e5a5000 rw-p 00058000 08:01 3059                       /lib/x86_64-linux-gnu/libssl.so.1.0.0
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288e5a5000-7f288e5c8000 r-xp 00000000 08:01 3010                       /lib/x86_64-linux-gnu/ld-2.19.so
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288e62d000-7f288e62f000 rw-p 00000000 00:00 0
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288e62f000-7f288e7b8000 r--p 00000000 08:01 28741                      /usr/lib/locale/locale-archive
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288e7be000-7f288e7c5000 r--s 00000000 08:01 29034                      /usr/lib/x86_64-linux-gnu/gconv/gconv-modules.cache
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288e7c5000-7f288e7c7000 rw-p 00000000 00:00 0
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288e7c7000-7f288e7c8000 r--p 00022000 08:01 3010                       /lib/x86_64-linux-gnu/ld-2.19.so
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288e7c8000-7f288e7c9000 rw-p 00023000 08:01 3010                       /lib/x86_64-linux-gnu/ld-2.19.so
2016-12-13T08:27:47.833+0000 F -        [conn849] 7f288e7c9000-7f288e7ca000 rw-p 00000000 00:00 0
2016-12-13T08:27:47.833+0000 F -        [conn849] 7ffcede0e000-7ffcede2f000 rw-p 00000000 00:00 0                          [stack]
2016-12-13T08:27:47.833+0000 F -        [conn849] 7ffcedf30000-7ffcedf32000 r--p 00000000 00:00 0                          [vvar]
2016-12-13T08:27:47.833+0000 F -        [conn849] 7ffcedf32000-7ffcedf34000 r-xp 00000000 00:00 0                          [vdso]
2016-12-13T08:27:47.833+0000 F -        [conn849] ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0                  [vsyscall]

Answer 1

The problem was that there were still some mongos instances using the old (mirrored) connection string. It seems that even mongos instances that are not up keep on pinging the config servers which cause the error and the fatal crash. Do use config; db.mongos.find() to find all your mongos instances and make sure they are all using the updated connection string without exception.