Federation supervisor crash during upgrade to 4.2.x on multi-node cluster

[gomoripeti]

Community Support Policy

• I have read RabbitMQ's Community Support Policy

RabbitMQ version used

4.2.x

How is RabbitMQ deployed?

Debian package

Steps to reproduce the behavior in question

• Create a 3 node cluster on 4.1.7 and another (single node) cluster that will serve as federation upstream
• Set up exchange federation from the 3 node cluster to the upstream cluster
• Upgrade the 3 node cluster to 4.2.2 in a rolling fashion

Supervisor rabbit_federation_exchange_link_sup_sup crashes with the following crash and is not running any more

<0.748.0>     supervisor: {local,rabbit_federation_exchange_link_sup_sup}
<0.748.0>     errorContext: child_terminated
<0.748.0>     reason: {shutdown,
<0.748.0>              [{{'EXIT',
<0.748.0>                 {undef,
<0.748.0>                  [{rabbit_federation_link_sup,start_link,

full crash log

<0.533.0> rabbit on node 'rabbit@host-03' down
<0.748.0>     supervisor: {local,rabbit_federation_exchange_link_sup_sup}
<0.748.0>     errorContext: child_terminated
<0.748.0>     reason: {shutdown,
<0.748.0>              [{{'EXIT',
<0.748.0>                 {undef,
<0.748.0>                  [{rabbit_federation_link_sup,start_link,
<0.748.0>                    [{exchange,
<0.748.0>                      {resource,<<"vhost1">>,exchange,<<"fed_ex3">>},
<0.748.0>                      fanout,true,false,false,[],undefined,
<0.748.0>                      [{vhost,<<"vhost1">>},
<0.748.0>                       {name,<<"fed_on_upgrade_policy">>},
<0.748.0>                       {pattern,<<"^fed_.*$">>},
<0.748.0>                       {'apply-to',<<"all">>},
<0.748.0>                       {definition,
<0.748.0>                        [{<<"federation-upstream">>,<<"fed_upstream">>}]},
<0.748.0>                       {priority,2}],
<0.748.0>                      undefined,
<0.748.0>                      {[],
<0.748.0>                       [rabbit_federation_exchange,
<0.748.0>                        rabbit_event_exchange_decorator]},
<0.748.0>                      #{user => <<"vhost1">>}}],
<0.748.0>                    []},
<0.748.0>                   {supervisor2,do_start_child_i,3,
<0.748.0>                    [{file,"supervisor2.erl"},{line,441}]},
<0.748.0>                   {supervisor2,do_start_child,2,
<0.748.0>                    [{file,"supervisor2.erl"},{line,427}]},
<0.748.0>                   {supervisor2,handle_start_child,2,
<0.748.0>                    [{file,"supervisor2.erl"},{line,750}]},
<0.748.0>                   {supervisor2,handle_call,3,
<0.748.0>                    [{file,"supervisor2.erl"},{line,482}]},
<0.748.0>                   {gen_server,try_handle_call,4,
<0.748.0>                    [{file,"gen_server.erl"},{line,2470}]},
<0.748.0>                   {gen_server,handle_msg,3,
<0.748.0>                    [{file,"gen_server.erl"},{line,2499}]},
<0.748.0>                   {proc_lib,init_p_do_apply,3,
<0.748.0>                    [{file,"proc_lib.erl"},{line,333}]}]}},
<0.748.0>                {child,undefined,
<0.748.0>                 {exchange,
<0.748.0>                  {resource,<<"vhost1">>,exchange,<<"fed_ex3">>},
<0.748.0>                  fanout,true,false,false,[],none,
<0.748.0>                  [{vhost,<<"vhost1">>},
<0.748.0>                   {name,<<"fed_on_upgrade_policy">>},
<0.748.0>                   {pattern,<<"^fed_.*$">>},
<0.748.0>                   {'apply-to',<<"all">>},
<0.748.0>                   {definition,
<0.748.0>                    [{<<"federation-upstream">>,<<"fed_upstream">>}]},
<0.748.0>                   {priority,2}],
<0.748.0>                  undefined,none,
<0.748.0>                  #{user => <<"vhost1">>}},
<0.748.0>                 {rabbit_federation_link_sup,start_link,
<0.748.0>                  [{exchange,
<0.748.0>                    {resource,<<"vhost1">>,exchange,<<"fed_ex3">>},
<0.748.0>                    fanout,true,false,false,[],undefined,
<0.748.0>                    [{vhost,<<"vhost1">>},
<0.748.0>                     {name,<<"fed_on_upgrade_policy">>},
<0.748.0>                     {pattern,<<"^fed_.*$">>},
<0.748.0>                     {'apply-to',<<"all">>},
<0.748.0>                     {definition,
<0.748.0>                      [{<<"federation-upstream">>,<<"fed_upstream">>}]},
<0.748.0>                     {priority,2}],
<0.748.0>                    undefined,
<0.748.0>                    {[],
<0.748.0>                     [rabbit_federation_exchange,
<0.748.0>                      rabbit_event_exchange_decorator]},
<0.748.0>                    #{user => <<"vhost1">>}}]},
<0.748.0>                 transient,infinity,supervisor,
<0.748.0>                 [rabbit_federation_link_sup]}},
<0.748.0>               {{'EXIT',
<0.748.0>                 {undef,
<0.748.0>                  [{rabbit_federation_link_sup,start_link,
<0.748.0>                    [{exchange,
<0.748.0>                      {resource,<<"vhost1">>,exchange,<<"fed_ex2">>},
<0.748.0>                      fanout,true,false,false,[],undefined,
<0.748.0>                      [{vhost,<<"vhost1">>},
<0.748.0>                       {name,<<"fed_on_upgrade_policy">>},
<0.748.0>                       {pattern,<<"^fed_.*$">>},
<0.748.0>                       {'apply-to',<<"all">>},
<0.748.0>                       {definition,
<0.748.0>                        [{<<"federation-upstream">>,<<"fed_upstream">>}]},
<0.748.0>                       {priority,2}],
<0.748.0>                      undefined,
<0.748.0>                      {[],
<0.748.0>                       [rabbit_federation_exchange,
<0.748.0>                        rabbit_event_exchange_decorator]},
<0.748.0>                      #{user => <<"vhost1">>}}],
<0.748.0>                    []},
<0.748.0>                   {supervisor2,do_start_child_i,3,
<0.748.0>                    [{file,"supervisor2.erl"},{line,441}]},
<0.748.0>                   {supervisor2,do_start_child,2,
<0.748.0>                    [{file,"supervisor2.erl"},{line,427}]},
<0.748.0>                   {supervisor2,handle_start_child,2,
<0.748.0>                    [{file,"supervisor2.erl"},{line,750}]},
<0.748.0>                   {supervisor2,handle_call,3,
<0.748.0>                    [{file,"supervisor2.erl"},{line,482}]},
<0.748.0>                   {gen_server,try_handle_call,4,
<0.748.0>                    [{file,"gen_server.erl"},{line,2470}]},
<0.748.0>                   {gen_server,handle_msg,3,
<0.748.0>                    [{file,"gen_server.erl"},{line,2499}]},
<0.748.0>                   {proc_lib,init_p_do_apply,3,
<0.748.0>                    [{file,"proc_lib.erl"},{line,333}]}]}},
<0.748.0>                {child,undefined,
<0.748.0>                 {exchange,
<0.748.0>                  {resource,<<"vhost1">>,exchange,<<"fed_ex2">>},
<0.748.0>                  fanout,true,false,false,[],none,
<0.748.0>                  [{vhost,<<"vhost1">>},
<0.748.0>                   {name,<<"fed_on_upgrade_policy">>},
<0.748.0>                   {pattern,<<"^fed_.*$">>},
<0.748.0>                   {'apply-to',<<"all">>},
<0.748.0>                   {definition,
<0.748.0>                    [{<<"federation-upstream">>,<<"fed_upstream">>}]},
<0.748.0>                   {priority,2}],
<0.748.0>                  undefined,none,
<0.748.0>                  #{user => <<"vhost1">>}},
<0.748.0>                 {rabbit_federation_link_sup,start_link,
<0.748.0>                  [{exchange,
<0.748.0>                    {resource,<<"vhost1">>,exchange,<<"fed_ex2">>},
<0.748.0>                    fanout,true,false,false,[],undefined,
<0.748.0>                    [{vhost,<<"vhost1">>},
<0.748.0>                     {name,<<"fed_on_upgrade_policy">>},
<0.748.0>                     {pattern,<<"^fed_.*$">>},
<0.748.0>                     {'apply-to',<<"all">>},
<0.748.0>                     {definition,
<0.748.0>                      [{<<"federation-upstream">>,<<"fed_upstream">>}]},
<0.748.0>                     {priority,2}],
<0.748.0>                    undefined,
<0.748.0>                    {[],
<0.748.0>                     [rabbit_federation_exchange,
<0.748.0>                      rabbit_event_exchange_decorator]},
<0.748.0>                    #{user => <<"vhost1">>}}]},
<0.748.0>                 transient,infinity,supervisor,
<0.748.0>                 [rabbit_federation_link_sup]}},
<0.748.0>               {{'EXIT',
<0.748.0>                 {undef,
<0.748.0>                  [{rabbit_federation_link_sup,start_link,
<0.748.0>                    [{exchange,
<0.748.0>                      {resource,<<"vhost1">>,exchange,<<"fed_ex">>},
<0.748.0>                      fanout,true,false,false,[],undefined,
<0.748.0>                      [{vhost,<<"vhost1">>},
<0.748.0>                       {name,<<"fed_on_upgrade_policy">>},
<0.748.0>                       {pattern,<<"^fed_.*$">>},
<0.748.0>                       {'apply-to',<<"all">>},
<0.748.0>                       {definition,
<0.748.0>                        [{<<"federation-upstream">>,<<"fed_upstream">>}]},
<0.748.0>                       {priority,2}],
<0.748.0>                      undefined,
<0.748.0>                      {[],
<0.748.0>                       [rabbit_federation_exchange,
<0.748.0>                        rabbit_event_exchange_decorator]},
<0.748.0>                      #{user => <<"vhost1">>}}],
<0.748.0>                    []},
<0.748.0>                   {supervisor2,do_start_child_i,3,
<0.748.0>                    [{file,"supervisor2.erl"},{line,441}]},
<0.748.0>                   {supervisor2,do_start_child,2,
<0.748.0>                    [{file,"supervisor2.erl"},{line,427}]},
<0.748.0>                   {supervisor2,handle_start_child,2,
<0.748.0>                    [{file,"supervisor2.erl"},{line,750}]},
<0.748.0>                   {supervisor2,handle_call,3,
<0.748.0>                    [{file,"supervisor2.erl"},{line,482}]},
<0.748.0>                   {gen_server,try_handle_call,4,
<0.748.0>                    [{file,"gen_server.erl"},{line,2470}]},
<0.748.0>                   {gen_server,handle_msg,3,
<0.748.0>                    [{file,"gen_server.erl"},{line,2499}]},
<0.748.0>                   {proc_lib,init_p_do_apply,3,
<0.748.0>                    [{file,"proc_lib.erl"},{line,333}]}]}},
<0.748.0>                {child,undefined,
<0.748.0>                 {exchange,
<0.748.0>                  {resource,<<"vhost1">>,exchange,<<"fed_ex">>},
<0.748.0>                  fanout,true,false,false,[],none,
<0.748.0>                  [{vhost,<<"vhost1">>},
<0.748.0>                   {name,<<"fed_on_upgrade_policy">>},
<0.748.0>                   {pattern,<<"^fed_.*$">>},
<0.748.0>                   {'apply-to',<<"all">>},
<0.748.0>                   {definition,
<0.748.0>                    [{<<"federation-upstream">>,<<"fed_upstream">>}]},
<0.748.0>                   {priority,2}],
<0.748.0>                  undefined,none,
<0.748.0>                  #{user => <<"vhost1">>}},
<0.748.0>                 {rabbit_federation_link_sup,start_link,
<0.748.0>                  [{exchange,
<0.748.0>                    {resource,<<"vhost1">>,exchange,<<"fed_ex">>},
<0.748.0>                    fanout,true,false,false,[],undefined,
<0.748.0>                    [{vhost,<<"vhost1">>},
<0.748.0>                     {name,<<"fed_on_upgrade_policy">>},
<0.748.0>                     {pattern,<<"^fed_.*$">>},
<0.748.0>                     {'apply-to',<<"all">>},
<0.748.0>                     {definition,
<0.748.0>                      [{<<"federation-upstream">>,<<"fed_upstream">>}]},
<0.748.0>                     {priority,2}],
<0.748.0>                    undefined,
<0.748.0>                    {[],
<0.748.0>                     [rabbit_federation_exchange,
<0.748.0>                      rabbit_event_exchange_decorator]},
<0.748.0>                    #{user => <<"vhost1">>}}]},
<0.748.0>                 transient,infinity,supervisor,
<0.748.0>                 [rabbit_federation_link_sup]}}]}
<0.748.0>     offender: [{pid,<0.750.0>},
<0.748.0>                {id,mirroring},
<0.748.0>                {mfargs,
<0.748.0>                    {mirrored_supervisor,start_internal,
<0.748.0>                        [rabbit_federation_exchange_link_sup_sup,[]]}},
<0.748.0>                {restart_type,permanent},
<0.748.0>                {shutdown,4294967295},
<0.748.0>                {child_type,worker}]
<0.748.0>     supervisor: {local,rabbit_federation_exchange_link_sup_sup}
<0.748.0>     errorContext: shutdown
<0.748.0>     reason: reached_max_restart_intensity
<0.748.0>     offender: [{pid,<0.750.0>},
<0.748.0>                {id,mirroring},
<0.748.0>                {mfargs,
<0.748.0>                    {mirrored_supervisor,start_internal,
<0.748.0>                        [rabbit_federation_exchange_link_sup_sup,[]]}},
<0.748.0>                {restart_type,permanent},
<0.748.0>                {shutdown,4294967295},
<0.748.0>                {child_type,worker}]									  

4> supervisor:which_children(rabbit_exchange_federation_sup).
[{x_links,undefined,supervisor,
          [rabbit_federation_exchange_link_sup_sup]}]
5> whereis(rabbit_federation_exchange_link_sup_sup).
undefined

This is because rabbit_federation_link_sup:start_link had arity 1 until 4.1.x. PR #13921 converted it to arity 2 (https://github.com/rabbitmq/rabbitmq-server/pull/13921/changes#diff-331ecadea9665c8db269b5f30f87851eddeb09b6968ec5daeae0c34355cbd4fbR39). But in a mixed version cluster the mirrored supervisor preserves the child definitions which still include a call with arity 1 (without the link module).

Maybe adding a compatibility start_link/1 would solve this?

modified   deps/rabbitmq_federation_common/src/rabbit_federation_link_sup.erl
@@ -17,6 +17,13 @@
 
 -export([start_link/2, adjust/4, restart/2]).
 -export([init/1]).
+-export([start_link/1]).
+
+%% For backwards compatibility
+start_link(X = #exchange{}) ->
+    start_link(rabbit_federation_exchange_link, X);
+start_link(Q) when ?is_amqqueue(Q) ->
+    start_link(rabbit_federation_queue_link, Q).
 
 start_link(LinkMod, Q) ->
     supervisor2:start_link(?MODULE, [LinkMod, Q]).

[michaelklishin]

@gomoripeti the proposed change looks acceptable to me. If we want to include it, we must do it early next week because 4.1.8 will ship in mid-January and it will be the last publicly available 4.1.x release.

[gomoripeti]

I just noticed that there is a test case that tests federation rolling upgrades but it is skipped in mixed version tests 4f86ae0
is that intentional that it is left like this? this test case covers exactly this reported issue.

I will try to make that test case work.

[michaelklishin]

Some tests cannot pass in mixed versions, so they are skipped. We usually can re-enable them once the release series rolls over and whatever wasn't available in the previous series that the test needed, should be available now after the N-1 series bump.

If the previous series is already on 4.2.x, we can try re-enabling that suite.

[michaelklishin]

4.2.3 is scheduled to be built later this week, so we should hurry up and likely include these two functions without investigating whether mixed version tests can be re-introduced.

[gomoripeti]

I am working on the test case in #15252, but if I'm not able to fix it by tomorrow I will remove the test commit