MDEV-34924 : gtid_slave_pos table neven been deleted on non replica n…

…odes (wsrep_gtid_mode = 1)

Problem was caused by MDEV-31413 commit 277968a where
mysql.gtid_slave_pos table was replicated by Galera.
However, as not all nodes in Galera cluster are replica
nodes, rows were not deleted from table.

In this fix this is corrected so that mysql.gtid_slave_pos
table is not replicated by Galera. Instead when Galera
node receives GTID event and wsrep_gtid_mode=1, this event
is stored to mysql.gtid_slave_pos table

Added test case galera_2primary_replica for 2 async
primaries replicating to galera cluster.

Added test case galera_circular_replication where
async primary replicates to galera cluster and
one of the galera cluster nodes is master
to async replica.

Modified test case galera_restart_replica to monitor
gtid positions and rows in mysql.gtid_pos_table

mysql-test/suite/galera/galera_2nodes_as_replica_2primary.cnf

@@ -0,0 +1,70 @@
+#
+# This .cnf file creates a setup with 2 standard MariaDB servers, followed by a 2-node Galera cluster
+#
+
+# Use default setting for mysqld processes
+!include include/default_mysqld.cnf
+
+[mysqld]
+loose-innodb
+log-bin=mysqld-bin
+log-slave-updates
+binlog-format=row
+innodb-autoinc-lock-mode=2
+default-storage-engine=innodb
+# enforce read-committed characteristics across the cluster
+# wsrep-causal-reads=ON
+wsrep-sync-wait=15
+
+[mysqld.1]
+wsrep-on=1
+server-id=1
+#[email protected]
+#[email protected]
+#[email protected]
+wsrep_provider[email protected]_PROVIDER
+wsrep_cluster_address=gcomm://
+wsrep_provider_options='repl.causal_read_timeout=PT90S;[email protected].#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;gcache.size=10M'
+wsrep_node_address='127.0.0.1:@mysqld.1.#galera_port'
+wsrep_node_incoming_address=127.0.0.1:@mysqld.1.port
+wsrep_sst_receive_address='127.0.0.1:@mysqld.1.#sst_port'
+
+[mysqld.2]
+wsrep-on=1
+server-id=2
+#[email protected]
+#[email protected]
+#[email protected]
+wsrep_provider[email protected]_PROVIDER
+wsrep_cluster_address='gcomm://127.0.0.1:@mysqld.1.#galera_port'
+wsrep_provider_options='repl.causal_read_timeout=PT90S;[email protected].#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;gcache.size=10M'
+wsrep_node_address='127.0.0.1:@mysqld.2.#galera_port'
+wsrep_node_incoming_address=127.0.0.1:@mysqld.2.port
+wsrep_sst_receive_address='127.0.0.1:@mysqld.2.#sst_port'
+
+[mysqld.3]
+wsrep-on=OFF
+server-id=3
+gtid_domain_id=3
+
+[mysqld.4]
+wsrep-on=OFF
+server-id=4
+gtid_domain_id=4
+
+
+[sst]
+sst-log-archive-dir[email protected]_VARDIR/log
+
+[ENV]
+NODE_MYPORT_1= @mysqld.1.port
+NODE_MYSOCK_1= @mysqld.1.socket
+
+NODE_MYPORT_2= @mysqld.2.port
+NODE_MYSOCK_2= @mysqld.2.socket
+
+NODE_MYPORT_3= @mysqld.3.port
+NODE_MYSOCK_3= @mysqld.3.socket
+
+NODE_MYPORT_4= @mysqld.4.port
+NODE_MYSOCK_4= @mysqld.4.socket

mysql-test/suite/galera/r/galera_2primary_replica.result

@@ -0,0 +1,95 @@
+connection node_2;
+connection node_1;
+connect primary1, 127.0.0.1, root, , test, $NODE_MYPORT_3;
+connect primary2, 127.0.0.1, root, , test, $NODE_MYPORT_4;
+connection primary1;
+# Primary1 creating user for replication
+create user repl@'%' identified by 'repl';
+grant all on *.* to  repl@'%';
+connection primary2;
+# Primary2 creating user for replication
+create user repl2@'%' identified by 'repl2';
+grant all on *.* to  repl2@'%';
+connect replica, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connection replica;
+connection node_2;
+connection replica;
+# Galera replica changing master to primary1
+SET @@default_master_connection='stream2';
+# Primary node changing master to primary2
+START ALL SLAVES;
+Warnings:
+Note	1937	SLAVE 'stream1' started
+Note	1937	SLAVE 'stream2' started
+connection primary1;
+# Primary 1: Creating table and populating it with data
+CREATE TABLE t1 (id bigint auto_increment primary key, msg varchar(100)) engine=innodb;
+# Intentionally generate 1k GTID-events
+SELECT COUNT(*) AS EXPECT_1000 FROM t1;
+EXPECT_1000
+1000
+connection primary2;
+# Primary 2: Creating table and populating it with data
+CREATE TABLE t2 (id bigint auto_increment primary key, msg varchar(100)) engine=innodb;
+# Intentionally generate 1k GTID-events
+SELECT COUNT(*) AS EXPECT_1000 FROM t2;
+EXPECT_1000
+1000
+connection replica;
+# Waiting for data to replicate to node_1
+SELECT COUNT(*) AS EXPECT_1000 FROM t1;
+EXPECT_1000
+1000
+SELECT COUNT(*) AS EXPECT_1000 FROM t2;
+EXPECT_1000
+1000
+SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos;
+EXPECT_1
+1
+SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos;
+EXPECT_1
+1
+SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos;
+@@gtid_slave_pos	@@gtid_binlog_pos	@@gtid_current_pos
+3-3-1003,4-4-1003	3-3-1003,4-4-1003	3-3-1003,4-4-1003
+connection node_2;
+# Waiting for data to replicate to node_2
+SELECT COUNT(*) AS EXPECT_1000 FROM t1;
+EXPECT_1000
+1000
+SELECT COUNT(*) AS EXPECT_1000 FROM t2;
+EXPECT_1000
+1000
+SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos;
+EXPECT_1
+1
+SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos;
+EXPECT_1
+1
+SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos;
+@@gtid_slave_pos	@@gtid_binlog_pos	@@gtid_current_pos
+3-3-1003,4-4-1003	3-3-1003,4-4-1003	3-3-1003,4-4-1003
+connection primary1;
+drop table t1;
+connection primary2;
+drop table t2;
+# Wait until drop table is replicated on Galera
+connection replica;
+connection node_2;
+connection replica;
+STOP ALL SLAVES;
+Warnings:
+Note	1938	SLAVE 'stream1' stopped
+Note	1938	SLAVE 'stream2' stopped
+RESET SLAVE ALL;
+connection primary1;
+RESET MASTER;
+connection primary2;
+RESET MASTER;
+connection node_1;
+disconnect primary1;
+disconnect primary2;
+disconnect replica;
+disconnect node_2;
+disconnect node_1;
+# End of test

mysql-test/suite/galera/r/galera_circular_replication.result

@@ -0,0 +1,138 @@
+connection node_2;
+connection node_1;
+connect replica1, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connect primary2, 127.0.0.1, root, , test, $NODE_MYPORT_3;
+connect primary1, 127.0.0.1, root, , test, $NODE_MYPORT_4;
+connect replica2, 127.0.0.1, root, , test, $NODE_MYPORT_4;
+connection primary1;
+# Primary1 node creating user for replication
+create user repl@'%' identified by 'repl';
+grant all on *.* to  repl@'%';
+ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB;
+connection replica1;
+connection node_2;
+connection primary2;
+connection replica1;
+# Galera replica changing master to primary1
+START SLAVE;
+connection primary2;
+# Primary2 creating user for replication
+create user repl2@'%' identified by 'repl2';
+grant all on *.* to  repl2@'%';
+connection replica2;
+# replica2 changing master to primary2
+START SLAVE;
+connection primary1;
+# Primary1: Creating table and populating it with data
+CREATE TABLE t1 (id bigint auto_increment primary key, msg varchar(100)) engine=innodb;
+# Intentionally generate 1k GTID-events
+SELECT COUNT(*) AS EXPECT_1000 FROM t1;
+EXPECT_1000
+1000
+connection replica1;
+# Waiting for data to replicate to replica
+SELECT COUNT(*) AS EXPECT_1000 FROM t1;
+EXPECT_1000
+1000
+# Writing more data to table
+# Intentionally generate 1k GTID-events
+SELECT COUNT(*) AS EXPECT_2000 FROM t1;
+EXPECT_2000
+2000
+connection node_2;
+# Waiting for data to replicate to Galera node_2
+SELECT COUNT(*) AS EXPECT_2000 FROM t1;
+EXPECT_2000
+2000
+# Writing more data to table
+# Intentionally generate 1k GTID-events
+SELECT COUNT(*) AS EXPECT_3000 FROM t1;
+EXPECT_3000
+3000
+connection primary2;
+# Waiting for data to replicate to primary2
+SELECT COUNT(*) AS EXPECT_3000 FROM t1;
+EXPECT_3000
+3000
+# Writing more data to table
+# Intentionally generate 1k GTID-events
+SELECT COUNT(*) AS EXPECT_4000 FROM t1;
+EXPECT_4000
+4000
+connection primary1;
+# Waiting for data to replicate to primary1
+SELECT COUNT(*) AS EXPECT_4000 FROM t1;
+EXPECT_4000
+4000
+SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos;
+EXPECT_1
+1
+SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos;
+EXPECT_1
+1
+SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos;
+@@gtid_slave_pos	@@gtid_binlog_pos	@@gtid_current_pos
+0-4-1004,16-15-3002	0-4-1004,16-15-3002	0-4-1004,16-15-3002
+connection replica1;
+# Waiting for data to replicate to replica
+SELECT COUNT(*) AS EXPECT_4000 FROM t1;
+EXPECT_4000
+4000
+SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos;
+EXPECT_1
+1
+SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos;
+EXPECT_1
+1
+SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos;
+@@gtid_slave_pos	@@gtid_binlog_pos	@@gtid_current_pos
+0-4-1004,16-15-3002	0-4-1004,16-15-3002	0-4-1004,16-15-3002
+connection node_2;
+# Waiting for data to replicate to node_2
+SELECT COUNT(*) AS EXPECT_4000 FROM t1;
+EXPECT_4000
+4000
+SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos;
+EXPECT_1
+1
+SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos;
+EXPECT_1
+1
+SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos;
+@@gtid_slave_pos	@@gtid_binlog_pos	@@gtid_current_pos
+0-4-1004	0-4-1004,16-15-3002	0-4-1004,16-15-3002
+connection primary2;
+# Waiting for data to replicate to node_3
+SELECT COUNT(*) AS EXPECT_4000 FROM t1;
+EXPECT_4000
+4000
+SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos;
+EXPECT_1
+1
+SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos;
+EXPECT_1
+1
+SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos;
+@@gtid_slave_pos	@@gtid_binlog_pos	@@gtid_current_pos
+0-4-1004	0-4-1004,16-15-3002	0-4-1004,16-15-3002
+connection primary1;
+drop table t1;
+# Wait until drop table is replicated on Galera
+connection replica1;
+connection node_2;
+connection primary2;
+connection replica1;
+STOP SLAVE;
+RESET SLAVE ALL;
+connection replica2;
+STOP SLAVE;
+RESET SLAVE ALL;
+RESET MASTER;
+connection node_1;
+disconnect primary1;
+disconnect replica1;
+disconnect primary2;
+disconnect replica2;
+disconnect node_2;
+disconnect node_1;
+# End of test