Add VTOrc recovery for mismatch in tablet type (vitessio#17870)

Signed-off-by: Manan Gupta <manan@planetscale.com>

Author: GuptaManan100

go/test/endtoend/cluster/cluster_process.go

@@ -148,6 +148,14 @@ type Vttablet struct {
 	VttabletProcess  *VttabletProcess
 }
 
+// GetAlias returns the tablet alias.
+func (tablet *Vttablet) GetAlias() *topodatapb.TabletAlias {
+	return &topodatapb.TabletAlias{
+		Cell: tablet.Cell,
+		Uid:  uint32(tablet.TabletUID),
+	}
+}
+
 // Keyspace : Cluster accepts keyspace to launch it
 type Keyspace struct {
 	Name             string

go/test/endtoend/cluster/cluster_util.go

@@ -30,6 +30,7 @@ import (
 
 	"vitess.io/vitess/go/vt/grpcclient"
 	"vitess.io/vitess/go/vt/log"
+	replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata"
 	"vitess.io/vitess/go/vt/vtgate/grpcvtgateconn"
 
 	"github.com/buger/jsonparser"
@@ -101,6 +102,15 @@ func GetPrimaryPosition(t *testing.T, vttablet Vttablet, hostname string) (strin
 	return pos, gtID
 }
 
+// FullStatus gets the full status from the given tablet.
+func FullStatus(t *testing.T, vttablet *Vttablet, hostname string) *replicationdatapb.FullStatus {
+	ctx := context.Background()
+	vtablet := getTablet(vttablet.GrpcPort, hostname)
+	status, err := tmClient.FullStatus(ctx, vtablet)
+	require.NoError(t, err)
+	return status
+}
+
 // VerifyRowsInTabletForTable verifies the total number of rows in a table.
 // This is used to check that replication has caught up with the changes on primary.
 func VerifyRowsInTabletForTable(t *testing.T, vttablet *Vttablet, ksName string, expectedRows int, tableName string) {

go/test/endtoend/vtorc/general/vtorc_test.go

@@ -26,9 +26,11 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"vitess.io/vitess/go/mysql"
+	"vitess.io/vitess/go/protoutil"
 	"vitess.io/vitess/go/test/endtoend/cluster"
 	"vitess.io/vitess/go/test/endtoend/vtorc/utils"
 	"vitess.io/vitess/go/vt/log"
+	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
 	"vitess.io/vitess/go/vt/vtorc/inst"
 	"vitess.io/vitess/go/vt/vtorc/logic"
 )
@@ -335,6 +337,34 @@ func TestVTOrcRepairs(t *testing.T) {
 			0,
 		)
 	})
+
+	t.Run("Primary tablet's display type doesn't match the topo record", func(t *testing.T) {
+		// There is no easy way to make a tablet type mismatch with the topo record.
+		// In production this only happens when the call to update the topo record fails with a timeout,
+		// but the operation has succeeded. We can't reliably simulate this in a test.
+		// So, instead we are explicitly changing the tablet record for one of the tablets
+		// to make it a primary and see that VTOrc detects the mismatch and promotes the tablet.
+
+		// Initially check that replication is working as intended
+		utils.CheckReplication(t, clusterInfo, curPrimary, []*cluster.Vttablet{replica, otherReplica}, 15*time.Second)
+
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		_, err := clusterInfo.Ts.UpdateTabletFields(ctx, replica.GetAlias(), func(tablet *topodatapb.Tablet) error {
+			tablet.Type = topodatapb.TabletType_PRIMARY
+			tablet.PrimaryTermStartTime = protoutil.TimeToProto(time.Now())
+			return nil
+		})
+		require.NoError(t, err)
+
+		// Wait for VTOrc to detect the mismatch and promote the tablet.
+		require.Eventuallyf(t, func() bool {
+			fs := cluster.FullStatus(t, replica, clusterInfo.ClusterInstance.Hostname)
+			return fs.TabletType == topodatapb.TabletType_PRIMARY
+		}, 10*time.Second, 1*time.Second, "Primary tablet's display type didn't match the topo record")
+		// Also check that the replica gets promoted and can accept writes.
+		utils.CheckReplication(t, clusterInfo, replica, []*cluster.Vttablet{curPrimary, otherReplica}, 15*time.Second)
+	})
 }
 
 func TestRepairAfterTER(t *testing.T) {

go/test/endtoend/vtorc/readtopologyinstance/main_test.go

@@ -24,6 +24,7 @@ import (
 
 	"vitess.io/vitess/go/test/endtoend/cluster"
 	"vitess.io/vitess/go/test/endtoend/vtorc/utils"
+	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
 	"vitess.io/vitess/go/vt/servenv"
 	"vitess.io/vitess/go/vt/vtorc/config"
 	"vitess.io/vitess/go/vt/vtorc/inst"
@@ -74,6 +75,7 @@ func TestReadTopologyInstanceBufferable(t *testing.T) {
 	require.NotNil(t, primaryInstance)
 	assert.Equal(t, utils.Hostname, primaryInstance.Hostname)
 	assert.Equal(t, primary.MySQLPort, primaryInstance.Port)
+	assert.Equal(t, topodatapb.TabletType_PRIMARY, primaryInstance.TabletType)
 	assert.Contains(t, primaryInstance.InstanceAlias, "zone1")
 	assert.NotEqual(t, 0, primaryInstance.ServerID)
 	assert.Greater(t, len(primaryInstance.ServerUUID), 10)
@@ -125,6 +127,7 @@ func TestReadTopologyInstanceBufferable(t *testing.T) {
 	require.NotNil(t, replicaInstance)
 	assert.Equal(t, utils.Hostname, replicaInstance.Hostname)
 	assert.Equal(t, replica.MySQLPort, replicaInstance.Port)
+	assert.Equal(t, topodatapb.TabletType_REPLICA, replicaInstance.TabletType)
 	assert.Contains(t, replicaInstance.InstanceAlias, "zone1")
 	assert.NotEqual(t, 0, replicaInstance.ServerID)
 	assert.Greater(t, len(replicaInstance.ServerUUID), 10)

go/vt/proto/replicationdata/replicationdata.pb.go

@@ -44,6 +44,7 @@ CREATE TABLE database_instance (
 	alias varchar(256) NOT NULL,
 	hostname varchar(128) NOT NULL,
 	port smallint NOT NULL,
+	tablet_type smallint(5) NOT NULL,
 	last_checked timestamp not null default (''),
 	last_seen timestamp NULL DEFAULT NULL,
 	server_id int NOT NULL,

go/vt/proto/replicationdata/replicationdata_vtproto.pb.go

@@ -38,6 +38,7 @@ const (
 	DeadPrimaryAndSomeReplicas             AnalysisCode = "DeadPrimaryAndSomeReplicas"
 	PrimaryHasPrimary                      AnalysisCode = "PrimaryHasPrimary"
 	PrimaryIsReadOnly                      AnalysisCode = "PrimaryIsReadOnly"
+	PrimaryCurrentTypeMismatch             AnalysisCode = "PrimaryCurrentTypeMismatch"
 	PrimarySemiSyncMustBeSet               AnalysisCode = "PrimarySemiSyncMustBeSet"
 	PrimarySemiSyncMustNotBeSet            AnalysisCode = "PrimarySemiSyncMustNotBeSet"
 	ReplicaIsWritable                      AnalysisCode = "ReplicaIsWritable"
@@ -88,6 +89,7 @@ type ReplicationAnalysis struct {
 	AnalyzedInstanceAlias        string
 	AnalyzedInstancePrimaryAlias string
 	TabletType                   topodatapb.TabletType
+	CurrentTabletType            topodatapb.TabletType
 	PrimaryTimeStamp             time.Time
 	ClusterDetails               ClusterInfo
 	AnalyzedKeyspace             string

go/vt/vtorc/db/generate_base.go

@@ -158,6 +158,9 @@ func GetReplicationAnalysis(keyspace string, shard string, hints *ReplicationAna
 		MIN(
 			primary_instance.semi_sync_replica_enabled
 		) AS semi_sync_replica_enabled,
+		MIN(
+			primary_instance.tablet_type
+		) AS current_tablet_type,
 		SUM(replica_instance.oracle_gtid) AS count_oracle_gtid_replicas,
 		IFNULL(
 			SUM(
@@ -300,6 +303,7 @@ func GetReplicationAnalysis(keyspace string, shard string, hints *ReplicationAna
 		}
 
 		a.TabletType = tablet.Type
+		a.CurrentTabletType = topodatapb.TabletType(m.GetInt("current_tablet_type"))
 		a.AnalyzedKeyspace = m.GetString("keyspace")
 		a.AnalyzedShard = m.GetString("shard")
 		a.PrimaryTimeStamp = m.GetTime("primary_timestamp")
@@ -448,6 +452,9 @@ func GetReplicationAnalysis(keyspace string, shard string, hints *ReplicationAna
 			a.Analysis = PrimarySemiSyncMustNotBeSet
 			a.Description = "Primary semi-sync must not be set"
 			//
+		} else if a.IsClusterPrimary && a.CurrentTabletType != topodatapb.TabletType_PRIMARY {
+			a.Analysis = PrimaryCurrentTypeMismatch
+			a.Description = "Primary tablet's current type is not PRIMARY"
 		} else if topo.IsReplicaType(a.TabletType) && a.ErrantGTID != "" {
 			a.Analysis = ErrantGTIDDetected
 			a.Description = "Tablet has errant GTIDs"