vtorc: add flag to set collection metric retention/ttl

Signed-off-by: Tim Vaillancourt <tim@timvaillancourt.com>

Author: timvaillancourt

go/flags/endtoend/vtorc.txt

@@ -35,6 +35,7 @@ Flags:
       --config-persistence-min-interval duration                    minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s)
       --config-type string                                          Config file type (omit to infer config type from file extension).
       --consul-auth-static-file string                              JSON File to read the topos/tokens from.
+      --discovery-collection-metrics-ttl duration                   The time-to-live for discovery collection metrics (default 2m0s)
       --discovery-workers int                                       Number of workers used for tablet discovery (default 300)
       --emit-stats                                                  If set, emit stats to push-based monitoring and stats backends
       --enable-primary-disk-stalled-recovery                        Whether VTOrc should detect a stalled disk on the primary and failover

go/vt/vtorc/collection/collection.go

@@ -45,7 +45,7 @@ data which is suitable for easy collection by monitoring systems.
 
 Expiry is triggered by default if the collection is created via
 CreateOrReturnCollection() and uses an expiry period of
-DiscoveryCollectionRetentionSeconds. It can also be enabled by
+config.GetDiscoveryCollectionMetricsTTL(). It can also be enabled by
 calling StartAutoExpiration() after setting the required expire
 period with SetExpirePeriod().
 
@@ -54,7 +54,7 @@ of metrics which have passed the time specified. Not enabling expiry
 will mean data is collected but never freed which will make
 vtorc run out of memory eventually.
 
-Current code uses DiscoveryCollectionRetentionSeconds as the
+Current code uses config.GetDiscoveryCollectionMetricsTTL() as the
 time to keep metric data.
 */
 package collection
@@ -111,7 +111,7 @@ func CreateOrReturnCollection(name string) *Collection {
 		collection: nil,
 		done:       make(chan struct{}),
 		// WARNING: use a different configuration name
-		expirePeriod: time.Duration(config.DiscoveryCollectionRetentionSeconds) * time.Second,
+		expirePeriod: config.GetDiscoveryCollectionMetricsTTL(),
 	}
 	go qmc.StartAutoExpiration()
 

go/vt/vtorc/config/config.go

@@ -34,7 +34,6 @@ const (
 	StaleInstanceCoordinatesExpireSeconds = 60
 	DiscoveryQueueCapacity                = 100000
 	DiscoveryQueueMaxStatisticsSize       = 120
-	DiscoveryCollectionRetentionSeconds   = 120
 	UnseenInstanceForgetHours             = 240 // Number of hours after which an unseen instance is forgotten
 )
 
@@ -218,6 +217,15 @@ var (
 			Dynamic:  true,
 		},
 	)
+
+	discoveryCollectionMetricsTTL = viperutil.Configure(
+		"discovery-collection-metrics-ttl",
+		viperutil.Options[time.Duration]{
+			FlagName: "discovery-collection-metrics-ttl",
+			Default:  120 * time.Second,
+			Dynamic:  true,
+		},
+	)
 )
 
 func init() {
@@ -246,6 +254,7 @@ func registerFlags(fs *pflag.FlagSet) {
 	fs.Bool("allow-recovery", allowRecovery.Default(), "Whether VTOrc should be allowed to run recovery actions")
 	fs.Bool("change-tablets-with-errant-gtid-to-drained", convertTabletsWithErrantGTIDs.Default(), "Whether VTOrc should be changing the type of tablets with errant GTIDs to DRAINED")
 	fs.Bool("enable-primary-disk-stalled-recovery", enablePrimaryDiskStalledRecovery.Default(), "Whether VTOrc should detect a stalled disk on the primary and failover")
+	fs.Duration("discovery-collection-metrics-ttl", discoveryCollectionMetricsTTL.Default(), "The time-to-live for discovery collection metrics")
 
 	viperutil.BindFlags(fs,
 		instancePollTime,
@@ -268,6 +277,7 @@ func registerFlags(fs *pflag.FlagSet) {
 		allowRecovery,
 		convertTabletsWithErrantGTIDs,
 		enablePrimaryDiskStalledRecovery,
+		discoveryCollectionMetricsTTL,
 	)
 }
 
@@ -381,6 +391,11 @@ func GetRecoveryPollDuration() time.Duration {
 	return recoveryPollDuration.Get()
 }
 
+// GetDiscoveryCollectionMetricsTTL is a getter function.
+func GetDiscoveryCollectionMetricsTTL() time.Duration {
+	return discoveryCollectionMetricsTTL.Get()
+}
+
 // ERSEnabled reports whether VTOrc is allowed to run ERS or not.
 func ERSEnabled() bool {
 	return ersEnabled.Get()