From 3ca5cd6691f4f357dab1bdf18e1c3b178946ec50 Mon Sep 17 00:00:00 2001 From: markcampv Date: Tue, 14 Mar 2023 16:18:06 -0400 Subject: [PATCH 1/2] Warn and Confirm KV key deletions when replicating --- runner.go | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/runner.go b/runner.go index 80fb16c..9ddc000 100644 --- a/runner.go +++ b/runner.go @@ -374,9 +374,9 @@ func (r *Runner) replicate(prefix *PrefixConfig, excludes *ExcludeConfigs, doneC errCh <- fmt.Errorf("failed to list keys: %s", err) return } + executed := false for _, key := range localKeys { excluded := false - // Ignore if the key falls under an excluded prefix if len(*excludes) > 0 { sourceKey := strings.Replace(key, config.StringVal(prefix.Destination), config.StringVal(prefix.Source), -1) @@ -390,12 +390,40 @@ func (r *Runner) replicate(prefix *PrefixConfig, excludes *ExcludeConfigs, doneC } if _, ok := usedKeys[key]; !ok && !excluded { - if _, err := kv.Delete(key, nil); err != nil { - errCh <- fmt.Errorf("failed to delete %q: %s", key, err) - return + if !executed { + executed = true + var input string + log.Printf("[INFO] Listing keys marked for deletion due to empty keys in primary") + //Wait 5 seconds for the operator to catch the message above in case there are multiple listed keys + time.Sleep(5 * time.Second) + //this will be pretty boilerplate, but this will print the keys marked for deletion in the destination + for _, emptyPrimaryKey := range localKeys { + if _, ok := usedKeys[emptyPrimaryKey]; !ok && !excluded { + fmt.Printf("%q\n", emptyPrimaryKey) + } + } + log.Print("[INFO] Do you want to continue with the deletion? (Y/N)") + fmt.Scan(&input) + if input == "Y" { + log.Printf("[INFO] Continuing...") + for _, emptyPrimaryKey := range localKeys { + if _, ok := usedKeys[emptyPrimaryKey]; !ok && !excluded { + if _, err := kv.Delete(emptyPrimaryKey, nil); err != nil { + errCh <- fmt.Errorf("failed to delete %q: %s", emptyPrimaryKey, err) + return + } + log.Printf("[DEBUG] (runner) deleted %q", emptyPrimaryKey) + deletes++ + } + } + } else if input == "N" { + log.Printf("[INFO] Ok, will not delete listed keys") + break + } else { + log.Printf("[INFO] Invalid input.") + break + } } - log.Printf("[DEBUG] (runner) deleted %q", key) - deletes++ } } From 92d93e43ae871e78110f1da4b259623d559e8e81 Mon Sep 17 00:00:00 2001 From: markcampv Date: Fri, 17 Mar 2023 17:25:21 -0400 Subject: [PATCH 2/2] Add optional delete flag for destination key deletion --- cli.go | 9 +++++++++ config.go | 18 ++++++++++++++++++ runner.go | 47 ++++++++++------------------------------------- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/cli.go b/cli.go index 54ec423..366f259 100644 --- a/cli.go +++ b/cli.go @@ -381,6 +381,11 @@ func (cli *CLI) ParseFlags(args []string) (*Config, []string, bool, bool, error) return nil }), "wait", "") + flags.Var((funcBoolVar)(func(b bool) error { + c.DeleteKey = config.Bool(b) + return nil + }), "delete", "") + flags.BoolVar(&isVersion, "v", false, "") flags.BoolVar(&isVersion, "version", false, "") @@ -618,6 +623,10 @@ Options: Sets the 'min(:max)' amount of time to wait before writing a template (and triggering a command) + -delete= + Enables deletion of keys in the destination datacenter that do not exist + in the source datacenter. Defaults to true + -v, -version Print the version of this daemon ` diff --git a/config.go b/config.go index 1efd211..c24441b 100644 --- a/config.go +++ b/config.go @@ -35,6 +35,8 @@ const ( // DefaultStatusDir is the default directory to post status information. DefaultStatusDir = "service/consul-replicate/statuses" + // DefaultDeleteKey is the default value for deleting destination keys if source keys are empty. + DefaultDeleteKey = true ) // Config is used to configure Consul ENV @@ -74,6 +76,8 @@ type Config struct { // Wait is the quiescence timers. Wait *config.WaitConfig `mapstructure:"wait"` + + DeleteKey *bool `mapstructure:"delete_key"` } // Copy returns a deep copy of the current configuration. This is useful because @@ -113,6 +117,8 @@ func (c *Config) Copy() *Config { o.Wait = c.Wait.Copy() } + o.DeleteKey = c.DeleteKey + return &o } @@ -174,6 +180,10 @@ func (c *Config) Merge(o *Config) *Config { r.Wait = r.Wait.Merge(o.Wait) } + if o.DeleteKey != nil { + r.DeleteKey = o.DeleteKey + } + return r } @@ -195,6 +205,7 @@ func (c *Config) GoString() string { "StatusDir:%s, "+ "Syslog:%s, "+ "Wait:%s"+ + "DeleteKey:%s"+ "}", c.Consul.GoString(), c.Excludes.GoString(), @@ -207,12 +218,14 @@ func (c *Config) GoString() string { config.StringGoString(c.StatusDir), c.Syslog.GoString(), c.Wait.GoString(), + config.BoolGoString(c.DeleteKey), ) } // DefaultConfig returns the default configuration struct. Certain environment // variables may be set which control the values for the default configuration. func DefaultConfig() *Config { + return &Config{ Consul: config.DefaultConsulConfig(), Excludes: DefaultExcludeConfigs(), @@ -220,6 +233,7 @@ func DefaultConfig() *Config { StatusDir: config.String(DefaultStatusDir), Syslog: config.DefaultSyslogConfig(), Wait: config.DefaultWaitConfig(), + DeleteKey: config.Bool(DefaultDeleteKey), } } @@ -284,6 +298,10 @@ func (c *Config) Finalize() { c.Wait = config.DefaultWaitConfig() } c.Wait.Finalize() + + if c.DeleteKey == nil { + c.DeleteKey = config.Bool(DefaultDeleteKey) + } } // Parse parses the given string contents as a config diff --git a/runner.go b/runner.go index 9ddc000..63bd27d 100644 --- a/runner.go +++ b/runner.go @@ -201,7 +201,7 @@ func (r *Runner) Run() error { // Replicate each prefix in a goroutine for _, prefix := range prefixes { - go r.replicate(prefix, r.config.Excludes, doneCh, errCh) + go r.replicate(prefix, r.config.Excludes, r.config.DeleteKey, doneCh, errCh) } var errs *multierror.Error @@ -268,7 +268,7 @@ func (r *Runner) get(prefix *PrefixConfig) (*watch.View, bool) { // replicate performs replication into the current datacenter from the given // prefix. This function is designed to be called via a goroutine since it is // expensive and needs to be parallelized. -func (r *Runner) replicate(prefix *PrefixConfig, excludes *ExcludeConfigs, doneCh chan struct{}, errCh chan error) { +func (r *Runner) replicate(prefix *PrefixConfig, excludes *ExcludeConfigs, deleteKey *bool, doneCh chan struct{}, errCh chan error) { // Ensure we are not self-replicating info, err := r.clients.Consul().Agent().Self() if err != nil { @@ -374,7 +374,6 @@ func (r *Runner) replicate(prefix *PrefixConfig, excludes *ExcludeConfigs, doneC errCh <- fmt.Errorf("failed to list keys: %s", err) return } - executed := false for _, key := range localKeys { excluded := false // Ignore if the key falls under an excluded prefix @@ -389,41 +388,15 @@ func (r *Runner) replicate(prefix *PrefixConfig, excludes *ExcludeConfigs, doneC } } - if _, ok := usedKeys[key]; !ok && !excluded { - if !executed { - executed = true - var input string - log.Printf("[INFO] Listing keys marked for deletion due to empty keys in primary") - //Wait 5 seconds for the operator to catch the message above in case there are multiple listed keys - time.Sleep(5 * time.Second) - //this will be pretty boilerplate, but this will print the keys marked for deletion in the destination - for _, emptyPrimaryKey := range localKeys { - if _, ok := usedKeys[emptyPrimaryKey]; !ok && !excluded { - fmt.Printf("%q\n", emptyPrimaryKey) - } - } - log.Print("[INFO] Do you want to continue with the deletion? (Y/N)") - fmt.Scan(&input) - if input == "Y" { - log.Printf("[INFO] Continuing...") - for _, emptyPrimaryKey := range localKeys { - if _, ok := usedKeys[emptyPrimaryKey]; !ok && !excluded { - if _, err := kv.Delete(emptyPrimaryKey, nil); err != nil { - errCh <- fmt.Errorf("failed to delete %q: %s", emptyPrimaryKey, err) - return - } - log.Printf("[DEBUG] (runner) deleted %q", emptyPrimaryKey) - deletes++ - } - } - } else if input == "N" { - log.Printf("[INFO] Ok, will not delete listed keys") - break - } else { - log.Printf("[INFO] Invalid input.") - break - } + if _, ok := usedKeys[key]; !ok && !excluded && *deleteKey { + if _, err := kv.Delete(key, nil); err != nil { + errCh <- fmt.Errorf("failed to delete %q: %s", key, err) + return } + log.Printf("[DEBUG] (runner) deleted %q", key) + deletes++ + } else if _, ok := usedKeys[key]; !ok && !excluded && !*deleteKey { + log.Printf("DEBUG (runner) %q key exists in destination and not source", key) } }