feat: add evictAfterOOMThreshold per vpa

vertical-pod-autoscaler/charts/vertical-pod-autoscaler/crds/vpa-v1-crd-gen.yaml

@@ -254,6 +254,14 @@ spec:
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
+    - jsonPath: .spec.updatePolicy.minReplicas
+      name: MinReplicas
+      priority: 1
+      type: integer
+    - jsonPath: .spec.updatePolicy.evictAfterOOMSeconds
+      name: OOMSeconds
+      priority: 1
+      type: integer
     name: v1
     schema:
       openAPIV3Schema:
@@ -425,6 +433,14 @@ spec:
                   If not specified, all fields in the `PodUpdatePolicy` are set to their
                   default values.
                 properties:
+                  evictAfterOOMSeconds:
+                    description: |-
+                      evictAfterOOMSeconds specifies the time in seconds to wait after an OOM event before
+                      considering the pod for eviction. Pods that have OOMed in less than this time
+                      since start will be evicted.
+                    format: int32
+                    minimum: 1
+                    type: integer
                   evictionRequirements:
                     description: |-
                       EvictionRequirements is a list of EvictionRequirements that need to

vertical-pod-autoscaler/deploy/vpa-v1-crd-gen.yaml

@@ -254,6 +254,14 @@ spec:
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
+    - jsonPath: .spec.updatePolicy.minReplicas
+      name: MinReplicas
+      priority: 1
+      type: integer
+    - jsonPath: .spec.updatePolicy.evictAfterOOMSeconds
+      name: OOMSeconds
+      priority: 1
+      type: integer
     name: v1
     schema:
       openAPIV3Schema:
@@ -425,6 +433,14 @@ spec:
                   If not specified, all fields in the `PodUpdatePolicy` are set to their
                   default values.
                 properties:
+                  evictAfterOOMSeconds:
+                    description: |-
+                      evictAfterOOMSeconds specifies the time in seconds to wait after an OOM event before
+                      considering the pod for eviction. Pods that have OOMed in less than this time
+                      since start will be evicted.
+                    format: int32
+                    minimum: 1
+                    type: integer
                   evictionRequirements:
                     description: |-
                       EvictionRequirements is a list of EvictionRequirements that need to

vertical-pod-autoscaler/docs/api.md

@@ -160,6 +160,7 @@ _Appears in:_
 | `updateMode` _[UpdateMode](#updatemode)_ | Controls when autoscaler applies changes to the pod resources.<br />The default is 'Recreate'. |  | Enum: [Off Initial Recreate InPlaceOrRecreate Auto] <br />Optional: \{\} <br /> |
 | `minReplicas` _integer_ | Minimal number of replicas which need to be alive for Updater to attempt<br />pod eviction (pending other checks like PDB). Only positive values are<br />allowed. Overrides global '--min-replicas' flag. |  | Optional: \{\} <br /> |
 | `evictionRequirements` _[EvictionRequirement](#evictionrequirement) array_ | EvictionRequirements is a list of EvictionRequirements that need to<br />evaluate to true in order for a Pod to be evicted. If more than one<br />EvictionRequirement is specified, all of them need to be fulfilled to allow eviction. |  | Optional: \{\} <br /> |
+| `evictAfterOOMSeconds` _integer_ | evictAfterOOMSeconds specifies the time in seconds to wait after an OOM event before<br />considering the pod for eviction. Pods that have OOMed in less than this time<br />since start will be evicted. |  | Minimum: 1 <br />Optional: \{\} <br /> |
 
 
 #### RecommendedContainerResources

vertical-pod-autoscaler/docs/flags.md

@@ -146,7 +146,7 @@ This document is auto-generated from the flag definitions in the VPA updater cod
 | `add-dir-header` |  |  | If true, adds the file directory to the header of the log messages |
 | `address` | string |  ":8943" | The address to expose Prometheus metrics.  |
 | `alsologtostderr` |  |  | log to standard error as well as files (no effect when -logtostderr=true) |
-| `evict-after-oom-threshold` |  |  10m0s | duration                              Evict pod that has OOMed in less than evict-after-oom-threshold since start.  |
+| `evict-after-oom-threshold` |  |  10m0s | duration                              The default duration to evict pods that have OOMed in less than evict-after-oom-threshold since start.  |
 | `eviction-rate-burst` | int |  1 | Burst of pods that can be evicted.  |
 | `eviction-rate-limit` | float |  | Number of pods that can be evicted per seconds. A rate limit set to 0 or -1 will disable<br>the rate limiter. (default -1) |
 | `eviction-tolerance` | float |  0.5 | Fraction of replica count that can be evicted for update, if more than one pod can be evicted.  |
@@ -181,4 +181,3 @@ This document is auto-generated from the flag definitions in the VPA updater cod
 | `v,` |  | : 4 | , --v Level                                                         set the log level verbosity  (default 4) |
 | `vmodule` | moduleSpec |  | comma-separated list of pattern=N settings for file-filtered logging |
 | `vpa-object-namespace` | string |  | Specifies the namespace to search for VPA objects. Leave empty to include all namespaces. If provided, the garbage collector will only clean this namespace. |
-

vertical-pod-autoscaler/e2e/utils/common.go

@@ -81,6 +81,9 @@ var RecommenderLabels = map[string]string{"app": "vpa-recommender"}
 // HamsterLabels are labels of hamster app
 var HamsterLabels = map[string]string{"app": "hamster"}
 
+// OOMLabels are labels for OOM test pods
+var OOMLabels = map[string]string{"app": "oom-test"}
+
 // SIGDescribe adds sig-autoscaling tag to test description.
 // Takes args that are passed to ginkgo.Describe.
 func SIGDescribe(scenario, name string, args ...any) bool {

vertical-pod-autoscaler/e2e/v1/admission_controller.go

@@ -956,7 +956,7 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() {
 				expectedErr: "admission webhook \"vpa.k8s.io\" denied the request: oomBumpUpRatio must be greater than or equal to 1.0, got -1",
 			},
 			{
-				name: "Invalid oomBumpUpRatio (string value)",
+				name: "Invalid oomBumpUpRatio (less than 1)",
 				vpaJSON: `{
             "apiVersion": "autoscaling.k8s.io/v1",
             "kind": "VerticalPodAutoscaler",
@@ -973,16 +973,16 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() {
                 "resourcePolicy": {
                     "containerPolicies": [{
                         "containerName": "*",
-                        "oomBumpUpRatio": "not-a-number",
+                        "oomBumpUpRatio": 0.5,
                         "oomMinBumpUp": 104857600
                     }]
                 }
             }
         }`,
-				expectedErr: "admission webhook \"vpa\\.k8s\\.io\" denied the request: quantities must match the regular expression",
+				expectedErr: "admission webhook \"vpa.k8s.io\" denied the request: oomBumpUpRatio must be greater than or equal to 1.0, got 0.5",
 			},
 			{
-				name: "Invalid oomBumpUpRatio (less than 1)",
+				name: "Invalid oomMinBumpUp (negative value)",
 				vpaJSON: `{
             "apiVersion": "autoscaling.k8s.io/v1",
             "kind": "VerticalPodAutoscaler",
@@ -999,16 +999,16 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() {
                 "resourcePolicy": {
                     "containerPolicies": [{
                         "containerName": "*",
-                        "oomBumpUpRatio": 0.5,
-                        "oomMinBumpUp": 104857600
+                        "oomBumpUpRatio": 2,
+                        "oomMinBumpUp": -1
                     }]
                 }
             }
         }`,
-				expectedErr: "admission webhook \"vpa.k8s.io\" denied the request: oomBumpUpRatio must be greater than or equal to 1.0, got 0.5",
+				expectedErr: "admission webhook \"vpa.k8s.io\" denied the request: oomMinBumpUp must be greater than or equal to 0, got -1 bytes",
 			},
 			{
-				name: "Invalid oomMinBumpUp (negative value)",
+				name: "Invalid oomBumpUpRatio (string value)",
 				vpaJSON: `{
             "apiVersion": "autoscaling.k8s.io/v1",
             "kind": "VerticalPodAutoscaler",
@@ -1025,13 +1025,13 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() {
                 "resourcePolicy": {
                     "containerPolicies": [{
                         "containerName": "*",
-                        "oomBumpUpRatio": 2,
-                        "oomMinBumpUp": -1
+                        "oomBumpUpRatio": "not-a-number",
+                        "oomMinBumpUp": 104857600
                     }]
                 }
             }
         }`,
-				expectedErr: "admission webhook \"vpa.k8s.io\" denied the request: oomMinBumpUp must be greater than or equal to 0, got -1 bytes",
+				expectedErr: "admission webhook \"vpa.k8s.io\" denied the request: quantities must match the regular expression",
 			},
 		}
 		for _, tc := range testCases {

vertical-pod-autoscaler/e2e/v1/autoscaling_utils.go

@@ -31,6 +31,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/autoscaler/vertical-pod-autoscaler/e2e/utils"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2edebug "k8s.io/kubernetes/test/e2e/framework/debug"
@@ -440,6 +441,7 @@ func runOomingReplicationController(c clientset.Interface, ns, name string, repl
 		Namespace:   ns,
 		Timeout:     timeoutRC,
 		Replicas:    replicas,
+		Labels:      utils.OOMLabels,
 		Annotations: make(map[string]string),
 		MemRequest:  1024 * 1024 * 1024,
 		MemLimit:    1024 * 1024 * 1024,

vertical-pod-autoscaler/e2e/v1/common.go

@@ -144,6 +144,14 @@ func GetHamsterPods(f *framework.Framework) (*apiv1.PodList, error) {
 	return f.ClientSet.CoreV1().Pods(f.Namespace.Name).List(context.TODO(), options)
 }
 
+// GetOOMPods returns running OOM test pods (matched by utils.OOMLabels)
+func GetOOMPods(f *framework.Framework) (*apiv1.PodList, error) {
+	// TODO(omerap12): merge GetHamsterPods and GetOOMPods functions.
+	label := labels.SelectorFromSet(labels.Set(utils.OOMLabels))
+	options := metav1.ListOptions{LabelSelector: label.String(), FieldSelector: getPodSelectorExcludingDonePodsOrDie()}
+	return f.ClientSet.CoreV1().Pods(f.Namespace.Name).List(context.TODO(), options)
+}
+
 // NewTestCronJob returns a CronJob for test purposes.
 func NewTestCronJob(name, schedule string, replicas int32) *batchv1.CronJob {
 	backoffLimit := utils.DefaultHamsterBackoffLimit
@@ -302,6 +310,21 @@ func WaitForPodsEvicted(f *framework.Framework, podList *apiv1.PodList) error {
 	})
 }
 
+// WaitForPodsEvictedOOM waits until some pods from the list are evicted.
+// TODO(omerap12): merge this with WaitForPodsEvicted
+func WaitForPodsEvictedOOM(f *framework.Framework, podList *apiv1.PodList) error {
+	initialPodSet := MakePodSet(podList)
+
+	return wait.PollUntilContextTimeout(context.Background(), utils.PollInterval, utils.PollTimeout, true, func(ctx context.Context) (done bool, err error) {
+		currentPodList, err := GetOOMPods(f)
+		if err != nil {
+			return false, err
+		}
+		currentPodSet := MakePodSet(currentPodList)
+		return GetEvictedPodsCount(currentPodSet, initialPodSet) > 0, nil
+	})
+}
+
 // WerePodsSuccessfullyRestarted returns true if some pods from initialPodSet have been
 // successfully restarted comparing to currentPodSet (pods were evicted and
 // are running).
@@ -342,6 +365,16 @@ func CheckNoPodsEvicted(f *framework.Framework, initialPodSet PodSet) {
 	gomega.Expect(restarted).To(gomega.Equal(0), "there should be no pod evictions")
 }
 
+// CheckNoPodsEvictedOOM waits for long enough period for VPA to start evicting
+// TODO(omerap12): merge this CheckNoPodsEvicted
+func CheckNoPodsEvictedOOM(f *framework.Framework, initialPodSet PodSet) {
+	time.Sleep(VpaEvictionTimeout)
+	currentPodList, err := GetOOMPods(f)
+	gomega.Expect(err).NotTo(gomega.HaveOccurred(), "unexpected error when listing hamster pods to check number of pod evictions")
+	restarted := GetEvictedPodsCount(MakePodSet(currentPodList), initialPodSet)
+	gomega.Expect(restarted).To(gomega.Equal(0), "there should be no pod evictions")
+}
+
 // WaitForUncappedCPURecommendationAbove pools VPA object until uncapped recommendation is above specified value.
 // Returns polled VPA object. On timeout returns error.
 func WaitForUncappedCPURecommendationAbove(c vpa_clientset.Interface, vpa *vpa_types.VerticalPodAutoscaler, minMilliCPU int64) (*vpa_types.VerticalPodAutoscaler, error) {