Merge pull request openshift#209 from jeefy/138

SDCICD-138: Add additional pod health checks to runner jobs

Author: openshift-merge-robot

pkg/runner/pod.go

@@ -18,6 +18,9 @@ import (
 const (
 	configMapCreateTimeout = 30 * time.Second
 	podCreateTimeout       = 90 * time.Second
+	podPendingTimeout      = 20 // 20 Iterations
+	fastPoll               = 5 * time.Second
+	slowPoll               = 15 * time.Second
 
 	resultsPort     = 8000
 	resultsPortName = "results"
@@ -111,7 +114,7 @@ func (r *Runner) createPod() (pod *kubev1.Pod, err error) {
 			return nil, fmt.Errorf("error creating ConfigMap: %v", err)
 		}
 
-		err = wait.PollImmediate(5*time.Second, configMapCreateTimeout, func() (done bool, err error) {
+		err = wait.PollImmediate(fastPoll, configMapCreateTimeout, func() (done bool, err error) {
 			if configMap, err = r.Kube.CoreV1().ConfigMaps(r.Namespace).Get(configMap.Name, metav1.GetOptions{}); err != nil {
 				log.Printf("Error creating %s config map: %v", configMap.Name, err)
 			}
@@ -144,7 +147,7 @@ func (r *Runner) createPod() (pod *kubev1.Pod, err error) {
 
 	// retry until Pod can be created or timeout occurs
 	var createdPod *kubev1.Pod
-	err = wait.PollImmediate(5*time.Second, podCreateTimeout, func() (done bool, err error) {
+	err = wait.PollImmediate(fastPoll, podCreateTimeout, func() (done bool, err error) {
 		if createdPod, err = r.Kube.CoreV1().Pods(r.Namespace).Create(pod); err != nil {
 			log.Printf("Error creating %s runner Pod: %v", r.Name, err)
 		}
@@ -154,17 +157,22 @@ func (r *Runner) createPod() (pod *kubev1.Pod, err error) {
 }
 
 func (r *Runner) waitForPodRunning(pod *kubev1.Pod) error {
-	return wait.PollImmediate(10*time.Second, 3*time.Minute, func() (done bool, err error) {
+	var pendingCount int = 0
+	return wait.PollImmediate(fastPoll, 3*time.Minute, func() (done bool, err error) {
 		pod, err = r.Kube.CoreV1().Pods(pod.Namespace).Get(pod.Name, metav1.GetOptions{})
 		if err != nil && !kerror.IsNotFound(err) {
 			return
 		} else if pod == nil {
 			err = errors.New("pod can't be nil")
-		} else if pod.Status.Phase == kubev1.PodFailed {
-			err = errors.New("failed waiting for Pod: the Pod has failed")
+		} else if pod.Status.Phase == kubev1.PodFailed || pod.Status.Phase == kubev1.PodUnknown {
+			err = fmt.Errorf("failed waiting for Pod: the Pod has a phase of %s", pod.Status.Phase)
 		} else if pod.Status.Phase == kubev1.PodRunning {
 			done = true
 		} else {
+			pendingCount++
+			if pendingCount > podPendingTimeout {
+				err = errors.New("timed out waiting for pod to start")
+			}
 			r.Printf("Waiting for Pod '%s/%s' to start Running...", pod.Namespace, pod.Name)
 		}
 		return

pkg/runner/service.go

@@ -33,7 +33,8 @@ func (r *Runner) createService(pod *kubev1.Pod) (svc *kubev1.Service, err error)
 
 func (r *Runner) waitForCompletion(timeoutInSeconds int) error {
 	var endpoints *kubev1.Endpoints
-	return wait.PollImmediate(15*time.Second, time.Duration(timeoutInSeconds)*time.Second, func() (done bool, err error) {
+	var pendingCount int = 0
+	return wait.PollImmediate(slowPoll, time.Duration(timeoutInSeconds)*time.Second, func() (done bool, err error) {
 		endpoints, err = r.Kube.CoreV1().Endpoints(r.svc.Namespace).Get(r.svc.Name, metav1.GetOptions{})
 		if err != nil && !kerror.IsNotFound(err) {
 			r.Printf("Encountered error getting endpoint '%s/%s': %v", r.svc.Namespace, r.svc.Name, err)
@@ -50,11 +51,16 @@ func (r *Runner) waitForCompletion(timeoutInSeconds int) error {
 			return false, err
 		}
 		for _, pod := range pods.Items {
-			if pod.Status.Phase == kubev1.PodFailed {
+			if pod.Status.Phase == kubev1.PodFailed || pod.Status.Phase == kubev1.PodUnknown {
 				r.Printf("Pod entered error state while waiting for endpoint: %+v", pod.Status)
 				return false, fmt.Errorf("pod failed while waiting for endpoints")
 			} else if pod.Status.Phase == kubev1.PodSucceeded {
 				return true, nil
+			} else if pod.Status.Phase == kubev1.PodPending {
+				pendingCount++
+				if pendingCount > podPendingTimeout {
+					return false, fmt.Errorf("timed out waiting for pod to start")
+				}
 			}
 		}
 		r.Printf("Waiting for test results using Endpoint '%s/%s'...", endpoints.Namespace, endpoints.Name)