Skip to content

Commit 65e603a

Browse files
shreddedbaconshreddedbacon
authored
Merge pull request #53 from amazeeio/env-cleanup
Clean up env before deleting namespaces
2 parents 8cc7358 + 8af6b40 commit 65e603a

File tree

3 files changed

+333
-42
lines changed

3 files changed

+333
-42
lines changed

‎controller-test.sh‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -324,6 +324,38 @@ fi
324324
done
325325
kubectl logs $(kubectl get pods -n ${CONTROLLER_NAMESPACE} --no-headers | awk '{print $1}') -c manager -n ${CONTROLLER_NAMESPACE} | grep "handlers.RotateRobotCredentials"
326326

327+
echo "==> Delete the environment"
328+
echo '
329+
{"properties":{"delivery_mode":2},"routing_key":"ci-local-controller-kubernetes:remove",
330+
"payload":"{
331+
\"projectName\": \"drupal-example\",
332+
\"type\":\"branch\",
333+
\"forceDeleteProductionEnvironment\":true,
334+
\"branch\":\"install\",
335+
\"openshiftProjectName\":\"drupal-example-install\"
336+
}",
337+
"payload_encoding":"string"
338+
}' >payload.json
339+
curl -s -u guest:guest -H "Accept: application/json" -H "Content-Type:application/json" -X POST -d @payload.json http://172.17.0.1:15672/api/exchanges/%2f/lagoon-tasks/publish
340+
echo ""
341+
CHECK_COUNTER=1
342+
until $(kubectl logs $(kubectl get pods -n ${CONTROLLER_NAMESPACE} --no-headers | awk '{print $1}') -c manager -n ${CONTROLLER_NAMESPACE} | grep -q "Deleted namespace drupal-example-install for project drupal-example, branch install")
343+
do
344+
if [ $CHECK_COUNTER -lt 20 ]; then
345+
let CHECK_COUNTER=CHECK_COUNTER+1
346+
echo "Environment not deleted yet"
347+
sleep 5
348+
else
349+
echo "Timeout of 100seconds for environment to be deleted"
350+
check_controller_log
351+
tear_down
352+
echo "================ END ================"
353+
echo "============== FAILED ==============="
354+
exit 1
355+
fi
356+
done
357+
kubectl logs $(kubectl get pods -n ${CONTROLLER_NAMESPACE} --no-headers | awk '{print $1}') -c manager -n ${CONTROLLER_NAMESPACE} | grep "handlers.LagoonTasks.Deletion"
358+
327359
check_controller_log
328360
tear_down
329361
echo "================ END ================"

‎handlers/delete_handler.go‎

Lines changed: 241 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,241 @@
1+
package handlers
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"time"
7+
8+
"github.com/go-logr/logr"
9+
"gopkg.in/matryer/try.v1"
10+
appsv1 "k8s.io/api/apps/v1"
11+
corev1 "k8s.io/api/core/v1"
12+
"k8s.io/apimachinery/pkg/types"
13+
"sigs.k8s.io/controller-runtime/pkg/client"
14+
)
15+
16+
// DeleteDeployments will delete any deployments from the namespace.
17+
func (h *Messaging) DeleteDeployments(ctx context.Context, opLog logr.Logger, ns, project, branch string) bool {
18+
deployments := &appsv1.DeploymentList{}
19+
listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{
20+
client.InNamespace(ns),
21+
})
22+
if err := h.Client.List(ctx, deployments, listOption); err != nil {
23+
opLog.Error(err,
24+
fmt.Sprintf(
25+
"Unable to list deployments in namespace %s for project %s, branch %s",
26+
ns,
27+
project,
28+
branch,
29+
),
30+
)
31+
//message.Ack(false) // ack to remove from queue
32+
return false
33+
}
34+
for _, dep := range deployments.Items {
35+
if err := h.Client.Delete(ctx, &dep); err != nil {
36+
opLog.Error(err,
37+
fmt.Sprintf(
38+
"Unable to delete deployment %s in %s for project %s, branch %s",
39+
dep.ObjectMeta.Name,
40+
ns,
41+
project,
42+
branch,
43+
),
44+
)
45+
//message.Ack(false) // ack to remove from queue
46+
return false
47+
}
48+
opLog.Info(
49+
fmt.Sprintf(
50+
"Deleted deployment %s in %s for project %s, branch %s",
51+
dep.ObjectMeta.Name,
52+
ns,
53+
project,
54+
branch,
55+
),
56+
)
57+
}
58+
return true
59+
}
60+
61+
// DeleteStatefulSets will delete any statefulsets from the namespace.
62+
func (h *Messaging) DeleteStatefulSets(ctx context.Context, opLog logr.Logger, ns, project, branch string) bool {
63+
statefulsets := &appsv1.StatefulSetList{}
64+
listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{
65+
client.InNamespace(ns),
66+
})
67+
if err := h.Client.List(ctx, statefulsets, listOption); err != nil {
68+
opLog.Error(err,
69+
fmt.Sprintf(
70+
"Unable to list statefulsets in namespace %s for project %s, branch %s",
71+
ns,
72+
project,
73+
branch,
74+
),
75+
)
76+
//message.Ack(false) // ack to remove from queue
77+
return false
78+
}
79+
for _, ss := range statefulsets.Items {
80+
if err := h.Client.Delete(ctx, &ss); err != nil {
81+
opLog.Error(err,
82+
fmt.Sprintf(
83+
"Unable to delete statefulset %s in %s for project %s, branch %s",
84+
ss.ObjectMeta.Name,
85+
ns,
86+
project,
87+
branch,
88+
),
89+
)
90+
//message.Ack(false) // ack to remove from queue
91+
return false
92+
}
93+
opLog.Info(
94+
fmt.Sprintf(
95+
"Deleted statefulset %s in %s for project %s, branch %s",
96+
ss.ObjectMeta.Name,
97+
ns,
98+
project,
99+
branch,
100+
),
101+
)
102+
}
103+
return true
104+
}
105+
106+
// DeleteDaemonSets will delete any daemonsets from the namespace.
107+
func (h *Messaging) DeleteDaemonSets(ctx context.Context, opLog logr.Logger, ns, project, branch string) bool {
108+
daemonsets := &appsv1.DaemonSetList{}
109+
listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{
110+
client.InNamespace(ns),
111+
})
112+
if err := h.Client.List(ctx, daemonsets, listOption); err != nil {
113+
opLog.Error(err,
114+
fmt.Sprintf(
115+
"Unable to list daemonsets in namespace %s for project %s, branch %s",
116+
ns,
117+
project,
118+
branch,
119+
),
120+
)
121+
//message.Ack(false) // ack to remove from queue
122+
return false
123+
}
124+
for _, ds := range daemonsets.Items {
125+
if err := h.Client.Delete(ctx, &ds); err != nil {
126+
opLog.Error(err,
127+
fmt.Sprintf(
128+
"Unable to delete daemonset %s in %s for project %s, branch %s",
129+
ds.ObjectMeta.Name,
130+
ns,
131+
project,
132+
branch,
133+
),
134+
)
135+
//message.Ack(false) // ack to remove from queue
136+
return false
137+
}
138+
opLog.Info(
139+
fmt.Sprintf(
140+
"Deleted daemonset %s in %s for project %s, branch %s",
141+
ds.ObjectMeta.Name,
142+
ns,
143+
project,
144+
branch,
145+
),
146+
)
147+
}
148+
return true
149+
}
150+
151+
// DeletePVCs will delete any PVCs from the namespace.
152+
func (h *Messaging) DeletePVCs(ctx context.Context, opLog logr.Logger, ns, project, branch string) bool {
153+
pvcs := &corev1.PersistentVolumeList{}
154+
listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{
155+
client.InNamespace(ns),
156+
})
157+
if err := h.Client.List(ctx, pvcs, listOption); err != nil {
158+
opLog.Error(err,
159+
fmt.Sprintf(
160+
"Unable to list pvcs in namespace %s for project %s, branch %s",
161+
ns,
162+
project,
163+
branch,
164+
),
165+
)
166+
//message.Ack(false) // ack to remove from queue
167+
return false
168+
}
169+
for _, pvc := range pvcs.Items {
170+
if err := h.Client.Delete(ctx, &pvc); err != nil {
171+
opLog.Error(err,
172+
fmt.Sprintf(
173+
"Unable to delete pvc %s in %s for project %s, branch %s",
174+
pvc.ObjectMeta.Name,
175+
ns,
176+
project,
177+
branch,
178+
),
179+
)
180+
//message.Ack(false) // ack to remove from queue
181+
return false
182+
}
183+
opLog.Info(
184+
fmt.Sprintf(
185+
"Deleted pvc %s in %s for project %s, branch %s",
186+
pvc.ObjectMeta.Name,
187+
ns,
188+
project,
189+
branch,
190+
),
191+
)
192+
}
193+
for _, pvc := range pvcs.Items {
194+
if err := h.CheckPVCExists(ctx, opLog, &pvc); err != nil {
195+
return false
196+
}
197+
}
198+
return true
199+
}
200+
201+
// DeleteNamespace will delete the namespace.
202+
func (h *Messaging) DeleteNamespace(ctx context.Context, opLog logr.Logger, namespace *corev1.Namespace, project, branch string) bool {
203+
if err := h.Client.Delete(ctx, namespace); err != nil {
204+
opLog.Error(err,
205+
fmt.Sprintf(
206+
"Unable to delete namespace %s for project %s, branch %s",
207+
namespace.ObjectMeta.Name,
208+
project,
209+
branch,
210+
),
211+
)
212+
//message.Ack(false) // ack to remove from queue
213+
return false
214+
}
215+
return true
216+
}
217+
218+
// CheckPVCExists .
219+
func (h *Messaging) CheckPVCExists(ctx context.Context, opLog logr.Logger, pvc *corev1.PersistentVolume) error {
220+
try.MaxRetries = 60
221+
err := try.Do(func(attempt int) (bool, error) {
222+
var ingressErr error
223+
err := h.Client.Get(ctx, types.NamespacedName{
224+
Namespace: pvc.ObjectMeta.Namespace,
225+
Name: pvc.ObjectMeta.Name,
226+
}, pvc)
227+
if err != nil {
228+
// the ingress doesn't exist anymore, so exit the retry
229+
ingressErr = nil
230+
opLog.Info(fmt.Sprintf("persistent volume claim %s in %s deleted", pvc.ObjectMeta.Name, pvc.ObjectMeta.Namespace))
231+
} else {
232+
// if the ingress still exists wait 5 seconds before trying again
233+
msg := fmt.Sprintf("persistent volume claim %s in %s still exists", pvc.ObjectMeta.Name, pvc.ObjectMeta.Namespace)
234+
ingressErr = fmt.Errorf("%s: %v", msg, err)
235+
opLog.Info(msg)
236+
}
237+
time.Sleep(1 * time.Second)
238+
return attempt < 60, ingressErr
239+
})
240+
return err
241+
}

0 commit comments

Comments
 (0)