Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 6 additions & 19 deletions .github/workflows/aergia-controller.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,21 +26,6 @@ jobs:
uses: actions/setup-go@v5
with:
go-version: '1.23'
- name: Install kustomize, kubebuilder, helm
run: |
#kubebuilder
curl -sL https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.3.2/kubebuilder_2.3.2_linux_amd64.tar.gz | tar -xz -C /tmp/
sudo mkdir -p /usr/local/kubebuilder/bin
sudo mv /tmp/kubebuilder_2.3.2_linux_amd64/bin/* /usr/local/kubebuilder/bin
chmod +x /usr/local/kubebuilder/bin/*
echo "/usr/local/kubebuilder/bin" >> $GITHUB_PATH
- name: Check go, kustomize, kubebuilder, helm, docker-compose, kind versions
run: |
go version
kustomize version
helm version
kubebuilder version
kind version

- name: Add dependency chart repos
run: |
Expand All @@ -57,8 +42,9 @@ jobs:
- name: Configure node IP in kind-config.yaml
run: |
docker network create kind
LAGOON_KIND_CIDR_BLOCK=$(docker network inspect kind | jq '. [0].IPAM.Config[0].Subnet' | tr -d '"')
export KIND_NODE_IP=$(echo ${LAGOON_KIND_CIDR_BLOCK%???} | awk -F'.' '{print $1,$2,$3,240}' OFS='.')
LAGOON_KIND_CIDR_BLOCK=$(docker network inspect kind | jq '. [0].IPAM.Config[0].Subnet' | tr -d '"')
KIND_NODE_IP=$(echo "${LAGOON_KIND_CIDR_BLOCK%???}" | awk -F'.' '{print $1,$2,$3,240}' OFS='.')
export KIND_NODE_IP
envsubst < test-resources/test-suite.kind-config.yaml.tpl > test-resources/test-suite.kind-config.yaml
envsubst < test/e2e/testdata/example-nginx.yaml.tpl > test/e2e/testdata/example-nginx.yaml

Expand All @@ -83,6 +69,7 @@ jobs:

- name: Run github/test-e2e
run: |
LAGOON_KIND_CIDR_BLOCK=$(docker network inspect kind | jq '. [0].IPAM.Config[0].Subnet' | tr -d '"')
export KIND_NODE_IP=$(echo ${LAGOON_KIND_CIDR_BLOCK%???} | awk -F'.' '{print $1,$2,$3,240}' OFS='.')
LAGOON_KIND_CIDR_BLOCK=$(docker network inspect kind | jq '. [0].IPAM.Config[0].Subnet' | tr -d '"')
KIND_NODE_IP=$(echo "${LAGOON_KIND_CIDR_BLOCK%???}" | awk -F'.' '{print $1,$2,$3,240}' OFS='.')
export KIND_NODE_IP
make github/test-e2e KIND_NETWORK=kind
33 changes: 23 additions & 10 deletions cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -145,15 +145,8 @@ func main() {
}))

// read the selector file into idlerdata struct.
file, err := os.Open(selectorsFile)
selectors, err := readSelectors(selectorsFile)
if err != nil {
setupLog.Error(err, "unable to open selectors file")
os.Exit(1)
}
defer file.Close()
d := yaml.NewDecoder(file)
selectors := &idler.Data{}
if err := d.Decode(&selectors); err != nil {
setupLog.Error(err, "unable to decode selectors yaml")
os.Exit(1)
}
Expand Down Expand Up @@ -245,16 +238,22 @@ func main() {
// CLI Idler
if enableCLIIdler {
setupLog.Info("starting cli idler")
c.AddFunc(cliCron, func() {
_, err := c.AddFunc(cliCron, func() {
idler.CLIIdler()
})
if err != nil {
setupLog.Error(err, "unable to create cli idler cronjob", "controller", "Idling")
}
}
// Service Idler
if enableServiceIdler {
setupLog.Info("starting service idler")
c.AddFunc(serviceCron, func() {
_, err := c.AddFunc(serviceCron, func() {
idler.ServiceIdler()
})
if err != nil {
setupLog.Error(err, "unable to create service idler cronjob", "controller", "Idling")
}
}
// start crons.
c.Start()
Expand All @@ -280,3 +279,17 @@ func main() {
os.Exit(1)
}
}

func readSelectors(selectorsFile string) (*idler.Data, error) {
file, err := os.Open(selectorsFile)
if err != nil {
return nil, err
}
defer file.Close()
d := yaml.NewDecoder(file)
selectors := &idler.Data{}
if err := d.Decode(&selectors); err != nil {
return nil, err
}
return selectors, nil
}
10 changes: 5 additions & 5 deletions internal/controllers/idling_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,9 @@ func (r *IdlingReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
return ctrl.Result{}, ignoreNotFound(err)
}

if val, ok := namespace.ObjectMeta.Labels["idling.amazee.io/force-scaled"]; ok && val == "true" {
if val, ok := namespace.Labels["idling.amazee.io/force-scaled"]; ok && val == "true" {
opLog.Info(fmt.Sprintf("Force scaling environment %s", namespace.Name))
r.Idler.KubernetesServiceIdler(ctx, opLog, namespace, namespace.ObjectMeta.Labels[r.Idler.Selectors.NamespaceSelectorsLabels.ProjectName], false, true)
r.Idler.KubernetesServiceIdler(ctx, opLog, namespace, namespace.Labels[r.Idler.Selectors.NamespaceSelectorsLabels.ProjectName], false, true)
nsMergePatch, _ := json.Marshal(map[string]interface{}{
"metadata": map[string]interface{}{
"labels": map[string]*string{
Expand All @@ -66,9 +66,9 @@ func (r *IdlingReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
return ctrl.Result{}, nil
}

if val, ok := namespace.ObjectMeta.Labels["idling.amazee.io/force-idled"]; ok && val == "true" {
if val, ok := namespace.Labels["idling.amazee.io/force-idled"]; ok && val == "true" {
opLog.Info(fmt.Sprintf("Force idling environment %s", namespace.Name))
r.Idler.KubernetesServiceIdler(ctx, opLog, namespace, namespace.ObjectMeta.Labels[r.Idler.Selectors.NamespaceSelectorsLabels.ProjectName], true, false)
r.Idler.KubernetesServiceIdler(ctx, opLog, namespace, namespace.Labels[r.Idler.Selectors.NamespaceSelectorsLabels.ProjectName], true, false)
nsMergePatch, _ := json.Marshal(map[string]interface{}{
"metadata": map[string]interface{}{
"labels": map[string]*string{
Expand All @@ -83,7 +83,7 @@ func (r *IdlingReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
return ctrl.Result{}, nil
}

if val, ok := namespace.ObjectMeta.Labels["idling.amazee.io/unidle"]; ok && val == "true" {
if val, ok := namespace.Labels["idling.amazee.io/unidle"]; ok && val == "true" {
opLog.Info(fmt.Sprintf("Unidling environment %s", namespace.Name))
r.Unidler.Unidle(ctx, &namespace, opLog)
nsMergePatch, _ := json.Marshal(map[string]interface{}{
Expand Down
30 changes: 15 additions & 15 deletions internal/handlers/idler/cli-kubernetes.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (
func (h *Idler) kubernetesCLI(ctx context.Context, opLog logr.Logger, namespace corev1.Namespace) {
labelRequirements := generateLabelRequirements(h.Selectors.CLI.Builds)
listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{
client.InNamespace(namespace.ObjectMeta.Name),
client.InNamespace(namespace.Name),
client.MatchingLabelsSelector{
Selector: labels.NewSelector().Add(labelRequirements...),
},
Expand All @@ -31,7 +31,7 @@ func (h *Idler) kubernetesCLI(ctx context.Context, opLog logr.Logger, namespace
runningBuild := false
if !h.Selectors.CLI.SkipBuildCheck {
if err := h.Client.List(ctx, builds, listOption); err != nil {
opLog.Error(err, fmt.Sprintf("Error getting running builds for namespace %s", namespace.ObjectMeta.Name))
opLog.Error(err, fmt.Sprintf("Error getting running builds for namespace %s", namespace.Name))
} else {
for _, build := range builds.Items {
if build.Status.Phase == "Running" {
Expand All @@ -47,7 +47,7 @@ func (h *Idler) kubernetesCLI(ctx context.Context, opLog logr.Logger, namespace
// @TODO: eventually replace the `lagoon.sh/service=cli` check with `lagoon.sh/service-type=cli|cli-persistent` for better coverage
labelRequirements := generateLabelRequirements(h.Selectors.CLI.Deployments)
listOption = (&client.ListOptions{}).ApplyOptions([]client.ListOption{
client.InNamespace(namespace.ObjectMeta.Name),
client.InNamespace(namespace.Name),
client.MatchingLabelsSelector{
Selector: labels.NewSelector().Add(labelRequirements...),
},
Expand All @@ -61,13 +61,13 @@ func (h *Idler) kubernetesCLI(ctx context.Context, opLog logr.Logger, namespace
zeroReps := new(int32)
*zeroReps = 0
if deployment.Spec.Replicas != zeroReps {
opLog.Info(fmt.Sprintf("Deployment %s has %d running replicas", deployment.ObjectMeta.Name, *deployment.Spec.Replicas))
opLog.Info(fmt.Sprintf("Deployment %s has %d running replicas", deployment.Name, *deployment.Spec.Replicas))
} else {
opLog.Info(fmt.Sprintf("Deployment %s is already idled", deployment.ObjectMeta.Name))
opLog.Info(fmt.Sprintf("Deployment %s is already idled", deployment.Name))
break
}
if h.Debug {
opLog.Info(fmt.Sprintf("Checking deployment %s for cronjobs", deployment.ObjectMeta.Name))
opLog.Info(fmt.Sprintf("Checking deployment %s for cronjobs", deployment.Name))
}

hasCrons := false
Expand All @@ -77,7 +77,7 @@ func (h *Idler) kubernetesCLI(ctx context.Context, opLog logr.Logger, namespace
if env.Name == "CRONJOBS" {
if len(env.Value) > 0 {
cronjobs := strings.Split(env.Value, `\n`)
opLog.Info(fmt.Sprintf("Deployment %s has %d cronjobs defined", deployment.ObjectMeta.Name, len(cronjobs)))
opLog.Info(fmt.Sprintf("Deployment %s has %d cronjobs defined", deployment.Name, len(cronjobs)))
hasCrons = true
break
}
Expand All @@ -89,7 +89,7 @@ func (h *Idler) kubernetesCLI(ctx context.Context, opLog logr.Logger, namespace
pods := &corev1.PodList{}
labelRequirements := generateLabelRequirements(h.Selectors.CLI.Pods)
listOption = (&client.ListOptions{}).ApplyOptions([]client.ListOption{
client.InNamespace(namespace.ObjectMeta.Name),
client.InNamespace(namespace.Name),
client.MatchingLabelsSelector{
Selector: labels.NewSelector().Add(labelRequirements...),
},
Expand All @@ -101,16 +101,16 @@ func (h *Idler) kubernetesCLI(ctx context.Context, opLog logr.Logger, namespace
processCount := 0
if !h.Selectors.CLI.SkipProcessCheck {
if h.Debug {
opLog.Info(fmt.Sprintf("Checking pod %s for running processes", pod.ObjectMeta.Name))
opLog.Info(fmt.Sprintf("Checking pod %s for running processes", pod.Name))
}
/*
Anything running with parent PID0 is likely a user process
/bin/bash -c "pgrep -P 0 | tail -n +3 | wc -l | tr -d ' '"
*/
var stdin io.Reader
stdout, _, err := execPod(pod.ObjectMeta.Name, namespace.ObjectMeta.Name, []string{`/bin/sh`, `-c`, `pgrep -P 0|tail -n +3|wc -l|tr -d ' '`}, stdin, false)
stdout, _, err := execPod(pod.Name, namespace.Name, []string{`/bin/sh`, `-c`, `pgrep -P 0|tail -n +3|wc -l|tr -d ' '`}, stdin, false)
if err != nil {
opLog.Error(err, fmt.Sprintf("Error when trying to exec to pod %s", pod.ObjectMeta.Name))
opLog.Error(err, fmt.Sprintf("Error when trying to exec to pod %s", pod.Name))
break
}
trimmed := strings.TrimSpace(string(stdout))
Expand All @@ -121,7 +121,7 @@ func (h *Idler) kubernetesCLI(ctx context.Context, opLog logr.Logger, namespace
}
}
if processCount == 0 {
opLog.Info(fmt.Sprintf("Pod %s has no running processes, idling", pod.ObjectMeta.Name))
opLog.Info(fmt.Sprintf("Pod %s has no running processes, idling", pod.Name))
}
}
if processCount == 0 {
Expand All @@ -133,13 +133,13 @@ func (h *Idler) kubernetesCLI(ctx context.Context, opLog logr.Logger, namespace
},
})
if err := h.Client.Patch(ctx, scaleDeployment, client.RawPatch(types.MergePatchType, mergePatch)); err != nil {
opLog.Error(err, fmt.Sprintf("Error scaling deployment %s", deployment.ObjectMeta.Name))
opLog.Error(err, fmt.Sprintf("Error scaling deployment %s", deployment.Name))
} else {
opLog.Info(fmt.Sprintf("Deployment %s scaled to 0", deployment.ObjectMeta.Name))
opLog.Info(fmt.Sprintf("Deployment %s scaled to 0", deployment.Name))
}
metrics.CliIdleEvents.Inc()
} else {
opLog.Info(fmt.Sprintf("Deployment %s would be scaled to 0", deployment.ObjectMeta.Name))
opLog.Info(fmt.Sprintf("Deployment %s would be scaled to 0", deployment.Name))
}
}
}
Expand Down
27 changes: 12 additions & 15 deletions internal/handlers/idler/cli.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,10 @@ import (
func (h *Idler) CLIIdler() {
ctx := context.Background()
opLog := h.Log.WithName("aergia-controller").WithName("CLIIdler")
listOption := &client.ListOptions{}
// in kubernetes, we can reliably check for the existence of this label so that
// we only check namespaces that have been deployed by a lagoon at one point
labelRequirements := generateLabelRequirements(h.Selectors.CLI.Namespace)
listOption = (&client.ListOptions{}).ApplyOptions([]client.ListOption{
listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{
client.MatchingLabelsSelector{
Selector: labels.NewSelector().Add(labelRequirements...),
},
Expand All @@ -32,28 +31,26 @@ func (h *Idler) CLIIdler() {
return
}
for _, namespace := range namespaces.Items {
projectAutoIdle, ok1 := namespace.ObjectMeta.Labels[h.Selectors.NamespaceSelectorsLabels.ProjectIdling]
environmentAutoIdle, ok2 := namespace.ObjectMeta.Labels[h.Selectors.NamespaceSelectorsLabels.EnvironmentIdling]
projectAutoIdle, ok1 := namespace.Labels[h.Selectors.NamespaceSelectorsLabels.ProjectIdling]
environmentAutoIdle, ok2 := namespace.Labels[h.Selectors.NamespaceSelectorsLabels.EnvironmentIdling]
if ok1 && ok2 {
if environmentAutoIdle == "1" && projectAutoIdle == "1" {
envOpLog := opLog.WithValues("namespace", namespace.ObjectMeta.Name).
WithValues("project", namespace.ObjectMeta.Labels[h.Selectors.NamespaceSelectorsLabels.ProjectName]).
WithValues("environment", namespace.ObjectMeta.Labels[h.Selectors.NamespaceSelectorsLabels.EnvironmentName]).
envOpLog := opLog.WithValues("namespace", namespace.Name).
WithValues("project", namespace.Labels[h.Selectors.NamespaceSelectorsLabels.ProjectName]).
WithValues("environment", namespace.Labels[h.Selectors.NamespaceSelectorsLabels.EnvironmentName]).
WithValues("dry-run", h.DryRun)
envOpLog.Info("Checking namespace")
h.kubernetesCLI(ctx, envOpLog, namespace)
} else {
if h.Debug {
opLog.Info(fmt.Sprintf("skipping namespace %s; autoidle values are env:%s proj:%s",
namespace.ObjectMeta.Name,
environmentAutoIdle,
projectAutoIdle))
}
} else if h.Debug {
opLog.Info(fmt.Sprintf("skipping namespace %s; autoidle values are env:%s proj:%s",
namespace.Name,
environmentAutoIdle,
projectAutoIdle))
}
} else {
if h.Debug {
opLog.Info(fmt.Sprintf("skipping namespace %s; not in lagoon",
namespace.ObjectMeta.Name))
namespace.Name))
}
}
}
Expand Down
Loading