Support NFS volumes with nfsbroker

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @cloudfoundry/wg-app-runtime-deployments-kind-deployment-approvers

docker-bake.hcl

@@ -3,7 +3,7 @@ variable "REGISTRY_PREFIX" {
 }
 
 group "all" {
-  targets = ["routing", "cf-networking", "capi", "diego", "loggregator", "loggregator-agent", "log-cache", "fileserver", "bosh-dns", "uaa", "cflinuxfs4", "misc"]
+  targets = ["routing", "cf-networking", "capi", "diego", "loggregator", "loggregator-agent", "log-cache", "fileserver", "bosh-dns", "uaa", "cflinuxfs4", "nfs-volume", "misc"]
 }
 
 group "default" {
@@ -228,6 +228,25 @@ target "cflinuxfs4" {
   }
 }
 
+variable "NFS_VOLUME_RELEASE_VERSION" {
+  # renovate: depName=cloudfoundry/nfs-volume-release
+  default = "7.47.0"
+}
+
+target "nfs-volume" {
+  dockerfile = "releases/nfs-volume-release/${component}.Dockerfile"
+  tags = [ "${REGISTRY_PREFIX}${component}:latest", "${REGISTRY_PREFIX}${component}:${NFS_VOLUME_RELEASE_VERSION}" ]
+  name = component
+
+  matrix = {
+    "component" = [ "nfsv3driver", "nfsbroker" ]
+  }
+
+  contexts = {
+    "src" = "https://github.com/cloudfoundry/nfs-volume-release.git#v${NFS_VOLUME_RELEASE_VERSION}:src"
+  }
+}
+
 target "misc" {
   dockerfile = "releases/capi/${component}.Dockerfile"
   tags = [  "${REGISTRY_PREFIX}${component}:latest" ]

releases/credhub/helm/files/credhub.yaml

@@ -29,6 +29,15 @@ security:
       operations:
       - read
       path: "/*"
+    - actors:
+      - uaa-client:nfs-broker-credhub-client
+      operations:
+      - read
+      - write
+      - delete
+      - read_acl
+      - write_acl
+      path: /nfsbroker/*
   oauth2:
     enabled: true
 server:

releases/diego/helm/templates/bbs.yaml

@@ -93,7 +93,6 @@ spec:
         - name: bbs
           image: {{ .Values.bbs.image.repository }}:{{ default .Chart.AppVersion .Values.bbs.image.tag }}
           imagePullPolicy: {{ .Values.bbs.image.imagePullPolicy }}
-          imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8889
           {{- if .Values.bbs.resources }}

releases/nfs-volume-release/helm/Chart.yaml

@@ -0,0 +1,6 @@
+name: nfs-volume-release
+apiVersion: v2
+version: 0.1.0
+description: A Helm chart for deploying NFS Volume Release components
+# renovate: depName=nfs-volume-release image=ghcr.io/cloudfoundry/k8s/nfsbroker
+appVersion: 7.47.0

releases/nfs-volume-release/helm/templates/nfsbroker.yaml

@@ -0,0 +1,146 @@
+{{- if .Values.nfsbroker.enabled }}
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: nfsbroker-configmap
+data:
+  services.json: |
+    [
+      {
+        "id": "997f8f26-e10c-11e7-80c1-9a214cf093ae",
+        "name": "nfs",
+        "description": "Existing NFSv3 and v4 volumes (see: https://code.cloudfoundry.org/nfs-volume-release/)",
+        "bindable": true,
+        "plan_updateable": false,
+        "tags": [
+          "nfs"
+        ],
+        "plans": [
+          {
+            "id": "09a09260-1df5-4445-9ed7-1ba56dadbbc8",
+            "name": "Existing",
+            "description": "A preexisting filesystem",
+            "metadata": {
+              "costs": [
+                {
+                  "amount": {
+                    "usd": 0.0
+                  },
+                  "unit": "MONTHLY"
+                }
+              ],
+              "displayName": "Existing Filesystems"
+            }
+          }
+        ],
+        "requires": [
+          "volume_mount"
+        ],
+        "metadata": {
+          "displayName": "NFS V3 / V4 Volume Broker",
+          "longDescription": "Broker for existing NFSv3 and v4 volumes",
+          "providerDisplayName": "Dell / Pivotal",
+          "documentationUrl": "https://docs.cloudfoundry.org/devguide/services/using-vol-services.html"
+        }
+      }
+    ]
+---
+kind: Secret
+apiVersion: v1
+type: Opaque
+metadata:
+  name: nfsbroker-credentials
+stringData:
+  USERNAME: nfsbroker
+  PASSWORD: nfsbroker
+  UAA_CLIENT_ID: "nfsbroker-credhub-client"
+  UAA_CLIENT_SECRET: {{ .Values.nfsbroker.oauthClientsSecret | quote }}
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: nfsbroker
+  labels:
+    app: nfsbroker
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: nfsbroker
+  template:
+    metadata:
+      labels:
+        app: nfsbroker
+    spec:
+      containers:
+        - name: nfsbroker
+          image: {{ .Values.nfsbroker.image.repository }}:{{ default .Chart.AppVersion .Values.nfsbroker.image.tag }}
+          imagePullPolicy: {{ .Values.nfsbroker.image.imagePullPolicy }}
+          ports:
+            - containerPort: 8080
+          envFrom:
+            - secretRef:
+                name: nfsbroker-credentials
+          args:
+            - --listenAddr=0.0.0.0:8080
+            - --servicesConfig=/services.json
+            - --credhubURL={{ .Values.nfsbroker.credhubURL }}
+            - --credhubCACertPath=/ssl/ca.crt
+            - --storeID=nfsbroker
+            - --logLevel=info
+            - --timeFormat=rfc3339
+            - --allowedOptions=source,uid,gid,auto_cache,readonly,version,mount,cache
+          volumeMounts:
+            - name: server-certs
+              mountPath: /ssl
+              readOnly: true
+            - name: services-config
+              mountPath: /services.json
+              subPath: services.json
+      nodeSelector:
+        cloudfoundry.org/workload: "false"
+      volumes:
+        - name: services-config
+          configMap:
+            name: nfsbroker-configmap
+        - name: server-certs
+          secret:
+            secretName: {{ default .Values.nfsbroker.certificateSecret "nfsbroker" }}
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: nfsbroker
+spec:
+  ports:
+    - port: 8080
+      targetPort: 8080
+      protocol: TCP
+  selector:
+    app: nfsbroker
+{{- if not .Values.nfsbroker.certificateSecret }}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: nfsbroker
+spec:
+  secretName: nfsbroker
+  commonName: client
+  dnsNames:
+    - nfsbroker
+    - nfsbroker.{{ .Release.Namespace }}.svc
+    - nfsbroker.{{ .Release.Namespace }}.svc.cluster.local
+  issuerRef:
+    name: ca-issuer
+    kind: ClusterIssuer
+  usages:
+    - key encipherment
+    - digital signature
+    - server auth
+    - client auth
+  privateKey:
+    rotationPolicy: Always
+{{- end }}
+{{- end }}

releases/nfs-volume-release/helm/templates/nfsv3driver.yaml

@@ -0,0 +1,56 @@
+{{- if .Values.nfsv3driver.enabled }}
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: nfsv3driver
+  labels:
+    app: nfsv3driver
+spec:
+  selector:
+    matchLabels:
+      app: nfsv3driver
+  template:
+    metadata:
+      labels:
+        app: nfsv3driver
+    spec:
+      containers:
+        - name: nfsv3driver
+          image: {{ .Values.nfsv3driver.image.repository }}:{{ default .Chart.AppVersion .Values.nfsv3driver.image.tag }}
+          imagePullPolicy: {{ .Values.nfsv3driver.image.imagePullPolicy }}
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+          ports:
+            - containerPort: 7589
+          args:
+            - --listenAddr=$(POD_IP):7589
+            - --transport=tcp-json
+            - --debugAddr=127.0.0.1:7689
+            - --adminAddr=127.0.0.1:7590
+            - --driversPath=/var/lib/rep/voldrivers
+            - --mountDir=/var/lib/rep/volumes/nfs
+            - --logLevel=info
+            - --timeFormat=rfc3339
+            - --mapfsPath=/usr/local/bin/mapfs
+          volumeMounts:
+            - name: voldrivers
+              mountPath: /var/lib/rep/voldrivers
+              mountPropagation: Bidirectional
+            - name: nfs-mounts
+              mountPath: /var/lib/rep/volumes/nfs
+              mountPropagation: Bidirectional
+          securityContext:
+            privileged: true
+      nodeSelector:
+        cloudfoundry.org/workload: "true"
+      volumes:
+        - name: voldrivers
+          hostPath:
+            path: /var/lib/rep/voldrivers
+        - name: nfs-mounts
+          hostPath:
+            path: /var/lib/rep/volumes/nfs
+{{- end }}

releases/nfs-volume-release/helm/values.yaml

@@ -0,0 +1,17 @@
+nfsbroker:
+  enabled: true
+  certificateSecret: ~
+  oauthClientsSecret: ~
+  credhubURL: https://credhub.default.svc.cluster.local:8844
+  image:
+    repository: ghcr.io/cloudfoundry/k8s/nfsbroker
+    tag: ~
+    imagePullPolicy: IfNotPresent
+
+
+nfsv3driver:
+  enabled: true
+  image:
+    repository: ghcr.io/cloudfoundry/k8s/nfsv3driver
+    tag: ~
+    imagePullPolicy: IfNotPresent

releases/nfs-volume-release/nfsbroker.Dockerfile

@@ -0,0 +1,14 @@
+FROM --platform=$BUILDPLATFORM golang:1-alpine AS builder
+
+ARG TARGETOS TARGETARCH
+
+COPY --from=src . /nfs-volume-release/src
+WORKDIR /nfs-volume-release/src/code.cloudfoundry.org/nfsbroker
+
+RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /usr/local/bin/nfsbroker code.cloudfoundry.org/nfsbroker
+
+FROM alpine:latest
+
+COPY --from=builder /usr/local/bin/nfsbroker /usr/local/bin
+
+ENTRYPOINT [ "/usr/local/bin/nfsbroker" ]

releases/nfs-volume-release/nfsv3driver.Dockerfile

@@ -0,0 +1,21 @@
+FROM --platform=$BUILDPLATFORM golang:1-alpine AS builder
+
+ARG TARGETOS TARGETARCH
+
+COPY --from=src . /nfs-volume-release/src
+
+WORKDIR /nfs-volume-release/src/code.cloudfoundry.org/nfsv3driver
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /usr/local/bin/nfsv3driver code.cloudfoundry.org/nfsv3driver/cmd/nfsv3driver
+
+WORKDIR /nfs-volume-release/src/code.cloudfoundry.org/mapfs
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /usr/local/bin/mapfs code.cloudfoundry.org/mapfs
+
+FROM ubuntu:latest
+
+COPY --from=builder /usr/local/bin/mapfs /usr/local/bin
+COPY --from=builder /usr/local/bin/nfsv3driver /usr/local/bin
+ADD --chmod=0755 releases/nfs-volume-release/nfsv3driver.sh /nfsv3driver.sh
+
+RUN apt-get update && apt-get install -y nfs-common fuse
+
+ENTRYPOINT [ "/nfsv3driver.sh" ]

releases/nfs-volume-release/nfsv3driver.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -e
+
+service rpcbind start
+
+exec /usr/local/bin/nfsv3driver "$@"

releases/uaa/helm/files/uaa.yml

@@ -245,6 +245,10 @@ oauth:
       authorities: clients.read,clients.write,clients.secret,uaa.admin,scim.read,scim.write,password.write
       id: admin
       secret: {{ .Values.oauthClientsSecret | quote }}
+    nfs-broker-credhub-client:
+      authorities: credhub.read,credhub.write
+      authorized-grant-types: client_credentials
+      secret: {{ .Values.oauthClientsSecret | quote }}
 authentication:
   enableUriEncodingCompatibilityMode: false
   policy:

scripts/deploy.sh

@@ -52,7 +52,6 @@ helm upgrade --install uaa releases/uaa/helm --set ccAdminPassword=$CC_ADMIN_PAS
 
 helm upgrade --install credhub releases/credhub/helm --set dbPassword=$DB_PASSWORD
 helm upgrade --install locket releases/diego/helm --set dbPassword=$DB_PASSWORD --set oauthClientsSecret=$OAUTH_CLIENTS_SECRET --set "locket.enabled=true" --wait
-
 helm upgrade --install diego releases/diego/helm --set dbPassword=$DB_PASSWORD --set diegoSSHCredentials=$DIEGO_SSH_CREDENTIALS --set oauthClientsSecret=$OAUTH_CLIENTS_SECRET --set-file sshProxyHostKey="$CERTS_DIR/ssh_key" --set "auctioneer.enabled=true" --set "bbs.enabled=true" --set "fileserver.enabled=true" --set "sshProxy.enabled=true"
 helm upgrade --install tps-watcher releases/capi/helm --set "tpsWatcher.enabled=true"
 helm upgrade --install route-emitter releases/diego/helm --set "routeEmitter.enabled=true"