Added support for 'grid_accounting' daemon (#142)

* Added support for 'grid_accounting' daemon

* Added missing migrid-accounting syslog volume

* Updated volume comments thanks to @rasmunk

* Added 'vgrid_home'. 'freeze_home' and 'user_settings' to migrid-accounting

* Added 'storage' to accounting description as suggested by @jonasbardine

* Removed debug leftover 'accounting_home' volume from 'migrid-quota' container

* Added accounting to 'Available target service names' thanks @jonasbardino

Author: Martin-Rehr

Dockerfile.rocky9

@@ -127,6 +127,7 @@ ARG ENABLE_TWOFACTOR_STRICT_ADDRESS=False
 ARG TWOFACTOR_AUTH_APPS=""
 ARG ENABLE_PEERS=True
 ARG ENABLE_QUOTA=False
+ARG ENABLE_ACCOUNTING=False
 ARG PEERS_MANDATORY=False
 ARG PEERS_EXPLICIT_FIELDS=""
 ARG PEERS_CONTACT_HINT="authorized to invite you as peer"
@@ -230,6 +231,7 @@ ARG QUOTA_LUSTRE_VERSION="2.15.4"
 ARG QUOTA_LUSTRE_BASE="/dev/null"
 ARG QUOTA_GOCRYPTFS_XRAY="/dev/null"
 ARG QUOTA_GOCRYPTFS_SOCK="/dev/null"
+ARG ACCOUNTING_UPDATE_INTERVAL=3600
 
 # Jupyter Arguments
 ARG JUPYTER_SERVICES=""
@@ -1030,6 +1032,7 @@ ARG ENABLE_TWOFACTOR_STRICT_ADDRESS
 ARG TWOFACTOR_AUTH_APPS
 ARG ENABLE_PEERS
 ARG ENABLE_QUOTA
+ARG ENABLE_ACCOUNTING
 ARG PEERS_MANDATORY
 ARG PEERS_EXPLICIT_FIELDS
 ARG PEERS_CONTACT_HINT
@@ -1111,6 +1114,7 @@ ARG QUOTA_UPDATE_INTERVAL
 ARG QUOTA_USER_LIMIT
 ARG QUOTA_VGRID_LIMIT
 ARG QUOTA_LUSTRE_VERSION
+ARG ACCOUNTING_UPDATE_INTERVAL
 
 # TODO: do we still need the ~/.local/ wrapper now that update-alternatives run?
 ENV PYTHONPATH=${MIG_ROOT}
@@ -1244,6 +1248,7 @@ RUN ./generateconfs.py --source=. \
     --enable_quota=${ENABLE_QUOTA} --quota_backend="${QUOTA_BACKEND}" \
     --quota_update_interval=${QUOTA_UPDATE_INTERVAL} \
     --quota_user_limit=${QUOTA_USER_LIMIT} --quota_vgrid_limit=${QUOTA_VGRID_LIMIT} \
+    --enable_accounting=${ENABLE_ACCOUNTING} --accounting_update_interval=${ACCOUNTING_UPDATE_INTERVAL} \
     --storage_protocols="${STORAGE_PROTOCOLS}" \
     --wwwserve_max_bytes=${WWWSERVE_MAX_BYTES} \
     --password_policy=${MIG_PASSWORD_POLICY} \

Makefile

@@ -112,6 +112,7 @@ initdirs: initcomposevars
 	mkdir -p ${PERSISTENT_ROOT}/events_home
 	mkdir -p ${PERSISTENT_ROOT}/sitestats_home
 	mkdir -p ${PERSISTENT_ROOT}/quota_home
+	mkdir -p ${PERSISTENT_ROOT}/accounting_home
 	mkdir -p ${PERSISTENT_ROOT}/sandbox_home
 	mkdir -p ${PERSISTENT_ROOT}/sss_home
 	mkdir -p ${PERSISTENT_ROOT}/workflows_db_home
@@ -143,6 +144,7 @@ initdirs: initcomposevars
 	mkdir -p ${LOG_ROOT}/syslog/migrid-webdavs
 	mkdir -p ${LOG_ROOT}/syslog/migrid-ftps
 	mkdir -p ${LOG_ROOT}/syslog/migrid-quota
+	mkdir -p ${LOG_ROOT}/syslog/migrid-accounting
 
 initcomposevars:
 	@echo "creating env variable map in docker-compose_shared.yml"
@@ -180,6 +182,10 @@ initservices:
 				&& "${ENABLE_QUOTA}" == "True" ]]; then
 			@ENABLED_SERVICES+=" $$service"
 		@fi
+		@if [[ "$$service" == "migrid-accounting" \
+				&& "${ENABLE_ACCOUNTING}" == "True" ]]; then
+			@ENABLED_SERVICES+=" $$service"
+		@fi
 	@done;
 	@echo $$ENABLED_SERVICES > ./.migrid_enabled_services
 

development.env

@@ -111,6 +111,7 @@ ENABLE_JOBS=True
 ENABLE_RESOURCES=True
 ENABLE_EVENTS=True
 ENABLE_QUOTA=False
+ENABLE_ACCOUNTING=False
 ENABLE_GRAVATARS=True
 ENABLE_SITESTATUS=True
 STATUS_SYSTEM_MATCH=ANY

development_gdp.env

@@ -111,6 +111,7 @@ ENABLE_JOBS=False
 ENABLE_RESOURCES=False
 ENABLE_EVENTS=False
 ENABLE_QUOTA=False
+ENABLE_ACCOUNTING=False
 ENABLE_GRAVATARS=False
 ENABLE_SITESTATUS=True
 STATUS_SYSTEM_MATCH=ANY

doc/source/sections/configuration/variables.rst

@@ -436,7 +436,10 @@ Variables
      - Enable the built-in janitor service to handle recurring tasks like clean up and cache update
    * - ENABLE_QUOTA
      - False
-     - Enable additional quota integration in the user pages if fundamentally enabled with the QUOTA_X variables.
+     - Enable additional storage quota daemon if fundamentally enabled with the QUOTA_X variables.
+   * - ENABLE_ACCOUNTING
+     - False
+     - Enable additional storage accounting daemon and integration in the user pages.
    * - ENABLE_GDP
      - False
      - Enable GDP mode for sensitive data with a lot of restrictions on access and logging
@@ -656,4 +659,7 @@ Variables
    * - QUOTA_GOCRYPTFS_SOCK
      - "/dev/null"
      - A gocryptfs socket is needed by gocryptfs-xray to resolve encoded data paths from MiG data paths.
+   * - ACCOUNTING_UPDATE_INTERVAL
+     - 3600
+     - The frequency (seconds) of storage accounting updates.
 

docker-compose_production.yml

@@ -33,6 +33,7 @@ services:
       # Available target service names follow migrid init script and are:
       #   httpd script monitor sshmux events cron transfers janitor
       #   openid sftp sftpsubsys webdavs ftps notify imnotify vmproxy quota
+      #   accounting
       RUN_SERVICES: 
     volumes:
       - type: volume
@@ -154,6 +155,9 @@ services:
       - type: volume
         source: quota_home
         target: /home/mig/state/quota_home
+      - type: volume
+        source: accounting_home
+        target: /home/mig/state/accounting_home
       - type: volume
         source: sandbox_home
         target: /home/mig/state/sandbox_home
@@ -236,6 +240,7 @@ services:
       # Available target service names follow migrid init script and are:
       #   httpd script monitor sshmux events cron transfers janitor
       #   openid sftp sftpsubsys webdavs ftps notify imnotify vmproxy quota
+      #   accounting
       RUN_SERVICES: openid rsyslogd
     depends_on:
       - migrid
@@ -306,6 +311,9 @@ services:
       #  source: quota_home
       #  target: /home/mig/state/quota_home
       #- type: volume
+      #  source: accounting_home
+      #  target: /home/mig/state/accounting_home
+      #- type: volume
       #  source: sandbox_home
       #  target: /home/mig/state/sandbox_home
       #- type: volume
@@ -381,6 +389,7 @@ services:
       # Available target service names follow migrid init script and are:
       #   httpd script monitor sshmux events cron transfers janitor
       #   openid sftp sftpsubsys webdavs ftps notify imnotify vmproxy quota
+      #   accounting
       RUN_SERVICES: sftp sftpsubsys rsyslogd
     depends_on:
       - migrid
@@ -452,6 +461,9 @@ services:
       #  source: quota_home
       #  target: /home/mig/state/quota_home
       #- type: volume
+      #  source: accounting_home
+      #  target: /home/mig/state/accounting_home
+      #- type: volume
       #  source: sandbox_home
       #  target: /home/mig/state/sandbox_home
       #- type: volume
@@ -527,6 +539,7 @@ services:
       # Available target service names follow migrid init script and are:
       #   httpd script monitor sshmux events cron transfers janitor
       #   openid sftp sftpsubsys webdavs ftps notify imnotify vmproxy quota
+      #   accounting
       RUN_SERVICES: ftps rsyslogd
     depends_on:
       - migrid
@@ -597,6 +610,9 @@ services:
       #  source: quota_home
       #  target: /home/mig/state/quota_home
       #- type: volume
+      #  source: accounting_home
+      #  target: /home/mig/state/accounting_home
+      #- type: volume
       #  source: sandbox_home
       #  target: /home/mig/state/sandbox_home
       #- type: volume
@@ -672,6 +688,7 @@ services:
       # Available target service names follow migrid init script and are:
       #   httpd script monitor sshmux events cron transfers janitor
       #   openid sftp sftpsubsys webdavs ftps notify imnotify vmproxy quota
+      #   accounting
       RUN_SERVICES: webdavs rsyslogd
     depends_on:
       - migrid
@@ -817,6 +834,7 @@ services:
       # Available target service names follow migrid init script and are:
       #   httpd script monitor sshmux events cron transfers janitor
       #   openid sftp sftpsubsys webdavs ftps notify imnotify vmproxy quota
+      #   accounting
       RUN_SERVICES: quota rsyslogd
     depends_on:
       - migrid
@@ -870,6 +888,58 @@ services:
         target: /home/mig/state/vgrid_files_writable
     command: /app/docker-entry.sh -k
 
+  migrid-accounting:
+    container_name: migrid-accounting
+    image: ${CONTAINER_REGISTRY}/ucphhpc/migrid${CONTAINER_TAG}
+    network_mode: host
+    environment:
+      TZ: ${TZ}
+      # RUN_SERVICES specifies which daemons to launch
+      # Available target service names follow migrid init script and are:
+      #   httpd script monitor sshmux events cron transfers janitor
+      #   openid sftp sftpsubsys webdavs ftps notify imnotify vmproxy quota
+      #   accounting
+      RUN_SERVICES: accounting rsyslogd
+    depends_on:
+      # IMPORTANT: finish volume init before migrid launch as explained above
+      migrid-volume-init:
+        condition: service_completed_successfully
+    volumes:
+      - type: volume
+        source: mig
+        target: /home/mig/mig
+      - type: volume
+        source: hotfixes
+        target: /hotfixes
+      - type: volume
+        source: mig-server-extconfs
+        target: /home/mig/mig/server/MiGserver.d
+      - type: volume
+        source: state
+        target: /home/mig/state
+      - type: volume
+        source: vgrid_home
+        target: /home/mig/state/vgrid_home
+      - type: volume
+        source: freeze_home
+        target: /home/mig/state/freeze_home
+      - type: volume
+        source: user_settings
+        target: /home/mig/state/user_settings
+      - type: volume
+        source: migrid-accounting-syslog
+        target: /var/log
+      - type: volume
+        source: log
+        target: /home/mig/state/log
+      - type: volume
+        source: quota_home
+        target: /home/mig/state/quota_home
+      - type: volume
+        source: accounting_home
+        target: /home/mig/state/accounting_home
+    command: /app/docker-entry.sh -k
+
 # NOTE: not used in stand-alone production mode
 #  nginx-proxy:
 #    image: ${CONTAINER_REGISTRY}/jwilder/nginx-proxy
@@ -1045,6 +1115,14 @@ volumes:
       device: ${PERSISTENT_ROOT}/quota_home
       o: bind
 
+  accounting_home:
+    # Volume used to contain the migrid accounting_home
+    driver: local
+    driver_opts:
+      type: none
+      device: ${PERSISTENT_ROOT}/accounting_home
+      o: bind
+
   sandbox_home:
     # Volume used to contain the migrid sandbox_home
     driver: local
@@ -1222,9 +1300,17 @@ volumes:
       o: bind
 
   migrid-quota-syslog:
-    # Volume used for exposing migrid lustre quota container system log
+    # Volume used for exposing migrid quota container system log
     driver: local
     driver_opts:
       type: none
       device: ${LOG_ROOT}/syslog/migrid-quota
       o: bind
+
+  migrid-accounting-syslog:
+    # Volume used for exposing migrid accounting container system log
+    driver: local
+    driver_opts:
+      type: none
+      device: ${LOG_ROOT}/syslog/migrid-accounting
+      o: bind

production.env

@@ -111,6 +111,7 @@ ENABLE_JOBS=True
 ENABLE_RESOURCES=True
 ENABLE_EVENTS=True
 ENABLE_QUOTA=False
+ENABLE_ACCOUNTING=False
 ENABLE_GRAVATARS=True
 ENABLE_SITESTATUS=True
 STATUS_SYSTEM_MATCH=ANY
@@ -274,3 +275,6 @@ QUOTA_LUSTRE_VERSION=2.15.4
 QUOTA_LUSTRE_BASE="/dev/null"
 QUOTA_GOCRYPTFS_XRAY="/dev/null"
 QUOTA_GOCRYPTFS_SOCK="/dev/null"
+
+# Accounting settings
+ACCOUNTING_UPDATE_INTERVAL=3600