cleanup

Author: bastelfreak

INSTALL.md

@@ -12,16 +12,16 @@ example IPA isn't managed and no real route53 keys are provided.
 We use Hetzner for cloud instances to test setups:
 
 ```
-hcloud server create --image=alma-8 --name=lsst.tim.betadots.training --type=cpx41 --ssh-key='bastelfreak betadots'
-hcloud server set-rdns lsst.tim.betadots.training --ip=95.217.179.41 --hostname=lsst.tim.betadots.training
-hcloud server set-rdns lsst.tim.betadots.training --ip=2a01:4f9:c012:acee::1 --hostname=lsst.tim.betadots.training
+hcloud server create --image=alma-8 --name=$host --type=cpx41 --ssh-key='bastelfreak'
+hcloud server set-rdns $host --ip=95.217.179.41 --hostname=$host
+hcloud server set-rdns $host --ip=2a01:4f9:c012:acee::1 --hostname=$host
 ```
 
 (Now also add matching A/AAAA records to make this easier)
 
 ```
-ssh-keygen -f ~/.ssh/known_hosts -R lsst.tim.betadots.training
-ssh-keyscan lsst.tim.betadots.training >> ~/.ssh/known_hosts
+ssh-keygen -f ~/.ssh/known_hosts -R $host
+ssh-keyscan $host >> ~/.ssh/known_hosts
 ```
 
 ## Patching
@@ -45,6 +45,14 @@ mkdir -p ~/.vim/{backupdir,undodir}
 wget https://gist.githubusercontent.com/bastelfreak/a3cfa50db2a7be92c47f246f8f22ca5c/raw/dab14889680d4a8bbcb83580185ca2e5040d5947/vla.vimrc -O ~/.vimrc
 ```
 
+### Helpful tools
+
+Those are helpful during testing
+
+```
+dnf -y install htop tig jq
+```
+
 ## install Puppet + Foreman
 
 ```
@@ -59,22 +67,22 @@ dnf -y install puppetdb puppetdb-termini postgresql-contrib
 Output from the installer should be like this:
 
 ```
-[root@lsst ~]# foreman-installer --enable-foreman-plugin-remote-execution --enable-foreman-cli-remote-execution --enable-foreman-proxy-plugin-remote-execution-script
-2024-02-11 13:36:41 [NOTICE] [root] Loading installer configuration. This will take some time.
-2024-02-11 13:36:43 [NOTICE] [root] Running installer with log based terminal output at level NOTICE.
-2024-02-11 13:36:43 [NOTICE] [root] Use -l to set the terminal output log level to ERROR, WARN, NOTICE, INFO, or DEBUG. See --full-help for definitions.
-2024-02-11 13:36:44 [NOTICE] [configure] Starting system configuration.
-2024-02-11 13:37:40 [NOTICE] [configure] 250 configuration steps out of 1323 steps complete.
-2024-02-11 13:37:48 [NOTICE] [configure] 500 configuration steps out of 1326 steps complete.
-2024-02-11 13:37:59 [NOTICE] [configure] 750 configuration steps out of 1351 steps complete.
-2024-02-11 13:38:00 [NOTICE] [configure] 1000 configuration steps out of 1351 steps complete.
-2024-02-11 13:38:17 [NOTICE] [configure] 1250 configuration steps out of 1351 steps complete.
-2024-02-11 13:39:33 [NOTICE] [configure] System configuration has finished.
+[root@lsst ~]# foreman-installer --enable-foreman-plugin-puppetdb
+2024-02-11 18:57:31 [NOTICE] [root] Loading installer configuration. This will take some time.
+2024-02-11 18:57:33 [NOTICE] [root] Running installer with log based terminal output at level NOTICE.
+2024-02-11 18:57:33 [NOTICE] [root] Use -l to set the terminal output log level to ERROR, WARN, NOTICE, INFO, or DEBUG. See --full-help for definitions.
+2024-02-11 18:57:35 [NOTICE] [configure] Starting system configuration.
+2024-02-11 18:58:30 [NOTICE] [configure] 250 configuration steps out of 1244 steps complete.
+2024-02-11 18:58:39 [NOTICE] [configure] 500 configuration steps out of 1247 steps complete.
+2024-02-11 18:58:45 [NOTICE] [configure] 750 configuration steps out of 1272 steps complete.
+2024-02-11 18:58:56 [NOTICE] [configure] 1000 configuration steps out of 1272 steps complete.
+2024-02-11 19:00:12 [NOTICE] [configure] 1250 configuration steps out of 1272 steps complete.
+2024-02-11 19:00:15 [NOTICE] [configure] System configuration has finished.
 Executing: foreman-rake upgrade:run
   Success!
-  * Foreman is running at https://lsst.tim.betadots.training
-      Initial credentials are admin / fbNn4VM4NjA2n2H4
-  * Foreman Proxy is running at https://lsst.tim.betadots.training:8443
+  * Foreman is running at https://foreman
+      Initial credentials are admin / s2hYUi7oEksKxaNM
+  * Foreman Proxy is running at https://foreman
 
 The full log is at /var/log/foreman-installer/foreman.log
 [root@lsst ~]#
@@ -170,7 +178,7 @@ Update Puppetserver to talk to PuppetDB
 ```
 puppet config set --section server storeconfigs true
 puppet config set --section main reports foreman,puppetdb
-echo -e "[main]\nserver_urls = https://$(hostname -f):8081/\nsoft_write_failure = false" > /etc/puppetlabs/puppet/puppetdb.conf
+echo -e "[main]\nserver_urls = https://$(hostname -f):8081/\nsoft_write_failure = true" > /etc/puppetlabs/puppet/puppetdb.conf
 systemctl restart puppetserver
 ```
 
@@ -179,26 +187,49 @@ systemctl restart puppetserver
 We need to ensure foreman knows the environment `bastelfreak` before we can
 assign it
 
-* login at https://lsst.tim.betadots.training/
-* got to https://lsst.tim.betadots.training/foreman_puppet/environments, import new environments
+* login at https://foreman/
+* got to https://foreman/foreman_puppet/environments, import new environments
 
 We need to set the environment in foreman
 
-* login at https://lsst.tim.betadots.training/
+* login at https://foreman/
 * select the node, click edit
-    * should bring you to https://lsst.tim.betadots.training/hosts/lsst.tim.betadots.training/edit
+    * should bring you to https://foreman/hosts/foreman/edit
 * At environment, select `bastelfreak`
 * save
 
 We need to set the role and site
 
-* login at https://lsst.tim.betadots.training/
-* At https://lsst.tim.betadots.training/hosts/lsst.tim.betadots.training/edit, go to `Parameters`
+* login at https://foreman/
+* At https://foreman/hosts/foreman/edit, go to `Parameters`
 * Select `Add Parameter`
 * Name=site, Value=test; save
 * Repeat: Name=role, Value=foreman; save
 
 
+At the moment a full puppet run doesn't succeed, but we can apply the following tags:
+
+```
+puppet agent -t --tags accounts,prometheus,chrony,yumrepo,auditd,tftp,convenience,debugutils,rsyslog,discovery,puppetserver,host,irqbalance,ssh,lldpd,sysstat,r10k,webhook,timezone,selinux,yum,docker,firewall,foreman_envsync,resolv_conf,sudo,postgresql_conf,udevd,reboot.target
+```
+
+Due to this we miss some migrations:
+
+```
+systemctl restart foreman
+foreman-rake db:migrate
+```
 
+Then we can reboot:
 
-puppet agent -t --tags accounts,prometheus,chrony,yumrepo,auditd,tftp,convenience,debugutils,rsyslog,discovery,puppetserver,host,irqbalance,ssh,lldpd,sysstat
+```
+sync; reboot
+```
+
+## Rebuilding the instance
+
+```
+hcloud server rebuild $host --image=alma-8
+ssh-keygen -f ~/.ssh/known_hosts -R $host
+ssh-keyscan $host >> ~/.ssh/known_hosts
+```

Puppetfile

@@ -112,7 +112,7 @@ mod 'stm/debconf', '5.0.0'
 mod 'syseleven/restic', '2.6.1'
 mod 'theforeman/dhcp', git: 'https://github.com/lsst-it/puppet-dhcp', ref: '4d48173'  # https://github.com/theforeman/puppet-dhcp/pull/226
 mod 'theforeman/dns', '10.1.0'
-mod 'theforeman/foreman', git: 'https://github.com/lsst-it/puppet-foreman', ref: '70b70bc'  # 20.2.0 + dep updates
+mod 'theforeman/foreman', git: 'https://github.com/theforeman/puppet-foreman', ref: '24.1.0'
 mod 'theforeman/foreman_proxy', git: 'https://github.com/lsst-it/puppet-foreman_proxy', ref: '39ef803'  # https://github.com/theforeman/puppet-foreman_proxy/pull/772 https://github.com/theforeman/puppet-foreman_proxy/pull/816
 mod 'theforeman/puppet', git: 'https://github.com/lsst-it/puppet-puppet', ref: '8ef01c3'  # https://github.com/theforeman/puppet-puppet/pull/891
 mod 'theforeman/puppetserver_foreman', '2.4.0'

hieradata/role/foreman.yaml

@@ -200,11 +200,15 @@ ssh::client_match_block:
       StrictHostKeyChecking: "no"  # el7 does not support "accept-new"
 
 profile::core::foreman::foreman_config:
+  # I'm not 100% sure why, but on foreman 3.8/ puppet-foreman 24.1.0 we cannot set booleans
+  # `/usr/sbin/foreman-rake -- config -k 'bmc_credentials_accessible' -v 'false'` is a workaround
+  # afterwards puppet detects that the value is correct and doesn't try to change it
   bmc_credentials_accessible: {value: false}  # disable bmc pass in enc yaml
   default_pxe_item_global: {value: "discovery"}
   destroy_vm_on_host_delete: {value: true}
-  discovery_fact_column: {value: "ipmi_ipaddress,ipmi_macaddress"}
-  discovery_hostname: {value: "ipmi_macaddress,discovery_bootif"}
+  # on older foreman/puppet-foreman it wasn't idempotent to set this, but it works withforeman 3.8/ puppet-foreman 24.1.0
+  discovery_fact_column: {value: '["ipmi_ipaddress","ipmi_macaddress"]'}
+  discovery_hostname: {value: '["ipmi_macaddress","discovery_bootif"]'}
   entries_per_page: {value: 100}
   # remove "docker*" from default excluded_facts
   # XXX using block scalar style results in the double quotes being preceeded
@@ -215,7 +219,9 @@ profile::core::foreman::foreman_config:
   #excluded_facts:
   #  value: '["lo", "en*v*", "usb*", "vnet*", "macvtap*", ";vdsmdummy;", "veth*", "tap*", "qbr*", "qvb*", "qvo*", "qr-*", "qg-*", "vlinuxbr*", "vovsbr*", "br-int", "vif*", "load_averages::*", "memory::swap::available*", "memory::swap::capacity", "memory::swap::used*", "memory::system::available*", "memory::system::capacity", "memory::system::used*", "memoryfree", "memoryfree_mb", "swapfree", "swapfree_mb", "uptime_hours", "uptime_days"]'
   host_details_ui: {value: false}  # https://projects.theforeman.org/issues/35115
-  host_power_status: {value: false}
+  # since isn't required/doesn't work anymore since https://github.com/theforeman/foreman/pull/9462/files
+  # the option host_power_status doesn't exist in foreman 3.8, I think due to #9462, but that's a bit of a guess
+  # host_power_status: {value: false}
   idle_timeout: {value: 7200}  # session timeout in minutes
   ignore_puppet_facts_for_provisioning: {value: true}
   matchers_inheritance: {value: false}

hieradata/site/test.yaml

@@ -1,6 +1,4 @@
 ---
-# some hacks to get puppet working outside of LSST
-ipa::domain_join_password: "foofoofoofoo"
 resolv_conf::nameservers:
   - "185.12.64.2"
   - "185.12.64.1"
@@ -38,5 +36,4 @@ puppetdb::globals::version: '7.16.0'
 profile::core::common::manage_sssd: false
 profile::core::common::manage_network_manager: false
 profile::core::common::manage_krb5: false
-# ipa class is still added to the catalog :thinking:
 profile::core::common::manage_ipa: false