Deploying Installer Provisioned Infrastructure (IPI) of OpenShift on Bare Metal - 4.8

apiVersion: v1
basedomain: <domain>
metadata:
  name: <cluster-name>
networking:
  machineCIDR: <public-cidr>
  networkType: OVNKubernetes
compute:
- name: worker
  replicas: 2 (1)
controlPlane:
  name: master
  replicas: 3
  platform:
    baremetal: {}
platform:
  baremetal:
    apiVIP: <api-ip>
    ingressVIP: <wildcard-ip>
    provisioningNetworkInterface: <NIC1>
    provisioningNetworkCIDR: <CIDR>
    hosts:
      - name: openshift-master-0
        role: master
        bmc:
          address: ipmi://<out-of-band-ip> (2)
          username: <user>
          password: <password>
        bootMACAddress: <NIC1-mac-address>
        hardwareProfile: default
      - name: openshift-master-1
        role: master
        bmc:
          address: ipmi://<out-of-band-ip>
          username: <user>
          password: <password>
        bootMACAddress: <NIC1-mac-address>
        hardwareProfile: default
      - name: openshift-master-2
        role: master
        bmc:
          address: ipmi://<out-of-band-ip>
          username: <user>
          password: <password>
        bootMACAddress: <NIC1-mac-address>
        hardwareProfile: default
      - name: openshift-worker-0
        role: worker
        bmc:
          address: ipmi://<out-of-band-ip>
          username: <user>
          password: <password>
        bootMACAddress: <NIC1-mac-address>
        hardwareProfile: unknown
      - name: openshift-worker-1
        role: worker
        bmc:
          address: ipmi://<out-of-band-ip>
          username: <user>
          password: <password>
        bootMACAddress: <NIC1-mac-address>
        hardwareProfile: unknown
pullSecret: '<pull_secret>'
sshKey: '<ssh_pub_key>'

[kni@provisioner ~]$ mkdir ~/clusterconfigs
[kni@provisioner ~]$ cp install-config.yaml ~/clusterconfigs

[kni@provisioner ~]$ ipmitool -I lanplus -U <user> -P <password> -H <management-server-ip> power off

for i in $(sudo virsh list | tail -n +3 | grep bootstrap | awk {'print $2'});
do
  sudo virsh destroy $i;
  sudo virsh undefine $i;
  sudo virsh vol-delete $i --pool $i;
  sudo virsh vol-delete $i.ign --pool $i;
  sudo virsh pool-destroy $i;
  sudo virsh pool-undefine $i;
done

[kni@provisioner ~]$ ./openshift-baremetal-install --dir ~/clusterconfigs create manifests

INFO Consuming Install Config from target directory
WARNING Making control-plane schedulable by setting MastersSchedulable to true for Scheduler cluster settings
WARNING Discarding the Openshift Manifest that was provided in the target directory because its dependencies are dirty and it needs to be regenerated

# Use public servers from the pool.ntp.org project.
# Please consider joining the pool (https://www.pool.ntp.org/join.html).

# This file is managed by the machine config operator
server openshift-master-0.<cluster-name>.<domain> iburst (1)
server openshift-master-1.<cluster-name>.<domain> iburst
server openshift-master-2.<cluster-name>.<domain> iburst

stratumweight 0
driftfile /var/lib/chrony/drift
rtcsync
makestep 10 3
bindcmdaddress 127.0.0.1
bindcmdaddress ::1
keyfile /etc/chrony.keys
commandkey 1
generatecommandkey
noclientlog
logchange 0.5
logdir /var/log/chrony

# Configure the control plane nodes to serve as local NTP servers
# for all worker nodes, even if they are not in sync with an
# upstream NTP server.

# Allow NTP client access from the local network.
allow all
# Serve time even if not synchronized to a time source.
local stratum 3 orphan

# This file is managed by the machine config operator
server openshift-master-0.<cluster-name>.<domain> iburst (1)
server openshift-master-1.<cluster-name>.<domain> iburst
server openshift-master-2.<cluster-name>.<domain> iburst

stratumweight 0
driftfile /var/lib/chrony/drift
rtcsync
makestep 10 3
bindcmdaddress 127.0.0.1
bindcmdaddress ::1
keyfile /etc/chrony.keys
commandkey 1
generatecommandkey
noclientlog
logchange 0.5
logdir /var/log/chrony

# This example MachineConfig replaces ~/control-plane-chrony.conf
apiVersion: machineconfiguration.openshift.io/v1
kind: MachineConfig
metadata:
  labels:
    machineconfiguration.openshift.io/role: master
  name: 99-master-etc-chrony-conf-override-to-server
spec:
  config:
    ignition:
      version: 2.2.0
    storage:
      files:
        - contents:
            source: data:text/plain;charset=utf-8;base64,BASE64ENCODEDCONFIGFILE(1)
          filesystem: root
          mode: 0644
          path: /etc/control-plane-chrony.conf

$ base64 ~/control-plane-chrony.conf

IyBVc2UgcHVibGljIHNlcnZlcnMgZnJvbSB0aGUgcG9vbC5udHAub3JnIHByb2plY3QuCiMgUGxl
YXNlIGNvbnNpZGVyIGpvaW5pbmcgdGhlIHBvb2wgKGh0dHBzOi8vd3d3LnBvb2wubnRwLm9yZy9q
b2luLmh0bWwpLgoKIyBUaGlzIGZpbGUgaXMgbWFuYWdlZCBieSB0aGUgbWFjaGluZSBjb25maWcg
b3BlcmF0b3IKc2VydmVyIG9wZW5zaGlmdC1tYXN0ZXItMC48Y2x1c3Rlci1uYW1lPi48ZG9tYWlu
PiBpYnVyc3QKc2VydmVyIG9wZW5zaGlmdC1tYXN0ZXItMS48Y2x1c3Rlci1uYW1lPi48ZG9tYWlu
PiBpYnVyc3QKc2VydmVyIG9wZW5zaGlmdC1tYXN0ZXItMi48Y2x1c3Rlci1uYW1lPi48ZG9tYWlu
PiBpYnVyc3QKCnN0cmF0dW13ZWlnaHQgMApkcmlmdGZpbGUgL3Zhci9saWIvY2hyb255L2RyaWZ0
CnJ0Y3N5bmMKbWFrZXN0ZXAgMTAgMwpiaW5kY21kYWRkcmVzcyAxMjcuMC4wLjEKYmluZGNtZGFk
ZHJlc3MgOjoxCmtleWZpbGUgL2V0Yy9jaHJvbnkua2V5cwpjb21tYW5ka2V5IDEKZ2VuZXJhdGVj
b21tYW5ka2V5Cm5vY2xpZW50bG9nCmxvZ2NoYW5nZSAwLjUKbG9nZGlyIC92YXIvbG9nL2Nocm9u
eQoKIyBDb25maWd1cmUgdGhlIGNvbnRyb2wgcGxhbmUgbm9kZXMgdG8gc2VydmUgYXMgbG9jYWwg
TlRQIHNlcnZlcnMKIyBmb3IgYWxsIHdvcmtlciBub2RlcywgZXZlbiBpZiB0aGV5IGFyZSBub3Qg
aW4gc3luYyB3aXRoIGFuCiMgdXBzdHJlYW0gTlRQIHNlcnZlci4KCiMgQWxsb3cgTlRQIGNsaWVu
dCBhY2Nlc3MgZnJvbSB0aGUgbG9jYWwgbmV0d29yay4KYWxsb3cgYWxsCiMgU2VydmUgdGltZSBl
dmVuIGlmIG5vdCBzeW5jaHJvbml6ZWQgdG8gYSB0aW1lIHNvdXJjZS4KbG9jYWwgc3RyYXR1bSAz
IG9ycGhhbgo=

# This example MachineConfig replaces ~/worker-chrony.conf
apiVersion: machineconfiguration.openshift.io/v1
kind: MachineConfig
metadata:
  labels:
    machineconfiguration.openshift.io/role: worker
  name: 99-master-etc-chrony-conf-override-for-worker
spec:
  config:
    ignition:
      version: 2.2.0
    storage:
      files:
        - contents:
            source: data:text/plain;charset=utf-8;base64,BASE64ENCODEDCONFIGFILE(1)
          filesystem: root
          mode: 0644
          path: /etc/worker-chrony.conf

$ base64 ~/worker-chrony.conf

IyBUaGlzIGZpbGUgaXMgbWFuYWdlZCBieSB0aGUgbWFjaGluZSBjb25maWcgb3BlcmF0b3IKc2Vy
dmVyIG9wZW5zaGlmdC1tYXN0ZXItMC48Y2x1c3Rlci1uYW1lPi48ZG9tYWluPiBpYnVyc3QKc2Vy
dmVyIG9wZW5zaGlmdC1tYXN0ZXItMS48Y2x1c3Rlci1uYW1lPi48ZG9tYWluPiBpYnVyc3QKc2Vy
dmVyIG9wZW5zaGlmdC1tYXN0ZXItMi48Y2x1c3Rlci1uYW1lPi48ZG9tYWluPiBpYnVyc3QKCnN0
cmF0dW13ZWlnaHQgMApkcmlmdGZpbGUgL3Zhci9saWIvY2hyb255L2RyaWZ0CnJ0Y3N5bmMKbWFr
ZXN0ZXAgMTAgMwpiaW5kY21kYWRkcmVzcyAxMjcuMC4wLjEKYmluZGNtZGFkZHJlc3MgOjoxCmtl
eWZpbGUgL2V0Yy9jaHJvbnkua2V5cwpjb21tYW5ka2V5IDEKZ2VuZXJhdGVjb21tYW5ka2V5Cm5v
Y2xpZW50bG9nCmxvZ2NoYW5nZSAwLjUKbG9nZGlyIC92YXIvbG9nL2Nocm9ueQo=

$ cp ~/ntp-server.yaml ~/clusterconfigs/manifests

$ cp ~/ntp-client.yaml ~/clusterconfigs/manifests

$ cd ~/clusterconfigs

$ cd manifests

$ touch cluster-network-avoid-workers-99-config.yaml

apiVersion: machineconfiguration.openshift.io/v1
kind: MachineConfig
metadata:
  name: 50-worker-fix-ipi-rwn
  labels:
    machineconfiguration.openshift.io/role: worker
spec:
  config:
    ignition:
      version: 3.1.0
    systemd:
      units:
      - name: nodeip-configuration.service
        enabled: true
        contents: |
          [Unit]
          Description=Writes IP address configuration so that kubelet and crio services select a valid node IP
          Wants=network-online.target
          After=network-online.target ignition-firstboot-complete.service
          Before=kubelet.service crio.service
          [Service]
          Type=oneshot
          ExecStart=/bin/bash -c "exit 0 "
          [Install]
          WantedBy=multi-user.target
    storage:
      files:
        - contents:
            source: data:,
            verification: {}
          filesystem: root
          mode: 420
          path: /etc/kubernetes/manifests/keepalived.yaml
        - contents:
            source: data:,
            verification: {}
          filesystem: root
          mode: 420
          path: /etc/kubernetes/manifests/mdns-publisher.yaml
        - contents:
            source: data:,
            verification: {}
          filesystem: root
          mode: 420
          path: /etc/kubernetes/manifests/coredns.yaml

apiVersion: operator.openshift.io/v1
kind: IngressController
metadata:
  name: default
  namespace: openshift-ingress-operator
spec:
  nodePlacement:
    nodeSelector:
      matchLabels:
        node-role.kubernetes.io/master: ""

$ sed -i "s;mastersSchedulable: false;mastersSchedulable: true;g" clusterconfigs/manifests/cluster-scheduler-02-config.yml

[user@registry ~]$ sudo firewall-cmd --add-port=5000/tcp --zone=libvirt  --permanent
[user@registry ~]$ sudo firewall-cmd --add-port=5000/tcp --zone=public   --permanent
[user@registry ~]$ sudo firewall-cmd --reload

[user@registry ~]$ sudo yum -y install python3 podman httpd httpd-tools jq

[user@registry ~]$ sudo mkdir -p /opt/registry/{auth,certs,data}

[user@registry ~]$ host_fqdn=$( hostname --long )
[user@registry ~]$ cert_c="<Country Name>"   # Country Name (C, 2 letter code)
[user@registry ~]$ cert_s="<State>"          # Certificate State (S)
[user@registry ~]$ cert_l="<Locality>"       # Certificate Locality (L)
[user@registry ~]$ cert_o="<Organization>"   # Certificate Organization (O)
[user@registry ~]$ cert_ou="<Org Unit>"      # Certificate Organizational Unit (OU)
[user@registry ~]$ cert_cn="${host_fqdn}"    # Certificate Common Name (CN)

[user@registry ~]$ openssl req \
    -newkey rsa:4096 \
    -nodes \
    -sha256 \
    -keyout /opt/registry/certs/domain.key \
    -x509 \
    -days 365 \
    -out /opt/registry/certs/domain.crt \
    -addext "subjectAltName = DNS:${host_fqdn}" \
    -subj "/C=${cert_c}/ST=${cert_s}/L=${cert_l}/O=${cert_o}/OU=${cert_ou}/CN=${cert_cn}"

[user@registry ~]$ sudo cp /opt/registry/certs/domain.crt /etc/pki/ca-trust/source/anchors/
[user@registry ~]$ sudo update-ca-trust extract

[user@registry ~]$ htpasswd -bBc /opt/registry/auth/htpasswd <user> <passwd>

[user@registry ~]$ podman create \
  --name ocpdiscon-registry \
  -p 5000:5000 \
  -e "REGISTRY_AUTH=htpasswd" \
  -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry" \
  -e "REGISTRY_HTTP_SECRET=ALongRandomSecretForRegistry" \
  -e "REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd" \
  -e "REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt" \
  -e "REGISTRY_HTTP_TLS_KEY=/certs/domain.key" \
  -e "REGISTRY_COMPATIBILITY_SCHEMA1_ENABLED=true" \
  -v /opt/registry/data:/var/lib/registry:z \
  -v /opt/registry/auth:/auth:z \
  -v /opt/registry/certs:/certs:z \
  docker.io/library/registry:2

[user@registry ~]$ podman start ocpdiscon-registry

[user@registry ~]$ scp kni@provisioner:/home/kni/pull-secret.txt pull-secret.txt

[user@registry ~]$ host_fqdn=$( hostname --long )

[user@registry ~]$ b64auth=$( echo -n '<username>:<passwd>' | openssl base64 )

[user@registry ~]$ AUTHSTRING="{\"$host_fqdn:5000\": {\"auth\": \"$b64auth\",\"email\": \"$USER@redhat.com\"}}"

[user@registry ~]$ jq ".auths += $AUTHSTRING" < pull-secret.txt > pull-secret-update.txt

[user@registry ~]$ sudo scp kni@provisioner:/usr/local/bin/oc /usr/local/bin

[user@registry ~]$ export VERSION=latest-4.8
[user@registry ~]$ UPSTREAM_REPO=$(curl -s https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/$VERSION/release.txt | awk  '/Pull From/ {print $3}')
[user@registry ~]$ /usr/local/bin/oc adm release mirror \
  -a pull-secret-update.txt
  --from=$UPSTREAM_REPO \
  --to-release-image=$LOCAL_REG/$LOCAL_REPO:${VERSION} \
  --to=$LOCAL_REG/$LOCAL_REPO

$ echo "additionalTrustBundle: |" >> install-config.yaml
$ sed -e 's/^/  /' /opt/registry/certs/domain.crt >> install-config.yaml

$ cat <<EOF >> install-config.yaml
<image-config>: (1)
- mirrors:
  - registry.example.com:5000/ocp4/openshift4
  source: quay.io/openshift-release-dev/ocp-v4.0-art-dev
- mirrors:
  - registry.example.com:5000/ocp4/openshift4
  source: registry.svc.ci.openshift.org/ocp/release
- mirrors:
  - registry.example.com:5000/ocp4/openshift4
  source: quay.io/openshift-release-dev/ocp-release
EOF

[kni@provisioner ~]$ export VERSION=latest-4.8
[kni@provisioner ~]$ curl -s https://mirror.openshift.com/pub/openshift-v4/clients/ocp/$VERSION/openshift-client-linux-$VERSION.tar.gz | tar zxvf - oc

[kni@provisioner ~]$ sudo cp oc /usr/local/bin

[kni@provisioner ~]$ echo -ne "root" | base64

[kni@provisioner ~]$ echo -ne "calvin" | base64

[kni@provisioner ~]$ vim bmh.yaml

---
apiVersion: v1
kind: Secret
metadata:
  name: openshift-worker-<num>-bmc-secret
type: Opaque
data:
  username: <base64-of-uid>
  password: <base64-of-pwd>
---
apiVersion: metal3.io/v1alpha1
kind: BareMetalHost
metadata:
  name: openshift-worker-<num>
spec:
  online: true
  bootMACAddress: <NIC1-mac-address>
  bmc:
    address: <protocol>://<bmc-ip>
    credentialsName: openshift-worker-<num>-bmc-secret

[kni@provisioner ~]$ oc -n openshift-machine-api create -f bmh.yaml

secret/openshift-worker-<num>-bmc-secret created
baremetalhost.metal3.io/openshift-worker-<num> created

[kni@provisioner ~]$ oc -n openshift-machine-api get bmh openshift-worker-<num>

NAME                 STATUS   PROVISIONING STATUS   CONSUMER   BMC                 HARDWARE PROFILE   ONLINE   ERROR
openshift-worker-<num>   OK       ready                            ipmi://<out-of-band-ip>   unknown            true

oc edit provisioning

  apiVersion: metal3.io/v1alpha1
  kind: Provisioning
  metadata:
    creationTimestamp: "2021-08-05T18:51:50Z"
    finalizers:
    - provisioning.metal3.io
    generation: 8
    name: provisioning-configuration
    resourceVersion: "551591"
    uid: f76e956f-24c6-4361-aa5b-feaf72c5b526
  spec:
    preProvisioningOSDownloadURLs: {}
    provisioningDHCPRange: 172.22.0.10,172.22.0.254
    provisioningIP: 172.22.0.3
    provisioningInterface: enp1s0
    provisioningNetwork: Managed
    provisioningNetworkCIDR: 172.22.0.0/24
    provisioningOSDownloadURL: http://192.168.111.1/images/rhcos-49.84.202107010027-0-openstack.x86_64.qcow2.gz?sha256=c7dde5f96826c33c97b5a4ad34110212281916128ae11100956f400db3d5299e
    virtualMediaViaExternalNetwork: true (1)
  status:
    generations:
    - group: apps
      hash: ""
      lastGeneration: 7
      name: metal3
      namespace: openshift-machine-api
      resource: deployments
    - group: apps
      hash: ""
      lastGeneration: 1
      name: metal3-image-cache
      namespace: openshift-machine-api
      resource: daemonsets
    observedGeneration: 8
    readyReplicas: 0

oc edit machineset

  apiVersion: machine.openshift.io/v1beta1
  kind: MachineSet
  metadata:
    creationTimestamp: "2021-08-05T18:51:52Z"
    generation: 11
    labels:
      machine.openshift.io/cluster-api-cluster: ostest-hwmdt
      machine.openshift.io/cluster-api-machine-role: worker
      machine.openshift.io/cluster-api-machine-type: worker
    name: ostest-hwmdt-worker-0
    namespace: openshift-machine-api
    resourceVersion: "551513"
    uid: fad1c6e0-b9da-4d4a-8d73-286f78788931
  spec:
    replicas: 2
    selector:
      matchLabels:
        machine.openshift.io/cluster-api-cluster: ostest-hwmdt
        machine.openshift.io/cluster-api-machineset: ostest-hwmdt-worker-0
    template:
      metadata:
        labels:
          machine.openshift.io/cluster-api-cluster: ostest-hwmdt
          machine.openshift.io/cluster-api-machine-role: worker
          machine.openshift.io/cluster-api-machine-type: worker
          machine.openshift.io/cluster-api-machineset: ostest-hwmdt-worker-0
      spec:
        metadata: {}
        providerSpec:
          value:
            apiVersion: baremetal.cluster.k8s.io/v1alpha1
            hostSelector: {}
            image:
              checksum: http:/172.22.0.3:6181/images/rhcos-49.84.202107010027-0-openstack.x86_64.qcow2/cached-rhcos-49.84.202107010027-0-openstack.x86_64.qcow2.md5sum (1)
              url: http://172.22.0.3:6181/images/rhcos-49.84.202107010027-0-openstack.x86_64.qcow2/cached-rhcos-49.84.202107010027-0-openstack.x86_64.qcow2 (2)
            kind: BareMetalMachineProviderSpec
            metadata:
              creationTimestamp: null
            userData:
              name: worker-user-data
  status:
    availableReplicas: 2
    fullyLabeledReplicas: 2
    observedGeneration: 11
    readyReplicas: 2
    replicas: 2

[kni@provisioner ~]$  oc -n openshift-machine-api get bmh openshift-worker-<num>

NAME                 STATUS   PROVISIONING STATUS   CONSUMER   BMC                 HARDWARE PROFILE   ONLINE   ERROR
openshift-worker-<num>   OK       ready                            ipmi://<out-of-band-ip>   unknown            true

[kni@provisioner ~]$ oc get nodes

NAME                                                STATUS   ROLES           AGE     VERSION
provisioner.openshift.example.com            Ready    master          30h     v1.16.2
openshift-master-1.openshift.example.com            Ready    master          30h     v1.16.2
openshift-master-2.openshift.example.com            Ready    master          30h     v1.16.2
openshift-master-3.openshift.example.com            Ready    master          30h     v1.16.2
openshift-worker-0.openshift.example.com            Ready    master          30h     v1.16.2
openshift-worker-1.openshift.example.com            Ready    master          30h     v1.16.2

[kni@provisioner ~]$ oc get machinesets -n openshift-machine-api

NAME                                DESIRED   CURRENT   READY   AVAILABLE   AGE
...
openshift-worker-0.example.com      1         1         1       1           55m
openshift-worker-1.example.com      1         1         1       1           55m

[kni@provisioner ~]$ oc scale --replicas=<num> machineset <machineset> -n openshift-machine-api

[kni@provisioner ~]$ oc -n openshift-machine-api get bmh openshift-worker-<num>

NAME                 STATUS   PROVISIONING STATUS   CONSUMER                  BMC                 HARDWARE PROFILE   ONLINE   ERROR
openshift-worker-<num>   OK       provisioning          openshift-worker-<num>-65tjz   ipmi://<out-of-band-ip>   unknown            true

NAME                 STATUS   PROVISIONING STATUS   CONSUMER                  BMC                 HARDWARE PROFILE   ONLINE   ERROR
openshift-worker-<num>   OK       provisioned           openshift-worker-<num>-65tjz   ipmi://<out-of-band-ip>   unknown            true

[kni@provisioner ~]$ oc get nodes

NAME                                          STATUS   ROLES   AGE     VERSION
provisioner.openshift.example.com             Ready    master  30h     v1.16.2
openshift-master-1.openshift.example.com      Ready    master  30h     v1.16.2
openshift-master-2.openshift.example.com      Ready    master  30h     v1.16.2
openshift-master-3.openshift.example.com      Ready    master  30h     v1.16.2
openshift-worker-0.openshift.example.com      Ready    master  30h     v1.16.2
openshift-worker-1.openshift.example.com      Ready    master  30h     v1.16.2
openshift-worker-<num>.openshift.example.com  Ready    worker  3m27s   v1.16.2

[kni@provisioner ~]$ ssh openshift-worker-<num>

[kni@openshift-worker-<num>]$ journalctl -fu kubelet

[kni@provisioner ~]$ echo -ne "root" | base64

[kni@provisioner ~]$ echo -ne "calvin" | base64

[kni@provisioner ~]$ vim bmh.yaml

---
apiVersion: v1
kind: Secret
metadata:
  name: openshift-worker-<num>-bmc-secret
type: Opaque
data:
  username: <base64-of-uid>
  password: <base64-of-pwd>
---
apiVersion: metal3.io/v1alpha1
kind: BareMetalHost
metadata:
  name: openshift-worker-<num>
spec:
  online: true
  bootMACAddress: <NIC1-mac-address>
  bmc:
    address: ipmi://<bmc-ip>
    credentialsName: openshift-worker-<num>-bmc-secret

[kni@provisioner ~]$ oc -n openshift-machine-api create -f bmh.yaml

secret/openshift-worker-<num>-bmc-secret created
baremetalhost.metal3.io/openshift-worker-<num> created

[kni@provisioner ~]$ oc -n openshift-machine-api get bmh openshift-worker-<num>

NAME                 STATUS   PROVISIONING STATUS   CONSUMER   BMC                 HARDWARE PROFILE   ONLINE   ERROR
openshift-worker-<num>   OK       ready                            ipmi://<out-of-band-ip>   unknown            true

[kni@provisioner ~]$  oc -n openshift-machine-api get bmh openshift-worker-<num>

NAME                 STATUS   PROVISIONING STATUS   CONSUMER   BMC                 HARDWARE PROFILE   ONLINE   ERROR
openshift-worker-<num>   OK       ready                            ipmi://<out-of-band-ip>   unknown            true

[kni@provisioner ~]$ oc get nodes

[kni@provisioner ~]$ oc get machinesets -n openshift-machine-api

NAME                                DESIRED   CURRENT   READY   AVAILABLE   AGE
openshift-worker-0.example.com      1         1         1       1           55m
openshift-worker-1.example.com      1         1         1       1           55m
openshift-worker-2.example.com      1         1         1       1           55m

[kni@provisioner ~]$ oc scale --replicas=<num> machineset <machineset> -n openshift-machine-api

[kni@provisioner ~]$ oc -n openshift-machine-api get bmh openshift-worker-<num>

NAME                 STATUS   PROVISIONING STATUS   CONSUMER                  BMC                 HARDWARE PROFILE   ONLINE   ERROR
openshift-worker-<num>   OK       provisioning          openshift-worker-<num>-65tjz   ipmi://<out-of-band-ip>   unknown            true

NAME                 STATUS   PROVISIONING STATUS   CONSUMER                  BMC                 HARDWARE PROFILE   ONLINE   ERROR
openshift-worker-<num>   OK       provisioned           openshift-worker-<num>-65tjz   ipmi://<out-of-band-ip>   unknown            true

[kni@provisioner ~]$ oc get nodes

NAME                                                STATUS   ROLES           AGE     VERSION
provisioner.openshift.example.com            Ready    master          30h     v1.16.2
openshift-master-1.openshift.example.com            Ready    master          30h     v1.16.2
openshift-master-2.openshift.example.com            Ready    master          30h     v1.16.2
openshift-worker-<num>.openshift.example.com        Ready    worker          3m27s   v1.16.2

[kni@provisioner ~]$ ssh openshift-worker-<num>

[kni@openshift-worker-<num>]$ journalctl -fu kubelet

$ORIGIN openshift.example.com.
$TTL 300        ; 5 minutes
@  IN  SOA  dns1.example.com.  hostmaster.example.com. (
       2001062501  ; serial
       21600       ; refresh after 6 hours
       3600        ; retry after 1 hour
       604800      ; expire after 1 week
       86400 )     ; minimum TTL of 1 day
;
api                     A       <api-ip>
ns1                     A       <dns-vip-ip>
$ORIGIN apps.openshift.example.com.
*                       A       <wildcard-ingress-lb-ip>
$ORIGIN openshift.example.com.
provisioner             A       <NIC2-ip-of-provision>
openshift-master-0      A       <NIC2-ip-of-openshift-master-0>
openshift-master-1      A       <NIC2-ip-of-openshift-master-1>
openshift-master-2      A       <NIC2-ip-of-openshift-master-2>
openshift-worker-0      A       <NIC2-ip-of-openshift-worker-0>
openshift-worker-1      A       <NIC2-ip-of-openshift-worker-1>

$ORIGIN 1.0.10.in-addr.arpa.
$TTL 300
@  IN  SOA  dns1.example.com.  hostmaster.example.com. (
       2001062501  ; serial
       21600       ; refresh after 6 hours
       3600        ; retry after 1 hour
       604800      ; expire after 1 week
       86400 )     ; minimum TTL of 1 day
;
126 IN      PTR      provisioner.openshift.example.com.
127	IN        	PTR    	openshift-master-0.openshift.example.com.
128	IN        	PTR    	openshift-master-1.openshift.example.com.
129	IN 	        PTR   	openshift-master-2.openshift.example.com.
130	IN 	        PTR   	openshift-worker-0.openshift.example.com.
131	IN        	PTR    	openshift-worker-1.openshift.example.com.
132 IN      PTR     api.openshift.example.com.
133 IN      PTR     ns1.openshift.example.com.

cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
<NIC2-IP> provisioner.openshift.example.com provisioner
<NIC2-IP> openshift-master-0.openshift.example.com openshift-master-0
<NIC2-IP> openshift-master-1.openshift.example.com openshift-master-1
<NIC2-IP> openshift-master-2.openshift.example.com openshift-master-2
<NIC2-IP> openshift-worker-0.openshift.example.com openshift-worker-0
<NIC2-IP> openshift-worker-1.openshift.example.com openshift-worker-1
<API-IP>  api.openshift.example.com api
<DNS-VIP-IP> ns1.openshift.example.com ns1

systemctl restart firewalld
firewall-cmd --add-service=dns --permanent
firewall-cmd --reload

host provisioner {
     option host-name "provisioner";
     hardware ethernet <mac-address-of-NIC2>;
     option domain-search "openshift.example.com";
     fixed-address <ip-address-of-NIC2>;
  }
host openshift-master-0 {
     option host-name "openshift-master-0";
     hardware ethernet <mac-address-of-NIC2>;
     option domain-search "openshift.example.com";
     fixed-address <ip-address-of-NIC2>;
  }

host openshift-master-1 {
     option host-name "openshift-master-1";
     hardware ethernet <mac-address-of-NIC2>;
     option domain-search "openshift.example.com";
     fixed-address <ip-address-of-NIC2>;
  }

host openshift-master-2 {
     option host-name "openshift-master-2";
     hardware ethernet <mac-address-of-NIC2>;
     option domain-search "openshift.example.com";
     fixed-address <ip-address-of-NIC2>;
  }
host openshift-worker-0 {
     option host-name "openshift-worker-0";
     hardware ethernet <mac-address-of-NIC2>;
     option domain-search "openshift.example.com";
     fixed-address <ip-address-of-NIC2>;
  }
host openshift-worker-1 {
     option host-name "openshift-worker-1";
     hardware ethernet <mac-address-of-NIC2>;
     option domain-search "openshift.example.com";
     fixed-address <ip-address-of-NIC2>;
  }

systemctl restart dhcpd

dnf install -y dnsmasq

cd /etc/dnsmasq.d

touch <filename>.dns

systemctl restart firewalld
firewall-cmd --add-service=dhcp --permanent
firewall-cmd --reload

domain-needed
bind-dynamic
bogus-priv
domain=openshift.example.com
dhcp-range=<baremetal-net-starting-ip,baremetal-net-ending-ip>
#dhcp-range=10.0.1.4,10.0.14
dhcp-option=3,<baremetal-net-gateway-ip>
#dhcp-option=3,10.0.1.254
resolv-file=/etc/resolv.conf.upstream
interface=<nic-with-access-to-baremetal-net>
#interface=em2
server=<ip-of-existing-server-on-baremetal-net>


#Wildcard for apps -- make changes to cluster-name (openshift) and domain (example.com)
address=/.apps.openshift.example.com/<wildcard-ingress-lb-ip>

#Static IPs for Masters
dhcp-host=<NIC2-mac-address>,provisioner.openshift.example.com,<ip-of-provisioner>
dhcp-host=<NIC2-mac-address>,openshift-master-0.openshift.example.com,<ip-of-openshift-master-0>
dhcp-host=<NIC2-mac-address>,openshift-master-1.openshift.example.com,<ip-of-openshift-master-1>
dhcp-host=<NIC2-mac-address>,openshift-master-2.openshift.example.com,<ip-of-openshift-master-2>
dhcp-host=<NIC2-mac-address>,openshift-worker-0.openshift.example.com,<ip-of-openshift-worker-0>
dhcp-host=<NIC2-mac-address>,openshift-worker-1.openshift.example.com,<ip-of-openshift-worker-1>

strict-order
bind-dynamic
bogus-priv
dhcp-authoritative
dhcp-range=baremetal,<baremetal-IPv6-dhcp-range-start>,<baremetal-IPv6-dhcp-range-end>,<range-prefix>
dhcp-option=baremetal,option6:dns-server,[<IPv6-DNS-Server>]

resolv-file=/etc/resolv.conf.upstream
except-interface=lo
dhcp-lease-max=81
log-dhcp

domain=openshift.example.com,<baremetal-IPv6-cidr>,local

# static host-records
address=/.apps.openshift.example.com/<wildcard-ingress-lb-ip>
host-record=api.openshift.example.com,<api-ip>
host-record=ns1.openshift.example.com,<dns-ip>
host-record=openshift-master-0.openshift.example.com,<ip-of-openshift-master-0>
host-record=openshift-master-1.openshift.example.com,<ip-of-openshift-master-1>
host-record=openshift-master-2.openshift.example.com,<ip-of-openshift-master-1>
# Registry
host-record=registry.openshift.example.com,<ip-of-registry-server>

#Static IPs for Masters
dhcp-host=<baremetal-nic-duid>,openshift-master-0.openshift.example.com,<ip-of-openshift-master-0>
dhcp-host=<baremetal-nic-duid>,openshift-master-1.openshift.example.com,<ip-of-openshift-master-1>
dhcp-host=<baremetal-nic-duid>,openshift-master-2.openshift.example.com,<ip-of-openshift-master-2>

search <domain.com>
nameserver <ip-of-my-existing-dns-nameserver>

systemctl restart dnsmasq

systemctl status dnsmasq