Consume openvswitch-ipsec systemd service for OVN IPsec deployment

The ovn-ipsec-host daemonset pod currently spins up ovs-monitor-ipsec process
to configure IPsec connections with the peer nodes. This would make ipsec
connections to be established for the existing nodes a bit later after kubelet
is started at the time node/service restart scenario, but by the time workloads
are scheduled on the node started hitting traffic drops because of
unavailability of IPsec connections between nodes. This makes IPsec jobs in CI
so unstable and monitor jobs always failing during IPsec upgrade.

The FDP story (https://issues.redhat.com/browse/FDP-1051) gets openvswitch-ipsec
systemd service (runs ovs-monitor-ipsec) with required configurable parameters,
It's available with OVS 3.5 version. So this commit does the following.

1. Stop spawning ovs-monitor-ipsec as foreground process in the ovn-ipsec
container, Instead setup required IPsec configuration parameters in the
/etc/sysconfig/openvswitch file, enable and start the openvswitch-ipsec
service on the host. This is done at the of when ovn-ipsec-host pod is
coming up for the first time, for the pod restart scenarios, it just
checks openvswitch-ipsec service is running on the host, otherwise exit
from the container with error.

2. Keep running an ovn-ipsec container and redirects /var/log/openvswitch/ovs-monitor-ipsec.log
to the ovn-ipsec container's stdout console.

3. There is no necessity of having ovn-ipsec-clean container anymore with
openvswitch-ipsec service as it's going to handle OVN IPsec states
appropriately.

4. During the OCP upgrade, the new ipsec os extension takes while to deploy
with openvswitch3.5-ipsec package, so by the time ovn-ipsec-host daemonset
is rendered, We need to handle that scenario by running ovs-monitor-ipsec
in the container. so this commit is also considering the transition phase
of the process that is moving from container to host.

5. The ovn-keys init container configures ovs with IPsec certificate paths,
so the container uses same host directory path to store and configure ovs
with certificates because the ovs-monitor-ipsec process is running on the
host now.

Signed-off-by: Periyasamy Palanisamy <[email protected]>

Author: pperiyasamy

bindata/network/ovn-kubernetes/common/ipsec-host.yaml

@@ -109,7 +109,7 @@ spec:
           # authentic.
           echo "Configuring IPsec keys"
 
-          cert_pem=/etc/openvswitch/keys/ipsec-cert.pem
+          cert_pem=/var/lib/openvswitch/etc/keys/ipsec-cert.pem
 
           # If the certificate does not exist or it will expire in the next 6 months
           # (15770000 seconds), we will generate a new one.
@@ -118,18 +118,18 @@ spec:
             # is a requirement by OVN.
             cn=$(ovs-vsctl --retry -t 60 get Open_vSwitch . external-ids:system-id | tr -d "\"")
 
-            mkdir -p /etc/openvswitch/keys
+            mkdir -p /var/lib/openvswitch/etc/keys
 
             # Generate an SSL private key and use the key to create a certitificate signing request
-            umask 077 && openssl genrsa -out /etc/openvswitch/keys/ipsec-privkey.pem 2048
+            umask 077 && openssl genrsa -out /var/lib/openvswitch/etc/keys/ipsec-privkey.pem 2048
             openssl req -new -text \
                         -extensions v3_req \
                         -addext "subjectAltName = DNS:${cn}" \
                         -subj "/C=US/O=ovnkubernetes/OU=kind/CN=${cn}" \
-                        -key /etc/openvswitch/keys/ipsec-privkey.pem \
-                        -out /etc/openvswitch/keys/ipsec-req.pem
+                        -key /var/lib/openvswitch/etc/keys/ipsec-privkey.pem \
+                        -out /var/lib/openvswitch/etc/keys/ipsec-req.pem
 
-            csr_64=$(base64 -w0 /etc/openvswitch/keys/ipsec-req.pem) # -w0 to avoid line-wrap
+            csr_64=$(base64 -w0 /var/lib/openvswitch/etc/keys/ipsec-req.pem) # -w0 to avoid line-wrap
 
             # Request that our generated certificate signing request is
             # signed by the "network.openshift.io/signer" signer that is
@@ -169,7 +169,7 @@ spec:
             # kubectl delete csr/$(hostname)
 
             # Get the CA certificate so we can authenticate peer nodes.
-            openssl x509 -in /signer-ca/ca-bundle.crt -outform pem -text -out /etc/openvswitch/keys/ipsec-cacert.pem
+            openssl x509 -in /signer-ca/ca-bundle.crt -outform pem -text -out /var/lib/openvswitch/etc/keys/ipsec-cacert.pem
           fi
 
           # Configure OVS with the relevant keys for this node. This is required by ovs-monitor-ipsec.
@@ -178,8 +178,8 @@ spec:
           # the will get read and loaded into NSS by the ovs-monitor-ipsec process
           # which has not started yet.
           ovs-vsctl --retry -t 60 set Open_vSwitch . other_config:certificate=$cert_pem \
-                                                     other_config:private_key=/etc/openvswitch/keys/ipsec-privkey.pem \
-                                                     other_config:ca_cert=/etc/openvswitch/keys/ipsec-cacert.pem
+                                                     other_config:private_key=/var/lib/openvswitch/etc/keys/ipsec-privkey.pem \
+                                                     other_config:ca_cert=/var/lib/openvswitch/etc/keys/ipsec-cacert.pem
         env:
         - name: K8S_NODE
           valueFrom:
@@ -196,8 +196,8 @@ spec:
           name: host-var-run
         - mountPath: /signer-ca
           name: signer-ca
-        - mountPath: /etc/openvswitch
-          name: etc-openvswitch
+        - mountPath: /var/lib
+          name: host-var-lib
         - mountPath: /etc
           name: host-etc
         resources:
@@ -242,6 +242,36 @@ spec:
             exit 2
           fi
 
+          # When openvswitch-ipsec service exists on the node, then reuse it to
+          # configure IPsec for the pod traffic. Otherwise fall back to configuring
+          # IPsec by spinning up ovs-monitor-ipsec process within the container.
+          ovsipsecservice="openvswitch-ipsec"
+          if chroot /proc/1/root systemctl list-unit-files --type=service | grep "$ovsipsecservice"; then
+            if ! grep -q "openshift.conf" /etc/sysconfig/openvswitch; then
+              sed -i 's|OPTIONS=\"\"|OPTIONS=\"--no-restart-ike-daemon --ovs-monitor-ipsec-options='\''--ipsec-conf=\/etc\/ipsec.d\/openshift.conf --root-ipsec-conf=\/etc\/ipsec.conf --ipsec-d=\/var\/lib\/ipsec\/nss --use-default-crypto'\''\"|' /etc/sysconfig/openvswitch
+              chroot /proc/1/root systemctl enable --now $ovsipsecservice
+              counter=0
+              until [ -r /var/run/openvswitch/ovs-monitor-ipsec.pid ]; do
+                counter=$((counter+1))
+                sleep 1
+                if [ $counter -gt 300 ];
+                then
+                  echo "$ovsipsecservice service has not started after $counter seconds"
+                  exit 1
+                fi
+              done
+            fi
+            if ! chroot /proc/1/root systemctl is-active --quiet $ovsipsecservice; then
+                echo "$ovsipsecservice service is not running, check system logs"
+                exit 1
+            fi
+            while true; do
+              tail -F /var/log/openvswitch/ovs-monitor-ipsec.log
+              echo "tail on ovs-monitor-ipsec.log failed. attempting again"
+              sleep 2s
+            done
+          fi
+
           # The ovs-monitor-ipsec doesn't set authby, so when it calls ipsec auto --start
           # the default ones defined at Libreswan's compile time will be used. On restart,
           # Libreswan will use authby from libreswan.config. If libreswan.config is
@@ -295,6 +325,10 @@ spec:
                  - |
                    #!/bin/bash
                    set -exuo pipefail
+                   if chroot /proc/1/root systemctl is-active --quiet openvswitch-ipsec; then
+                     echo "openvswitch-ipsec service is running on the host, no action needed in pre stop"
+                     exit 0
+                   fi
                    # In order to maintain traffic flows during container restart, we
                    # need to ensure that xfrm state and policies are not flushed.
 
@@ -322,8 +356,6 @@ spec:
           name: host-var-run
         - mountPath: /var/log/openvswitch/
           name: host-var-log-ovs
-        - mountPath: /etc/openvswitch
-          name: etc-openvswitch
         - mountPath: /var/lib
           name: host-var-lib
         - mountPath: /etc
@@ -332,6 +364,8 @@ spec:
           name: usr-sbin
         - mountPath: /usr/libexec
           name: usr-libexec
+        - mountPath: /usr/lib/systemd
+          name: usr-libsystemd
         resources:
           requests:
             cpu: 10m
@@ -430,7 +464,15 @@ spec:
 
           # Function to handle SIGTERM
           cleanup() {
-            echo "received SIGTERM, flushing ipsec config"
+            echo "received SIGTERM, flushing ipsec config if required"
+            # When IPsec connections are managed by openvswitch-ipsec systemd service,
+            # It means ovs-monitor-ipsec process is running on the host always, so ip
+            # xfrm state and policy entries are managed properly according to ipsec
+            # configuration on the OVN. so skip flushing ipsec for this case.
+            if chroot /proc/1/root systemctl is-active --quiet openvswitch-ipsec; then
+              echo "openvswitch-ipsec service is running on the host, no ipsec flush needed"
+              exit 0
+            fi
             # Wait upto 15 seconds for ovs-monitor-ipsec process to terminate before
             # cleaning up ipsec entries.
             counter=0
@@ -462,14 +504,19 @@ spec:
           done
           echo "ovs-monitor-ipsec is started"
 
-          # Monitor the ovs-monitor-ipsec process.
-          while kill -0 "$(cat /var/run/openvswitch/ovs-monitor-ipsec.pid 2>/dev/null)"; do
-            sleep 1
-          done
+          # When ovs-monitor-ipsec process is not running via openvswitch-ipsec systemd
+          # service, then keep monitoring ovn-ipsec container process until it terminates
+          # in the ovn-ipsec container's pre stop hook.
+          if ! chroot /proc/1/root systemctl is-active --quiet openvswitch-ipsec; then
+            # Monitor the ovs-monitor-ipsec process.
+            while kill -0 "$(cat /var/run/openvswitch/ovs-monitor-ipsec.pid 2>/dev/null)"; do
+              sleep 1
+            done
 
-          # Once the ovs-monitor-ipsec process terminates, execute the cleanup command.
-          echo "ovs-monitor-ipsec is terminated, flushing ipsec config"
-          ipsecflush
+            # Once the ovs-monitor-ipsec process terminates, execute the cleanup command.
+            echo "ovs-monitor-ipsec is terminated, flushing ipsec config"
+            ipsecflush
+          fi
 
           # Continue running until SIGTERM is received (or exit naturally)
           while true; do
@@ -508,10 +555,6 @@ spec:
           defaultMode: 420
           name: signer-ca
         name: signer-ca
-      - hostPath:
-          path: /var/lib/openvswitch/etc
-          type: DirectoryOrCreate
-        name: etc-openvswitch
       - hostPath:
           path: "{{.CNIConfDir}}"
         name: host-cni-netd
@@ -535,6 +578,10 @@ spec:
           path: /usr/libexec
           type: Directory
         name: usr-libexec
+      - hostPath:
+          path: /usr/lib/systemd
+          type: DirectoryOrCreate
+        name: usr-libsystemd
       tolerations:
       - operator: "Exists"
 {{end}}