Merge pull request #16571 from smarterclayton/bootstrap_config

Automatic merge from submit-queue.

Allow bootstrap configuration to be configured and reentrant

Make bootstrapping a real production node possible.

1. Simplify and streamline the process whereby the bootstrap config is looked up.  
    1. `--bootstrap-config-name` can be used to customize which config is looked up (one per node group)
    2. Any failure in fetching node config results in termination of the pass - no client side defaulting
    3. Handle 0.0.0.0 in dnsIP as a special case which results in the local node IP lookup
2. Backport the one remaining cert rotation patch and make client and server node side cert rotation mandatory when bootstrapping
    1. Fix a number of small issues upstream where node bootstrapping is not reentrant
    2. pass cert-dir to the kubelet instead of passing individual key and cert, which was preventing rotation from working
3. Make `openshift start network` work podified
    1.  Allow kubeconfig to be specified on the CLI and override the node-config
    2. The proxy healthz was not starting due to missing config - this has been corrected
    3. Provide a daemonset example that correctly starts OpenShift SDN in a pod
    4. Fix a few minor bugs in openshift-sdn

There is still one bug outstanding upstream that can be fixed separately - the kubelet client rotation can fail due to the cert expiring and be unable to get new certs, so it never exits.

Tested the following scenario extensively (requires a new openshift/node image tagged as v3.7.0-alpha.1):

1. Update master-config with cert signing on on 10m rotation
2. create a node-config `oc create configmap -n openshift-node node-config --from-file=node-config.yaml=contrib/kubernetes/default-node-config.yaml`
3. start node in bootstrapping mode `openshift start node --bootstrap-config-name=node-config --config=/etc/origin/node/node-config.yaml --enable=kubelet --loglevel=3` (which has it run only the kubelet)
4. run a background `oc observe csr -- oc adm certificate approve` to approve both csr
5. run `oc create -f contrib/kubernetes/static/network-policy.yaml`
6. run `oc create -f contrib/kubernetes/static/network-daemonset.yaml`
7. verify the daemonset starts correctly and that it passes health checks
8. launch a new pod and verify it has dns `oc run --restart=Never --attach -it --image=centos:7 -- /bin/bash` and then `yum install bind-utils -y && dig +search kubernetes.default.svc`
9. launch two pods in two namespaces and verify multi tenant SDN works

Follow up for the daemonset - openshift-sdn expects to have access to the dockershim.sock which this doesn't bind mount in.

Author: openshift-merge-robot

contrib/completions/bash/openshift

@@ -33708,6 +33708,8 @@ _openshift_start_network()
     local_nonpersistent_flags+=("--kubernetes=")
     flags+=("--latest-images")
     local_nonpersistent_flags+=("--latest-images")
+    flags+=("--listen=")
+    local_nonpersistent_flags+=("--listen=")
     flags+=("--network-plugin=")
     local_nonpersistent_flags+=("--network-plugin=")
     flags+=("--recursive-resolv-conf=")
@@ -33731,8 +33733,8 @@ _openshift_start_node()
     flags_with_completion=()
     flags_completion=()
 
-    flags+=("--bootstrap")
-    local_nonpersistent_flags+=("--bootstrap")
+    flags+=("--bootstrap-config-name=")
+    local_nonpersistent_flags+=("--bootstrap-config-name=")
     flags+=("--config=")
     flags_with_completion+=("--config")
     flags_completion+=("__handle_filename_extension_flag yaml|yml")

contrib/completions/zsh/openshift

@@ -33857,6 +33857,8 @@ _openshift_start_network()
     local_nonpersistent_flags+=("--kubernetes=")
     flags+=("--latest-images")
     local_nonpersistent_flags+=("--latest-images")
+    flags+=("--listen=")
+    local_nonpersistent_flags+=("--listen=")
     flags+=("--network-plugin=")
     local_nonpersistent_flags+=("--network-plugin=")
     flags+=("--recursive-resolv-conf=")
@@ -33880,8 +33882,8 @@ _openshift_start_node()
     flags_with_completion=()
     flags_completion=()
 
-    flags+=("--bootstrap")
-    local_nonpersistent_flags+=("--bootstrap")
+    flags+=("--bootstrap-config-name=")
+    local_nonpersistent_flags+=("--bootstrap-config-name=")
     flags+=("--config=")
     flags_with_completion+=("--config")
     flags_completion+=("__handle_filename_extension_flag yaml|yml")

contrib/kubernetes/controllers.yaml

@@ -0,0 +1,51 @@
+allowDisabledDocker: false
+apiVersion: v1
+authConfig:
+  authenticationCacheSize: 1000
+  authenticationCacheTTL: 5m
+  authorizationCacheSize: 1000
+  authorizationCacheTTL: 5m
+dnsDomain: cluster.local
+dnsIP: 0.0.0.0
+dnsBindAddress: 0.0.0.0:53
+dnsRecursiveResolvConf: ""
+dockerConfig:
+  dockerShimRootDirectory: /var/lib/dockershim
+  dockerShimSocket: /var/run/kubernetes/dockershim.sock
+  execHandlerName: native
+enableUnidling: true
+imageConfig:
+  format: openshift/origin-${component}:${version}
+  latest: false
+iptablesSyncPeriod: 30s
+kind: NodeConfig
+kubeletArguments:
+  cert-dir:
+  - ./certificates
+  feature-gates:
+  - RotateKubeletClientCertificate=true,RotateKubeletServerCertificate=true
+masterClientConnectionOverrides:
+  acceptContentTypes: application/vnd.kubernetes.protobuf,application/json
+  burst: 40
+  contentType: application/vnd.kubernetes.protobuf
+  qps: 20
+masterKubeConfig: node.kubeconfig
+networkConfig:
+  mtu: 1450
+  networkPluginName: redhat/openshift-ovs-multitenant
+nodeIP: ""
+proxyArguments:
+  healthz-bind-address: 
+  - 0.0.0.0
+  healthz-port:
+  - "10256"
+  metrics-bind-address:
+  - 0.0.0.0:10257
+servingInfo:
+  bindAddress: 0.0.0.0:10250
+  bindNetwork: tcp4
+  namedCertificates: null
+volumeConfig:
+  localQuota:
+    perFSGroup: null
+volumeDirectory: /var/lib/origin/volumes

contrib/kubernetes/default-node-config.yaml

@@ -5,7 +5,7 @@ metadata:
 spec:
   containers:
   - name: controllers
-    image: openshift/origin:v3.6.0-rc.0
+    image: openshift/origin:v3.6.0
     command: ["/usr/bin/openshift", "start", "master", "controllers"]
     args:
     - "--config=/etc/origin/master/master-config.yaml"

contrib/kubernetes/static/controllers-pod.yaml

@@ -0,0 +1,157 @@
+kind: DaemonSet
+apiVersion: extensions/v1beta1
+metadata:
+  name: sdn
+  annotations:
+    kubernetes.io/description: |
+      This daemon set launches the OpenShift networking components (kube-proxy, DNS, and openshift-sdn).
+      It expects that OVS is running on the node.
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        component: network
+        type: infra
+        openshift.io/role: network
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+      # Requires fairly broad permissions - ability to read all services and network functions as well
+      # as all pods.
+      serviceAccountName: sdn
+      hostNetwork: true
+      hostPID: true
+      containers:
+      - name: network
+        image: openshift/node:v3.7.0-alpha.1
+        command: 
+        - /bin/bash
+        - -c
+        - |
+          #!/bin/sh
+          set -o errexit
+          # Take over network functions on the node
+          rm -Rf /etc/cni/net.d/*
+          rm -Rf /host/opt/cni/bin/*
+          cp -Rf /opt/cni/bin/* /host/opt/cni/bin/
+          # Use whichever node-config exists
+          cfg=/etc/openshift/node
+          if [[ ! -f "${cfg}/node-config.yaml" ]]; then
+            cfg=/etc/origin/node
+          fi
+          # Use the same config as the node, but with the service account token
+          openshift cli config "--config=${cfg}/node.kubeconfig" view --flatten > /tmp/kubeconfig
+          openshift cli config --config=/tmp/kubeconfig set-credentials sa "--token=$( cat /var/run/secrets/kubernetes.io/serviceaccount/token )"
+          openshift cli config --config=/tmp/kubeconfig set-context "$( openshift cli config current-context)" --user=sa
+          # Launch the network process
+          exec openshift start network "--config=${cfg}/node-config.yaml" --kubeconfig=/tmp/kubeconfig --loglevel=5
+
+        securityContext:
+          runAsUser: 0
+          # Permission could be reduced by selecting an appropriate SELinux policy
+          privileged: true
+        # TODO: debugging only
+        imagePullPolicy: Never
+        volumeMounts:
+        # Directory which contains the host configuration. We look at both locations
+        # to simplify setup.
+        - mountPath: /etc/origin/node/
+          name: host-config
+          readOnly: true
+        - mountPath: /etc/openshift/node/
+          name: host-config-alt
+          readOnly: true
+        # Run directories where we need to be able to access sockets
+        - mountPath: /var/run/dbus/
+          name: host-var-run-dbus
+          readOnly: true
+        - mountPath: /var/run/openvswitch/
+          name: host-var-run-ovs
+          readOnly: true
+        - mountPath: /var/run/kubernetes/
+          name: host-var-run-kubernetes
+          readOnly: true
+        # We mount our socket here
+        - mountPath: /var/run/openshift-sdn
+          name: host-var-run-openshift-sdn
+        # CNI related mounts which we take over
+        - mountPath: /host/opt/cni/bin
+          name: host-opt-cni-bin
+        - mountPath: /etc/cni/net.d
+          name: host-etc-cni-netd
+        - mountPath: /var/lib/cni/networks/openshift-sdn
+          name: host-var-lib-cni-networks-openshift-sdn
+
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+        env:
+        - name: OPENSHIFT_DNS_DOMAIN
+          value: cluster.local
+        ports:
+        - name: healthz
+          containerPort: 10256
+        livenessProbe:
+          initialDelaySeconds: 10
+          httpGet:
+            path: /healthz
+            port: 10256
+            scheme: HTTP
+        lifecycle:
+          # postStart:
+          #   exec:
+          #     command:
+          #     - /usr/bin/dbus-send
+          #     - --system
+          #     - --dest=uk.org.thekelleys.dnsmasq
+          #     - /uk/org/thekelleys/dnsmasq
+          #     - uk.org.thekelleys.SetDomainServers
+          #     - array:string:/in-addr.arpa/127.0.0.1,/$(OPENSHIFT_DNS_DOMAIN)/127.0.0.1
+          # preStop:
+          #   exec:
+          #     command:
+          #     - /usr/bin/dbus-send
+          #     - --system
+          #     - --dest=uk.org.thekelleys.dnsmasq
+          #     - /uk/org/thekelleys/dnsmasq
+          #     - uk.org.thekelleys.SetDomainServers
+          #     - "array:string:"
+
+      volumes:
+      # In bootstrap mode, the host config contains information not easily available
+      # from other locations.
+      - name: host-config
+        hostPath:
+          path: /etc/origin/node
+      - name: host-config-alt
+        hostPath:
+          path: /etc/openshift/node
+      - name: host-modules
+        hostPath:
+          path: /lib/modules
+
+      - name: host-var-run-ovs
+        hostPath:
+          path: /var/run/openvswitch
+      - name: host-var-run-kubernetes
+        hostPath:
+          path: /var/run/kubernetes
+      - name: host-var-run-dbus
+        hostPath:
+          path: /var/run/dbus
+      - name: host-var-run-openshift-sdn
+        hostPath:
+          path: /var/run/openshift-sdn
+
+      - name: host-opt-cni-bin
+        hostPath:
+          path: /opt/cni/bin
+      - name: host-etc-cni-netd
+        hostPath:
+          path: /etc/cni/net.d
+      - name: host-var-lib-cni-networks-openshift-sdn
+        hostPath:
+          path: /var/lib/cni/networks/openshift-sdn

contrib/kubernetes/static/network-daemonset.yaml

@@ -0,0 +1,61 @@
+kind: DaemonSet
+apiVersion: extensions/v1beta1
+metadata:
+  name: ovs
+  annotations:
+    kubernetes.io/description: |
+      This daemon set launches the openvswitch daemon.
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        component: network
+        type: infra
+        openshift.io/role: network
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+      # Requires fairly broad permissions - ability to read all services and network functions as well
+      # as all pods.
+      serviceAccountName: sdn
+      hostNetwork: true
+      containers:
+      - name: openvswitch
+        image: openshift/openvswitch:v3.7.0-alpha.1
+        securityContext:
+          runAsUser: 0
+          privileged: true
+        volumeMounts:
+        - mountPath: /lib/modules
+          name: host-modules
+          readOnly: true
+        - mountPath: /run/openvswitch
+          name: host-run-ovs
+        - mountPath: /sys
+          name: host-sys
+          readOnly: true
+        - mountPath: /etc/openvswitch
+          name: host-config-openvswitch
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+          limits:
+            cpu: 200m
+            memory: 300Mi
+
+      volumes:
+      - name: host-modules
+        hostPath:
+          path: /lib/modules
+      - name: host-run-ovs
+        hostPath:
+          path: /run/openvswitch
+      - name: host-sys
+        hostPath:
+          path: /sys
+      - name: host-config-openvswitch
+        hostPath:
+          path: /etc/origin/openvswitch

contrib/kubernetes/static/network-ovs.yaml

@@ -0,0 +1,29 @@
+kind: List
+apiVersion: v1
+items:
+- kind: ServiceAccount
+  apiVersion: v1
+  metadata:
+    name: sdn
+    namespace: openshift-node
+- apiVersion: authorization.openshift.io/v1
+  kind: ClusterRoleBinding
+  metadata:
+    name: sdn-cluster-reader
+  roleRef:
+    name: cluster-reader
+  subjects:
+  - kind: ServiceAccount
+    name: sdn
+    namespace: openshift-node
+- apiVersion: authorization.openshift.io/v1
+  kind: ClusterRoleBinding
+  metadata:
+    name: sdn-reader
+  roleRef:
+    name: system:sdn-reader
+  subjects:
+  - kind: ServiceAccount
+    name: sdn
+    namespace: openshift-node
+# TODO: PSP binding