Skip to content

Commit 32695af

Browse files
deads2kbertinatto
deads2k
authored and
bertinatto
committed
UPSTREAM: <carry>: kube-controller-manager: allow running bare kube-controller-manager
UPSTREAM: <carry>: (squash) kube-controller-manager: allow running bare kube-controller-manager UPSTREAM: <carry>: kube-controller-manager: allow running bare kube-controller-manager UPSTREAM: <carry>: (squash) remove egressnetworkpolicies from gc ignored resources egressnetworkpolicies should not be in garbage collector ignored resources, so users can delete them using "--cascade=foreground" flag. Signed-off-by: Flavio Fernandes <[email protected]> OpenShift-Rebase-Source: 6c1dee4 UPSTREAM: <carry>: (squash) kube-controller-manager: allow running bare kube-controller-manager UPSTREAM: <carry>: kube-controller-manager: allow running bare kube-controller-manager UPSTREAM: <carry>: kube-controller-manager: allow running bare kube-controller-manager Fix garbage-collection for CRDs. These types are backed by a CRD and not by openshift-apiserver anymore. DefaultGarbageCollectionPolicy (Unsupported) does not work with CRDs. The `foregroundDeletion` finalizer was set on these CRD objects which blocks deletion indifinetelly as GC will ignore these resources.
1 parent 0ae8123 commit 32695af

13 files changed

+852
-12
lines changed

‎cmd/kube-controller-manager/app/apps.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,11 @@ func newDaemonSetControllerDescriptor() *ControllerDescriptor {
4141
}
4242
}
4343
func startDaemonSetController(ctx context.Context, controllerContext ControllerContext, controllerName string) (controller.Interface, bool, error) {
44-
dsc, err := daemon.NewDaemonSetsController(
44+
dsc, err := daemon.NewNodeSelectorAwareDaemonSetsController(
4545
ctx,
46+
controllerContext.OpenShiftContext.OpenShiftDefaultProjectNodeSelector,
47+
controllerContext.OpenShiftContext.KubeDefaultProjectNodeSelector,
48+
controllerContext.InformerFactory.Core().V1().Namespaces(),
4649
controllerContext.InformerFactory.Apps().V1().DaemonSets(),
4750
controllerContext.InformerFactory.Apps().V1().ControllerRevisions(),
4851
controllerContext.InformerFactory.Core().V1().Pods(),

‎cmd/kube-controller-manager/app/config/config.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,8 @@ import (
2929
// Config is the main context object for the controller manager.
3030
type Config struct {
3131
// Flagz is the Reader interface to get flags for the flagz page.
32-
Flagz flagz.Reader
32+
Flagz flagz.Reader
33+
OpenShiftContext OpenShiftContext
3334

3435
ComponentConfig kubectrlmgrconfig.KubeControllerManagerConfiguration
3536

‎cmd/kube-controller-manager/app/config/patch.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
package config
2+
3+
// OpenShiftContext is additional context that we need to launch the kube-controller-manager for openshift.
4+
// Basically, this holds our additional config information.
5+
type OpenShiftContext struct {
6+
OpenShiftConfig string
7+
OpenShiftDefaultProjectNodeSelector string
8+
KubeDefaultProjectNodeSelector string
9+
}

‎cmd/kube-controller-manager/app/controllermanager.go

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,11 @@ controller, and serviceaccounts controller.`,
140140
return err
141141
}
142142

143+
if err := ShimForOpenShift(s, c); err != nil {
144+
fmt.Fprintf(os.Stderr, "%v\n", err)
145+
return err
146+
}
147+
143148
// add feature enablement metrics
144149
fg := s.ComponentGlobalsRegistry.FeatureGateFor(basecompatibility.DefaultKubeComponent)
145150
fg.(featuregate.MutableFeatureGate).AddMetrics()
@@ -376,6 +381,8 @@ func Run(ctx context.Context, c *config.CompletedConfig) error {
376381

377382
// ControllerContext defines the context object for controller
378383
type ControllerContext struct {
384+
OpenShiftContext config.OpenShiftContext
385+
379386
// ClientBuilder will provide a client for this controller to use
380387
ClientBuilder clientbuilder.ControllerClientBuilder
381388

@@ -620,7 +627,12 @@ func CreateControllerContext(ctx context.Context, s *config.CompletedConfig, roo
620627
}
621628

622629
versionedClient := rootClientBuilder.ClientOrDie("shared-informers")
623-
sharedInformers := informers.NewSharedInformerFactoryWithOptions(versionedClient, ResyncPeriod(s)(), informers.WithTransform(trim))
630+
var sharedInformers informers.SharedInformerFactory
631+
if InformerFactoryOverride == nil {
632+
sharedInformers = informers.NewSharedInformerFactoryWithOptions(versionedClient, ResyncPeriod(s)(), informers.WithTransform(trim))
633+
} else {
634+
sharedInformers = InformerFactoryOverride
635+
}
624636

625637
metadataClient := metadata.NewForConfigOrDie(rootClientBuilder.ConfigOrDie("metadata-informers"))
626638
metadataInformers := metadatainformer.NewSharedInformerFactoryWithOptions(metadataClient, ResyncPeriod(s)(), metadatainformer.WithTransform(trim))
@@ -640,6 +652,7 @@ func CreateControllerContext(ctx context.Context, s *config.CompletedConfig, roo
640652
}, 30*time.Second, ctx.Done())
641653

642654
controllerContext := ControllerContext{
655+
OpenShiftContext: s.OpenShiftContext,
643656
ClientBuilder: clientBuilder,
644657
InformerFactory: sharedInformers,
645658
ObjectOrMetadataInformerFactory: informerfactory.NewInformerFactory(sharedInformers, metadataInformers),
@@ -822,10 +835,10 @@ func startServiceAccountTokenController(ctx context.Context, controllerContext C
822835
controllerContext.InformerFactory.Core().V1().ServiceAccounts(),
823836
controllerContext.InformerFactory.Core().V1().Secrets(),
824837
rootClientBuilder.ClientOrDie("tokens-controller"),
825-
serviceaccountcontroller.TokensControllerOptions{
838+
applyOpenShiftServiceServingCertCA(serviceaccountcontroller.TokensControllerOptions{
826839
TokenGenerator: tokenGenerator,
827840
RootCA: rootCA,
828-
},
841+
}),
829842
)
830843
if err != nil {
831844
return nil, true, fmt.Errorf("error creating Tokens controller: %v", err)

‎cmd/kube-controller-manager/app/options/options.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,8 @@ type KubeControllerManagerOptions struct {
109109

110110
// Parsedflags holds the parsed CLI flags.
111111
ParsedFlags *cliflag.NamedFlagSets
112+
113+
OpenShiftContext kubecontrollerconfig.OpenShiftContext
112114
}
113115

114116
// NewKubeControllerManagerOptions creates a new KubeControllerManagerOptions with a default config.
@@ -295,6 +297,8 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy
295297
fs.StringVar(&s.Generic.ClientConnection.Kubeconfig, "kubeconfig", s.Generic.ClientConnection.Kubeconfig, "Path to kubeconfig file with authorization and master location information (the master location can be overridden by the master flag).")
296298

297299
s.ComponentGlobalsRegistry.AddFlags(fss.FlagSet("generic"))
300+
fs.StringVar(&s.OpenShiftContext.OpenShiftConfig, "openshift-config", s.OpenShiftContext.OpenShiftConfig, "indicates that this process should be compatible with openshift start master")
301+
fs.MarkHidden("openshift-config")
298302

299303
return fss
300304
}
@@ -402,6 +406,9 @@ func (s *KubeControllerManagerOptions) ApplyTo(c *kubecontrollerconfig.Config, a
402406
return err
403407
}
404408
}
409+
410+
c.OpenShiftContext = s.OpenShiftContext
411+
405412
return nil
406413
}
407414

‎cmd/kube-controller-manager/app/patch.go

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
package app
2+
3+
import (
4+
"io/ioutil"
5+
"path"
6+
7+
"k8s.io/apimachinery/pkg/util/json"
8+
kyaml "k8s.io/apimachinery/pkg/util/yaml"
9+
"k8s.io/client-go/informers"
10+
"k8s.io/kubernetes/cmd/kube-controller-manager/app/config"
11+
"k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
12+
)
13+
14+
var InformerFactoryOverride informers.SharedInformerFactory
15+
16+
func ShimForOpenShift(controllerManagerOptions *options.KubeControllerManagerOptions, controllerManager *config.Config) error {
17+
if len(controllerManager.OpenShiftContext.OpenShiftConfig) == 0 {
18+
return nil
19+
}
20+
21+
// TODO this gets removed when no longer take flags and no longer build a recycler template
22+
openshiftConfig, err := getOpenShiftConfig(controllerManager.OpenShiftContext.OpenShiftConfig)
23+
if err != nil {
24+
return err
25+
}
26+
27+
// TODO this should be replaced by using a flex volume to inject service serving cert CAs into pods instead of adding it to the sa token
28+
if err := applyOpenShiftServiceServingCertCAFunc(path.Dir(controllerManager.OpenShiftContext.OpenShiftConfig), openshiftConfig); err != nil {
29+
return err
30+
}
31+
32+
// skip GC on some openshift resources
33+
// TODO this should be replaced by discovery information in some way
34+
if err := applyOpenShiftGCConfig(controllerManager); err != nil {
35+
return err
36+
}
37+
38+
if err := applyOpenShiftConfigDefaultProjectSelector(controllerManagerOptions, openshiftConfig); err != nil {
39+
return err
40+
}
41+
42+
// Overwrite the informers, because we have our custom generic informers for quota.
43+
// TODO update quota to create its own informer like garbage collection
44+
if informers, err := newInformerFactory(controllerManager.Kubeconfig); err != nil {
45+
return err
46+
} else {
47+
InformerFactoryOverride = informers
48+
}
49+
50+
return nil
51+
}
52+
53+
func getOpenShiftConfig(configFile string) (map[string]interface{}, error) {
54+
configBytes, err := ioutil.ReadFile(configFile)
55+
if err != nil {
56+
return nil, err
57+
}
58+
jsonBytes, err := kyaml.ToJSON(configBytes)
59+
if err != nil {
60+
return nil, err
61+
}
62+
config := map[string]interface{}{}
63+
if err := json.Unmarshal(jsonBytes, &config); err != nil {
64+
return nil, err
65+
}
66+
67+
return config, nil
68+
}
69+
70+
func applyOpenShiftConfigDefaultProjectSelector(controllerManagerOptions *options.KubeControllerManagerOptions, openshiftConfig map[string]interface{}) error {
71+
projectConfig, ok := openshiftConfig["projectConfig"]
72+
if !ok {
73+
return nil
74+
}
75+
76+
castProjectConfig := projectConfig.(map[string]interface{})
77+
defaultNodeSelector, ok := castProjectConfig["defaultNodeSelector"]
78+
if !ok {
79+
return nil
80+
}
81+
controllerManagerOptions.OpenShiftContext.OpenShiftDefaultProjectNodeSelector = defaultNodeSelector.(string)
82+
83+
return nil
84+
}

‎cmd/kube-controller-manager/app/patch_gc.go

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
package app
2+
3+
import (
4+
gcconfig "k8s.io/kubernetes/pkg/controller/garbagecollector/config"
5+
6+
"k8s.io/kubernetes/cmd/kube-controller-manager/app/config"
7+
)
8+
9+
func applyOpenShiftGCConfig(controllerManager *config.Config) error {
10+
// TODO make this configurable or discoverable. This is going to prevent us from running the stock GC controller
11+
// IF YOU ADD ANYTHING TO THIS LIST, MAKE SURE THAT YOU UPDATE THEIR STRATEGIES TO PREVENT GC FINALIZERS
12+
//
13+
// DO NOT PUT CRDs into the list. apiexstension-apiserver does not implement GarbageCollectionPolicy
14+
// so the deletion of these will be blocked because of foregroundDeletion finalizer when foreground deletion strategy is specified.
15+
controllerManager.ComponentConfig.GarbageCollectorController.GCIgnoredResources = append(controllerManager.ComponentConfig.GarbageCollectorController.GCIgnoredResources,
16+
// explicitly disabled from GC for now - not enough value to track them
17+
gcconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthclientauthorizations"},
18+
gcconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthclients"},
19+
gcconfig.GroupResource{Group: "user.openshift.io", Resource: "groups"},
20+
gcconfig.GroupResource{Group: "user.openshift.io", Resource: "identities"},
21+
gcconfig.GroupResource{Group: "user.openshift.io", Resource: "users"},
22+
gcconfig.GroupResource{Group: "image.openshift.io", Resource: "images"},
23+
24+
// virtual resource
25+
gcconfig.GroupResource{Group: "project.openshift.io", Resource: "projects"},
26+
// virtual and unwatchable resource, surfaced via rbac.authorization.k8s.io objects
27+
gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "clusterroles"},
28+
gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "clusterrolebindings"},
29+
gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "roles"},
30+
gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "rolebindings"},
31+
// these resources contain security information in their names, and we don't need to track them
32+
gcconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthaccesstokens"},
33+
gcconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthauthorizetokens"},
34+
)
35+
36+
return nil
37+
}

0 commit comments

Comments
 (0)