Skip to content

Commit 1557bb3

Browse files
deads2kdeads2k
committed
QUOTA: refactor our use of quota
1 parent 722867b commit 1557bb3

File tree

14 files changed

+240
-429
lines changed

14 files changed

+240
-429
lines changed

‎pkg/cmd/server/origin/admission/plugin_initializer.go

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,10 @@ import (
1717
imageinformer "github.com/openshift/origin/pkg/image/generated/informers/internalversion"
1818
imageclient "github.com/openshift/origin/pkg/image/generated/internalclientset"
1919
projectcache "github.com/openshift/origin/pkg/project/cache"
20-
oquota "github.com/openshift/origin/pkg/quota"
2120
"github.com/openshift/origin/pkg/quota/controller/clusterquotamapping"
2221
quotainformer "github.com/openshift/origin/pkg/quota/generated/informers/internalversion"
2322
quotaclient "github.com/openshift/origin/pkg/quota/generated/internalclientset"
23+
"github.com/openshift/origin/pkg/quota/image"
2424
securityinformer "github.com/openshift/origin/pkg/security/generated/informers/internalversion"
2525
"github.com/openshift/origin/pkg/service"
2626
templateclient "github.com/openshift/origin/pkg/template/generated/internalclientset"
@@ -42,6 +42,8 @@ import (
4242
kclientsetinternal "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
4343
kinternalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
4444
kadmission "k8s.io/kubernetes/pkg/kubeapiserver/admission"
45+
"k8s.io/kubernetes/pkg/quota/generic"
46+
"k8s.io/kubernetes/pkg/quota/install"
4547
)
4648

4749
type InformerAccess interface {
@@ -99,12 +101,15 @@ func NewPluginInitializer(
99101
return nil, nil, err
100102
}
101103

102-
quotaRegistry := oquota.NewAllResourceQuotaRegistryForAdmission(
103-
informers.GetExternalKubeInformers(),
104+
// TODO make a union registry
105+
quotaRegistry := generic.NewRegistry(install.NewQuotaConfigurationForAdmission().Evaluators())
106+
imageEvaluators := image.NewReplenishmentEvaluatorsForAdmission(
104107
informers.GetImageInformers().Image().InternalVersion().ImageStreams(),
105108
imageClient.Image(),
106-
kubeExternalClient,
107109
)
110+
for i := range imageEvaluators {
111+
quotaRegistry.Add(imageEvaluators[i])
112+
}
108113

109114
defaultRegistry := env("OPENSHIFT_DEFAULT_REGISTRY", "${DOCKER_REGISTRY_SERVICE_HOST}:${DOCKER_REGISTRY_SERVICE_PORT}")
110115
svcCache := service.NewServiceResolverCache(kubeInternalClient.Core().Services(metav1.NamespaceDefault).Get)

‎pkg/cmd/server/origin/controller/interfaces.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"github.com/golang/glog"
55

66
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
7+
"k8s.io/apimachinery/pkg/runtime/schema"
78
kexternalinformers "k8s.io/client-go/informers"
89
kclientsetinternal "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
910
kinternalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
@@ -40,9 +41,13 @@ type ControllerContext struct {
4041
QuotaInformers quotainformer.SharedInformerFactory
4142
AuthorizationInformers authorizationinformer.SharedInformerFactory
4243
SecurityInformers securityinformer.SharedInformerFactory
44+
GenericInformerFunc func(schema.GroupVersionResource) (kexternalinformers.GenericInformer, error)
4345

4446
// Stop is the stop channel
4547
Stop <-chan struct{}
48+
// InformersStarted is closed after all of the controllers have been initialized and are running. After this point it is safe,
49+
// for an individual controller to start the shared informers. Before it is closed, they should not.
50+
InformersStarted chan struct{}
4651
}
4752

4853
// OpenshiftControllerOptions contain the options used to run the controllers. Eventually we need to construct a way to properly

‎pkg/cmd/server/origin/controller/quota.go

Lines changed: 46 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -5,39 +5,44 @@ import (
55
"time"
66

77
"github.com/openshift/origin/pkg/cmd/server/bootstrappolicy"
8-
quotacontroller "github.com/openshift/origin/pkg/quota/controller"
98
"github.com/openshift/origin/pkg/quota/controller/clusterquotamapping"
109
"github.com/openshift/origin/pkg/quota/controller/clusterquotareconciliation"
10+
"github.com/openshift/origin/pkg/quota/image"
1111
"k8s.io/kubernetes/pkg/controller"
1212
kresourcequota "k8s.io/kubernetes/pkg/controller/resourcequota"
13-
14-
"github.com/openshift/origin/pkg/quota"
13+
"k8s.io/kubernetes/pkg/quota/generic"
14+
quotainstall "k8s.io/kubernetes/pkg/quota/install"
1515
)
1616

1717
func RunResourceQuotaManager(ctx ControllerContext) (bool, error) {
1818
concurrentResourceQuotaSyncs := int(ctx.OpenshiftControllerOptions.ResourceQuotaOptions.ConcurrentSyncs)
1919
resourceQuotaSyncPeriod := ctx.OpenshiftControllerOptions.ResourceQuotaOptions.SyncPeriod.Duration
2020
replenishmentSyncPeriodFunc := calculateResyncPeriod(ctx.OpenshiftControllerOptions.ResourceQuotaOptions.MinResyncPeriod.Duration)
2121
saName := "resourcequota-controller"
22+
listerFuncForResource := generic.ListerFuncForResourceFunc(ctx.GenericInformerFunc)
23+
quotaConfiguration := quotainstall.NewQuotaConfigurationForControllers(listerFuncForResource)
2224

23-
resourceQuotaRegistry := quota.NewOriginQuotaRegistry(
25+
imageEvaluators := image.NewReplenishmentEvaluators(
26+
listerFuncForResource,
2427
ctx.ImageInformers.Image().InternalVersion().ImageStreams(),
25-
ctx.ClientBuilder.OpenshiftInternalImageClientOrDie(saName).Image(),
26-
)
28+
ctx.ClientBuilder.OpenshiftInternalImageClientOrDie(saName).Image())
29+
resourceQuotaRegistry := generic.NewRegistry(imageEvaluators)
2730

2831
resourceQuotaControllerOptions := &kresourcequota.ResourceQuotaControllerOptions{
29-
QuotaClient: ctx.ClientBuilder.ClientOrDie(saName).Core(),
30-
ResourceQuotaInformer: ctx.ExternalKubeInformers.Core().V1().ResourceQuotas(),
31-
ResyncPeriod: controller.StaticResyncPeriodFunc(resourceQuotaSyncPeriod),
32-
Registry: resourceQuotaRegistry,
33-
GroupKindsToReplenish: quota.AllEvaluatedGroupKinds,
34-
ControllerFactory: quotacontroller.NewAllResourceReplenishmentControllerFactory(
35-
ctx.ExternalKubeInformers,
36-
ctx.ImageInformers.Image().InternalVersion().ImageStreams(),
37-
),
32+
QuotaClient: ctx.ClientBuilder.ClientOrDie(saName).Core(),
33+
ResourceQuotaInformer: ctx.ExternalKubeInformers.Core().V1().ResourceQuotas(),
34+
ResyncPeriod: controller.StaticResyncPeriodFunc(resourceQuotaSyncPeriod),
35+
Registry: resourceQuotaRegistry,
3836
ReplenishmentResyncPeriod: replenishmentSyncPeriodFunc,
37+
IgnoredResourcesFunc: quotaConfiguration.IgnoredResources,
38+
InformersStarted: ctx.InformersStarted,
39+
InformerFactory: ctx.ExternalKubeInformers,
40+
}
41+
controller, err := kresourcequota.NewResourceQuotaController(resourceQuotaControllerOptions)
42+
if err != nil {
43+
return true, err
3944
}
40-
go kresourcequota.NewResourceQuotaController(resourceQuotaControllerOptions).Run(concurrentResourceQuotaSyncs, ctx.Stop)
45+
go controller.Run(concurrentResourceQuotaSyncs, ctx.Stop)
4146

4247
return true, nil
4348
}
@@ -49,32 +54,42 @@ type ClusterQuotaReconciliationControllerConfig struct {
4954

5055
func (c *ClusterQuotaReconciliationControllerConfig) RunController(ctx ControllerContext) (bool, error) {
5156
saName := bootstrappolicy.InfraClusterQuotaReconciliationControllerServiceAccountName
52-
resourceQuotaRegistry := quota.NewAllResourceQuotaRegistry(
53-
ctx.ExternalKubeInformers,
54-
ctx.ImageInformers.Image().InternalVersion().ImageStreams(),
55-
ctx.ClientBuilder.OpenshiftInternalImageClientOrDie(saName).Image(),
56-
ctx.ClientBuilder.ClientOrDie(saName),
57-
)
58-
groupKindsToReplenish := quota.AllEvaluatedGroupKinds
5957

6058
clusterQuotaMappingController := clusterquotamapping.NewClusterQuotaMappingController(
6159
ctx.ExternalKubeInformers.Core().V1().Namespaces(),
6260
ctx.QuotaInformers.Quota().InternalVersion().ClusterResourceQuotas())
61+
resourceQuotaControllerClient := ctx.ClientBuilder.ClientOrDie("resourcequota-controller")
62+
discoveryFunc := resourceQuotaControllerClient.Discovery().ServerPreferredNamespacedResources
63+
listerFuncForResource := generic.ListerFuncForResourceFunc(ctx.GenericInformerFunc)
64+
quotaConfiguration := quotainstall.NewQuotaConfigurationForControllers(listerFuncForResource)
65+
66+
// TODO make a union registry
67+
resourceQuotaRegistry := generic.NewRegistry(quotaConfiguration.Evaluators())
68+
imageEvaluators := image.NewReplenishmentEvaluators(
69+
listerFuncForResource,
70+
ctx.ImageInformers.Image().InternalVersion().ImageStreams(),
71+
ctx.ClientBuilder.OpenshiftInternalImageClientOrDie(saName).Image())
72+
for i := range imageEvaluators {
73+
resourceQuotaRegistry.Add(imageEvaluators[i])
74+
}
75+
6376
options := clusterquotareconciliation.ClusterQuotaReconcilationControllerOptions{
6477
ClusterQuotaInformer: ctx.QuotaInformers.Quota().InternalVersion().ClusterResourceQuotas(),
6578
ClusterQuotaMapper: clusterQuotaMappingController.GetClusterQuotaMapper(),
6679
ClusterQuotaClient: ctx.ClientBuilder.OpenshiftInternalQuotaClientOrDie(saName).Quota().ClusterResourceQuotas(),
6780

68-
Registry: resourceQuotaRegistry,
69-
ResyncPeriod: c.DefaultResyncPeriod,
70-
ControllerFactory: quotacontroller.NewAllResourceReplenishmentControllerFactory(
71-
ctx.ExternalKubeInformers,
72-
ctx.ImageInformers.Image().InternalVersion().ImageStreams(),
73-
),
81+
Registry: resourceQuotaRegistry,
82+
ResyncPeriod: c.DefaultResyncPeriod,
7483
ReplenishmentResyncPeriod: controller.StaticResyncPeriodFunc(c.DefaultReplenishmentSyncPeriod),
75-
GroupKindsToReplenish: groupKindsToReplenish,
84+
DiscoveryFunc: discoveryFunc,
85+
IgnoredResourcesFunc: quotaConfiguration.IgnoredResources,
86+
InformersStarted: ctx.InformersStarted,
87+
InformerFactory: ctx.ExternalKubeInformers,
88+
}
89+
clusterQuotaReconciliationController, err := clusterquotareconciliation.NewClusterQuotaReconcilationController(options)
90+
if err != nil {
91+
return true, err
7692
}
77-
clusterQuotaReconciliationController := clusterquotareconciliation.NewClusterQuotaReconcilationController(options)
7893
clusterQuotaMappingController.GetClusterQuotaMapper().AddListener(clusterQuotaReconciliationController)
7994

8095
go clusterQuotaMappingController.Run(5, ctx.Stop)

‎pkg/cmd/server/start/controllers.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ func newControllerContext(
2828
kubeExternal kclientsetexternal.Interface,
2929
informers *informers,
3030
stopCh <-chan struct{},
31+
informersStarted chan struct{},
3132
) origincontrollers.ControllerContext {
3233

3334
// divide up the QPS since it re-used separately for every client
@@ -51,15 +52,17 @@ func newControllerContext(
5152
},
5253
},
5354
InternalKubeInformers: informers.internalKubeInformers,
54-
ExternalKubeInformers: informers.externalKubeInformers,
55+
ExternalKubeInformers: newGenericInformers(informers),
5556
AppInformers: informers.appInformers,
5657
AuthorizationInformers: informers.authorizationInformers,
5758
BuildInformers: informers.buildInformers,
5859
ImageInformers: informers.imageInformers,
5960
QuotaInformers: informers.quotaInformers,
6061
SecurityInformers: informers.securityInformers,
6162
TemplateInformers: informers.templateInformers,
63+
GenericInformerFunc: newGenericInformers(informers).ForResource,
6264
Stop: stopCh,
65+
InformersStarted: informersStarted,
6366
}
6467

6568
return openshiftControllerContext

‎pkg/cmd/server/start/start_master.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -392,6 +392,7 @@ func (m *Master) Start() error {
392392
if err != nil {
393393
return err
394394
}
395+
395396
_, config, err := configapi.GetExternalKubeClient(m.config.MasterClients.OpenShiftLoopbackKubeConfig, m.config.MasterClients.OpenShiftLoopbackClientConnectionOverrides)
396397
if err != nil {
397398
return err
@@ -489,12 +490,15 @@ func (m *Master) Start() error {
489490
if err != nil {
490491
glog.Fatal(err)
491492
}
492-
controllerContext := newControllerContext(openshiftControllerOptions, privilegedLoopbackConfig, kubeExternal, informers, utilwait.NeverStop)
493+
494+
controllerContext := newControllerContext(openshiftControllerOptions, privilegedLoopbackConfig, kubeExternal, informers, utilwait.NeverStop, make(chan struct{}))
493495
if err := startControllers(*m.config, allocationController, controllerContext); err != nil {
494496
glog.Fatal(err)
495497
}
496498

497499
informers.Start(utilwait.NeverStop)
500+
close(controllerContext.InformersStarted)
501+
498502
}()
499503
}
500504

‎pkg/quota/admission/clusterresourcequota/admission.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
kinternalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
1414
kcorelisters "k8s.io/kubernetes/pkg/client/listers/core/internalversion"
1515
"k8s.io/kubernetes/pkg/quota"
16+
"k8s.io/kubernetes/pkg/quota/install"
1617
"k8s.io/kubernetes/plugin/pkg/admission/resourcequota"
1718
resourcequotaapi "k8s.io/kubernetes/plugin/pkg/admission/resourcequota/apis/resourcequota"
1819

@@ -88,7 +89,7 @@ func (q *clusterQuotaAdmission) Admit(a admission.Attributes) (err error) {
8889

8990
q.init.Do(func() {
9091
clusterQuotaAccessor := newQuotaAccessor(q.clusterQuotaLister, q.namespaceLister, q.clusterQuotaClient, q.clusterQuotaMapper)
91-
q.evaluator = resourcequota.NewQuotaEvaluator(clusterQuotaAccessor, q.registry, q.lockAquisition, &resourcequotaapi.Configuration{}, numEvaluatorThreads, utilwait.NeverStop)
92+
q.evaluator = resourcequota.NewQuotaEvaluator(clusterQuotaAccessor, install.DefaultIgnoredResources(), q.registry, q.lockAquisition, &resourcequotaapi.Configuration{}, numEvaluatorThreads, utilwait.NeverStop)
9293
})
9394

9495
return q.evaluator.Evaluate(a)

0 commit comments

Comments
 (0)