Skip to content

Commit 7e4268c

Browse files
libeszlibesz
committed
Router publish strategy related changes for IBM Cloud platform
When HCP contains router publish strategy for the master services, IBM Cloud platform implementation will take care of the proper exposure of the services, without using the actual cluster Router (similar implementation to Azure). That is, CPO should not create any LoadBalancer or actual router deployment as part of reconciling the HCP. The current change also makes the migration from NodePort services backward compatible, for existing clusters. That is, the NodePort services will remain as is (they are not converted to regular ClusterIP services, resulting in permanently losing the reserved nodeports) allowing existing external clients (e.g. kubelet, master proxy) to work as before, until they are also upgraded. The expected behavior is to: * Do not manage any ingress component (LB Svc, Router) * Keep NodePort master services (if they are already existing) * (Re)configure node agents (e.g. connectivity) to use 443 as server port
1 parent ab02c94 commit 7e4268c

File tree

6 files changed

+84
-24
lines changed

6 files changed

+84
-24
lines changed

‎control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1742,7 +1742,7 @@ func (r *HostedControlPlaneReconciler) reconcileOAuthServerService(ctx context.C
17421742
p := oauth.NewOAuthServiceParams(hcp)
17431743
oauthServerService := manifests.OauthServerService(hcp.Namespace)
17441744
if _, err := createOrUpdate(ctx, r.Client, oauthServerService, func() error {
1745-
return oauth.ReconcileService(oauthServerService, p.OwnerRef, serviceStrategy)
1745+
return oauth.ReconcileService(oauthServerService, p.OwnerRef, serviceStrategy, hcp.Spec.Platform.Type)
17461746
}); err != nil {
17471747
return fmt.Errorf("failed to reconcile OAuth service: %w", err)
17481748
}
@@ -1836,7 +1836,7 @@ func (r *HostedControlPlaneReconciler) reconcileOLMPackageServerService(ctx cont
18361836
}
18371837

18381838
func (r *HostedControlPlaneReconciler) reconcileHCPRouterServices(ctx context.Context, hcp *hyperv1.HostedControlPlane, createOrUpdate upsert.CreateOrUpdateFN) error {
1839-
if sharedingress.UseSharedIngress() {
1839+
if sharedingress.UseSharedIngress() || hcp.Spec.Platform.Type == hyperv1.IBMCloudPlatform {
18401840
return nil
18411841
}
18421842
exposeKASThroughRouter := util.IsRouteKAS(hcp)
@@ -1969,14 +1969,14 @@ func (r *HostedControlPlaneReconciler) defaultReconcileInfrastructureStatus(ctx
19691969
}
19701970

19711971
func (r *HostedControlPlaneReconciler) reconcileInternalRouterServiceStatus(ctx context.Context, hcp *hyperv1.HostedControlPlane) (host string, needed bool, message string, err error) {
1972-
if !util.IsPrivateHCP(hcp) {
1972+
if !util.IsPrivateHCP(hcp) || hcp.Spec.Platform.Type == hyperv1.IBMCloudPlatform {
19731973
return
19741974
}
19751975
return r.reconcileRouterServiceStatus(ctx, manifests.PrivateRouterService(hcp.Namespace), events.NewMessageCollector(ctx, r.Client))
19761976
}
19771977

19781978
func (r *HostedControlPlaneReconciler) reconcileExternalRouterServiceStatus(ctx context.Context, hcp *hyperv1.HostedControlPlane) (host string, needed bool, message string, err error) {
1979-
if !util.IsPublicHCP(hcp) || !util.IsRouteKAS(hcp) || sharedingress.UseSharedIngress() {
1979+
if !util.IsPublicHCP(hcp) || !util.IsRouteKAS(hcp) || sharedingress.UseSharedIngress() || hcp.Spec.Platform.Type == hyperv1.IBMCloudPlatform {
19801980
return
19811981
}
19821982
return r.reconcileRouterServiceStatus(ctx, manifests.RouterPublicService(hcp.Namespace), events.NewMessageCollector(ctx, r.Client))
@@ -2010,12 +2010,12 @@ func (r *HostedControlPlaneReconciler) reconcileAPIServerServiceStatus(ctx conte
20102010
return "", 0, "", errors.New("APIServer service strategy not specified")
20112011
}
20122012

2013-
if sharedingress.UseSharedIngress() {
2013+
if sharedingress.UseSharedIngress() || hcp.Spec.Platform.Type == hyperv1.IBMCloudPlatform {
20142014
return sharedingress.Hostname(hcp), sharedingress.ExternalDNSLBPort, "", nil
20152015
}
20162016

20172017
var svc *corev1.Service
2018-
if serviceStrategy.Type == hyperv1.Route {
2018+
if serviceStrategy.Type == hyperv1.Route && hcp.Spec.Platform.Type != hyperv1.IBMCloudPlatform {
20192019
if util.IsPublicHCP(hcp) {
20202020
svc = manifests.RouterPublicService(hcp.Namespace)
20212021
} else {
@@ -4336,6 +4336,10 @@ func (r *HostedControlPlaneReconciler) reconcileCoreIgnitionConfig(ctx context.C
43364336
}
43374337

43384338
func (r *HostedControlPlaneReconciler) reconcileRouter(ctx context.Context, hcp *hyperv1.HostedControlPlane, releaseImageProvider imageprovider.ReleaseImageProvider, createOrUpdate upsert.CreateOrUpdateFN) error {
4339+
if hcp.Spec.Platform.Type == hyperv1.IBMCloudPlatform {
4340+
return nil
4341+
}
4342+
43394343
routeList := &routev1.RouteList{}
43404344
if err := r.List(ctx, routeList, client.InNamespace(hcp.Namespace)); err != nil {
43414345
return fmt.Errorf("failed to list routes: %w", err)

‎control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller_test.go

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1072,13 +1072,21 @@ func TestReconcileRouter(t *testing.T) {
10721072

10731073
testCases := []struct {
10741074
name string
1075+
platformType hyperv1.PlatformType
10751076
endpointAccess hyperv1.AWSEndpointAccessType
10761077
exposeAPIServerThroughRouter bool
10771078
existingObjects []client.Object
10781079
expectedDeployments []appsv1.Deployment
10791080
}{
1081+
{
1082+
name: "IBM Cloud",
1083+
platformType: hyperv1.IBMCloudPlatform,
1084+
endpointAccess: hyperv1.Public,
1085+
exposeAPIServerThroughRouter: true,
1086+
},
10801087
{
10811088
name: "Public HCP, uses public service host name",
1089+
platformType: hyperv1.AWSPlatform,
10821090
endpointAccess: hyperv1.Public,
10831091
exposeAPIServerThroughRouter: true,
10841092
existingObjects: []client.Object{},
@@ -1104,6 +1112,7 @@ func TestReconcileRouter(t *testing.T) {
11041112
},
11051113
{
11061114
name: "PublicPrivate HCP, deployment gets hostname from public service",
1115+
platformType: hyperv1.AWSPlatform,
11071116
endpointAccess: hyperv1.PublicAndPrivate,
11081117
exposeAPIServerThroughRouter: true,
11091118
existingObjects: []client.Object{},
@@ -1130,6 +1139,7 @@ func TestReconcileRouter(t *testing.T) {
11301139

11311140
{
11321141
name: "Private HCP, deployment gets hostname from private service",
1142+
platformType: hyperv1.AWSPlatform,
11331143
endpointAccess: hyperv1.Private,
11341144
exposeAPIServerThroughRouter: true,
11351145
existingObjects: []client.Object{},
@@ -1155,11 +1165,13 @@ func TestReconcileRouter(t *testing.T) {
11551165
},
11561166
{
11571167
name: "Public HCP apiserver not exposed through router, nothing gets created",
1168+
platformType: hyperv1.AWSPlatform,
11581169
endpointAccess: hyperv1.Public,
11591170
exposeAPIServerThroughRouter: false,
11601171
},
11611172
{
11621173
name: "PublicPrivate HCP apiserver not exposed through router, router without custom template and private router service get created",
1174+
platformType: hyperv1.AWSPlatform,
11631175
endpointAccess: hyperv1.PublicAndPrivate,
11641176
exposeAPIServerThroughRouter: false,
11651177
expectedDeployments: []appsv1.Deployment{
@@ -1184,6 +1196,7 @@ func TestReconcileRouter(t *testing.T) {
11841196
},
11851197
{
11861198
name: "Private HCP apiserver not exposed through router, router without custom template and private router service get created",
1199+
platformType: hyperv1.AWSPlatform,
11871200
endpointAccess: hyperv1.Private,
11881201
exposeAPIServerThroughRouter: false,
11891202
expectedDeployments: []appsv1.Deployment{
@@ -1208,6 +1221,7 @@ func TestReconcileRouter(t *testing.T) {
12081221
},
12091222
{
12101223
name: "Old router resources get cleaned up when exposed through route",
1224+
platformType: hyperv1.AWSPlatform,
12111225
endpointAccess: hyperv1.PublicAndPrivate,
12121226
exposeAPIServerThroughRouter: true,
12131227
existingObjects: []client.Object{
@@ -1238,6 +1252,7 @@ func TestReconcileRouter(t *testing.T) {
12381252
},
12391253
{
12401254
name: "Old router resources get cleaned up when exposed through LB",
1255+
platformType: hyperv1.AWSPlatform,
12411256
endpointAccess: hyperv1.PublicAndPrivate,
12421257
exposeAPIServerThroughRouter: false,
12431258
existingObjects: []client.Object{
@@ -1287,7 +1302,7 @@ func TestReconcileRouter(t *testing.T) {
12871302
},
12881303
Spec: hyperv1.HostedControlPlaneSpec{
12891304
Platform: hyperv1.PlatformSpec{
1290-
Type: hyperv1.AWSPlatform,
1305+
Type: tc.platformType,
12911306
AWS: &hyperv1.AWSPlatformSpec{
12921307
EndpointAccess: tc.endpointAccess,
12931308
},

‎control-plane-operator/controllers/hostedcontrolplane/ignitionserver/ignitionserver.go

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -67,14 +67,14 @@ func ReconcileIgnitionServer(ctx context.Context,
6767
}
6868
ignitionServerProxyService := ignitionserver.ProxyService(controlPlaneNamespace)
6969
if _, err := createOrUpdate(ctx, c, ignitionServerProxyService, func() error {
70-
return reconcileIgnitionServerProxyService(ignitionServerProxyService, serviceStrategy)
70+
return reconcileIgnitionServerProxyService(ignitionServerProxyService, serviceStrategy, hcp.Spec.Platform.Type)
7171
}); err != nil {
7272
return fmt.Errorf("failed to reconcile ignition proxy service: %w", err)
7373
}
7474
routeServiceName = ignitionServerProxyService.Name
7575
} else {
7676
if _, err := createOrUpdate(ctx, c, ignitionServerService, func() error {
77-
return reconcileIgnitionServerService(ignitionServerService, serviceStrategy)
77+
return reconcileIgnitionServerService(ignitionServerService, serviceStrategy, hcp.Spec.Platform.Type)
7878
}); err != nil {
7979
return fmt.Errorf("failed to reconcile ignition service: %w", err)
8080
}
@@ -291,15 +291,15 @@ func ReconcileIgnitionServer(ctx context.Context,
291291
return nil
292292
}
293293

294-
func reconcileIgnitionServerService(svc *corev1.Service, strategy *hyperv1.ServicePublishingStrategy) error {
295-
return reconcileIgnitionExternalService(svc, strategy, false)
294+
func reconcileIgnitionServerService(svc *corev1.Service, strategy *hyperv1.ServicePublishingStrategy, platformType hyperv1.PlatformType) error {
295+
return reconcileIgnitionExternalService(svc, strategy, false, platformType)
296296
}
297297

298-
func reconcileIgnitionServerProxyService(svc *corev1.Service, strategy *hyperv1.ServicePublishingStrategy) error {
299-
return reconcileIgnitionExternalService(svc, strategy, true)
298+
func reconcileIgnitionServerProxyService(svc *corev1.Service, strategy *hyperv1.ServicePublishingStrategy, platformType hyperv1.PlatformType) error {
299+
return reconcileIgnitionExternalService(svc, strategy, true, platformType)
300300
}
301301

302-
func reconcileIgnitionExternalService(svc *corev1.Service, strategy *hyperv1.ServicePublishingStrategy, isProxy bool) error {
302+
func reconcileIgnitionExternalService(svc *corev1.Service, strategy *hyperv1.ServicePublishingStrategy, isProxy bool, platformType hyperv1.PlatformType) error {
303303
appLabel := ignitionserver.ResourceName
304304
targetPort := intstr.FromInt(9090)
305305
if isProxy {
@@ -327,7 +327,9 @@ func reconcileIgnitionExternalService(svc *corev1.Service, strategy *hyperv1.Ser
327327
portSpec.NodePort = strategy.NodePort.Port
328328
}
329329
case hyperv1.Route:
330-
svc.Spec.Type = corev1.ServiceTypeClusterIP
330+
if ((platformType == hyperv1.IBMCloudPlatform) && (svc.Spec.Type != corev1.ServiceTypeNodePort)) || (platformType != hyperv1.IBMCloudPlatform) {
331+
svc.Spec.Type = corev1.ServiceTypeClusterIP
332+
}
331333
default:
332334
return fmt.Errorf("invalid publishing strategy for Ignition service: %s", strategy.Type)
333335
}

‎control-plane-operator/controllers/hostedcontrolplane/ignitionserver/ignitionserver_test.go

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,18 +13,21 @@ import (
1313
func TestReconcileIgnitionServerServiceNodePortFreshInitialization(t *testing.T) {
1414
tests := []struct {
1515
name string
16+
platformType hyperv1.PlatformType
1617
inputIgnitionServerService *corev1.Service
1718
inputServicePublishingStrategy *hyperv1.ServicePublishingStrategy
1819
}{
1920
{
2021
name: "fresh service initialization",
22+
platformType: hyperv1.AWSPlatform,
2123
inputIgnitionServerService: ignitionserver.Service("default"),
2224
inputServicePublishingStrategy: &hyperv1.ServicePublishingStrategy{
2325
Type: hyperv1.NodePort,
2426
},
2527
},
2628
{
2729
name: "fresh service with node port specified",
30+
platformType: hyperv1.AWSPlatform,
2831
inputIgnitionServerService: ignitionserver.Service("default"),
2932
inputServicePublishingStrategy: &hyperv1.ServicePublishingStrategy{
3033
Type: hyperv1.NodePort,
@@ -36,7 +39,7 @@ func TestReconcileIgnitionServerServiceNodePortFreshInitialization(t *testing.T)
3639
}
3740
for _, test := range tests {
3841
t.Run(test.name, func(t *testing.T) {
39-
reconcileIgnitionServerService(test.inputIgnitionServerService, test.inputServicePublishingStrategy)
42+
reconcileIgnitionServerService(test.inputIgnitionServerService, test.inputServicePublishingStrategy, test.platformType)
4043
g := NewGomegaWithT(t)
4144
g.Expect(len(test.inputIgnitionServerService.Spec.Ports)).To(Equal(1))
4245
g.Expect(test.inputIgnitionServerService.Spec.Type).To(Equal(corev1.ServiceTypeNodePort))
@@ -54,11 +57,13 @@ func TestReconcileIgnitionServerServiceNodePortFreshInitialization(t *testing.T)
5457
func TestReconcileIgnitionServerServiceNodePortExistingService(t *testing.T) {
5558
tests := []struct {
5659
name string
60+
platformType hyperv1.PlatformType
5761
inputIgnitionServerService *corev1.Service
5862
inputServicePublishingStrategy *hyperv1.ServicePublishingStrategy
5963
}{
6064
{
61-
name: "existing service keeps nodeport",
65+
name: "existing service keeps nodeport",
66+
platformType: hyperv1.AWSPlatform,
6267
inputIgnitionServerService: &corev1.Service{
6368
ObjectMeta: ignitionserver.Service("default").ObjectMeta,
6469
Spec: corev1.ServiceSpec{
@@ -81,7 +86,7 @@ func TestReconcileIgnitionServerServiceNodePortExistingService(t *testing.T) {
8186
for _, test := range tests {
8287
t.Run(test.name, func(t *testing.T) {
8388
initialNodePort := test.inputIgnitionServerService.Spec.Ports[0].NodePort
84-
reconcileIgnitionServerService(test.inputIgnitionServerService, test.inputServicePublishingStrategy)
89+
reconcileIgnitionServerService(test.inputIgnitionServerService, test.inputServicePublishingStrategy, test.platformType)
8590
g := NewGomegaWithT(t)
8691
g.Expect(len(test.inputIgnitionServerService.Spec.Ports)).To(Equal(1))
8792
g.Expect(test.inputIgnitionServerService.Spec.Type).To(Equal(corev1.ServiceTypeNodePort))
@@ -97,18 +102,21 @@ func TestReconcileIgnitionServerServiceNodePortExistingService(t *testing.T) {
97102
func TestReconcileIgnitionServerServiceRoute(t *testing.T) {
98103
tests := []struct {
99104
name string
105+
platformType hyperv1.PlatformType
100106
inputIgnitionServerService *corev1.Service
101107
inputServicePublishingStrategy *hyperv1.ServicePublishingStrategy
102108
}{
103109
{
104110
name: "fresh service initialization",
111+
platformType: hyperv1.AWSPlatform,
105112
inputIgnitionServerService: ignitionserver.Service("default"),
106113
inputServicePublishingStrategy: &hyperv1.ServicePublishingStrategy{
107114
Type: hyperv1.Route,
108115
},
109116
},
110117
{
111-
name: "existing service",
118+
name: "existing service",
119+
platformType: hyperv1.AWSPlatform,
112120
inputIgnitionServerService: &corev1.Service{
113121
ObjectMeta: ignitionserver.Service("default").ObjectMeta,
114122
Spec: corev1.ServiceSpec{
@@ -126,17 +134,42 @@ func TestReconcileIgnitionServerServiceRoute(t *testing.T) {
126134
Type: hyperv1.Route,
127135
},
128136
},
137+
{
138+
name: "existing service, IBM Cloud",
139+
platformType: hyperv1.AWSPlatform,
140+
inputIgnitionServerService: &corev1.Service{
141+
ObjectMeta: ignitionserver.Service("default").ObjectMeta,
142+
Spec: corev1.ServiceSpec{
143+
Type: corev1.ServiceTypeNodePort,
144+
Ports: []corev1.ServicePort{
145+
{
146+
Name: "https",
147+
Port: 9090,
148+
TargetPort: intstr.FromInt(9090),
149+
Protocol: corev1.ProtocolTCP,
150+
NodePort: 30000,
151+
},
152+
},
153+
},
154+
},
155+
inputServicePublishingStrategy: &hyperv1.ServicePublishingStrategy{
156+
Type: hyperv1.Route,
157+
},
158+
},
129159
}
130160
for _, test := range tests {
131161
t.Run(test.name, func(t *testing.T) {
132-
reconcileIgnitionServerService(test.inputIgnitionServerService, test.inputServicePublishingStrategy)
162+
reconcileIgnitionServerService(test.inputIgnitionServerService, test.inputServicePublishingStrategy, test.platformType)
133163
g := NewGomegaWithT(t)
134164
g.Expect(len(test.inputIgnitionServerService.Spec.Ports)).To(Equal(1))
135165
g.Expect(test.inputIgnitionServerService.Spec.Type).To(Equal(corev1.ServiceTypeClusterIP))
136166
g.Expect(test.inputIgnitionServerService.Spec.Ports[0].TargetPort).To(Equal(intstr.FromInt(9090)))
137167
g.Expect(test.inputIgnitionServerService.Spec.Ports[0].Port).To(Equal(int32(443)))
138168
g.Expect(test.inputIgnitionServerService.Spec.Ports[0].Name).To(Equal("https"))
139169
g.Expect(test.inputIgnitionServerService.Spec.Ports[0].Protocol).To(Equal(corev1.ProtocolTCP))
170+
if test.platformType == hyperv1.IBMCloudPlatform {
171+
g.Expect(test.inputIgnitionServerService.Spec.Type).To(Equal(corev1.ServiceTypeNodePort))
172+
}
140173
})
141174
}
142175
}

‎control-plane-operator/controllers/hostedcontrolplane/kas/service.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,9 @@ func ReconcileService(svc *corev1.Service, strategy *hyperv1.ServicePublishingSt
7979
portSpec.NodePort = strategy.NodePort.Port
8080
}
8181
case hyperv1.Route:
82-
svc.Spec.Type = corev1.ServiceTypeClusterIP
82+
if ((hcp.Spec.Platform.Type == hyperv1.IBMCloudPlatform) && (svc.Spec.Type != corev1.ServiceTypeNodePort)) || (hcp.Spec.Platform.Type != hyperv1.IBMCloudPlatform) {
83+
svc.Spec.Type = corev1.ServiceTypeClusterIP
84+
}
8385
default:
8486
return fmt.Errorf("invalid publishing strategy for Kube API server service: %s", strategy.Type)
8587
}
@@ -279,7 +281,9 @@ func ReconcileKonnectivityServerService(svc *corev1.Service, ownerRef config.Own
279281
portSpec.NodePort = strategy.NodePort.Port
280282
}
281283
case hyperv1.Route:
282-
svc.Spec.Type = corev1.ServiceTypeClusterIP
284+
if ((hcp.Spec.Platform.Type == hyperv1.IBMCloudPlatform) && (svc.Spec.Type != corev1.ServiceTypeNodePort)) || (hcp.Spec.Platform.Type != hyperv1.IBMCloudPlatform) {
285+
svc.Spec.Type = corev1.ServiceTypeClusterIP
286+
}
283287
default:
284288
return fmt.Errorf("invalid publishing strategy for Konnectivity service: %s", strategy.Type)
285289
}

‎control-plane-operator/controllers/hostedcontrolplane/oauth/service.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ var (
2525
}
2626
)
2727

28-
func ReconcileService(svc *corev1.Service, ownerRef config.OwnerRef, strategy *hyperv1.ServicePublishingStrategy) error {
28+
func ReconcileService(svc *corev1.Service, ownerRef config.OwnerRef, strategy *hyperv1.ServicePublishingStrategy, platformType hyperv1.PlatformType) error {
2929
ownerRef.ApplyTo(svc)
3030
if svc.Spec.Selector == nil {
3131
svc.Spec.Selector = oauthServerLabels
@@ -51,7 +51,9 @@ func ReconcileService(svc *corev1.Service, ownerRef config.OwnerRef, strategy *h
5151
portSpec.NodePort = strategy.NodePort.Port
5252
}
5353
case hyperv1.Route:
54-
svc.Spec.Type = corev1.ServiceTypeClusterIP
54+
if ((platformType == hyperv1.IBMCloudPlatform) && (svc.Spec.Type != corev1.ServiceTypeNodePort)) || (platformType != hyperv1.IBMCloudPlatform) {
55+
svc.Spec.Type = corev1.ServiceTypeClusterIP
56+
}
5557
default:
5658
return fmt.Errorf("invalid publishing strategy for OAuth service: %s", strategy.Type)
5759
}

0 commit comments

Comments
 (0)