Skip to content

Commit fc3523f

Browse files
tkashemsoltysh
tkashem
authored and
soltysh
committed
UPSTREAM: <carry>: optionally enable retry after until apiserver is ready
1 parent 7bb1650 commit fc3523f

File tree

4 files changed

+282
-10
lines changed

4 files changed

+282
-10
lines changed

‎staging/src/k8s.io/apiserver/pkg/server/config.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,18 @@ type Config struct {
265265

266266
// StorageVersionManager holds the storage versions of the API resources installed by this server.
267267
StorageVersionManager storageversion.Manager
268+
269+
// SendRetryAfterWhileNotReadyOnce, if enabled, the apiserver will
270+
// reject all incoming requests with a 503 status code and a
271+
// 'Retry-After' response header until the apiserver has fully
272+
// initialized, except for requests from a designated debugger group.
273+
// This option ensures that the system stays consistent even when
274+
// requests are received before the server has been initialized.
275+
// In particular, it prevents child deletion in case of GC or/and
276+
// orphaned content in case of the namespaces controller.
277+
// NOTE: this option is applicable to Microshift only,
278+
// this should never be enabled for OCP.
279+
SendRetryAfterWhileNotReadyOnce bool
268280
}
269281

270282
// EventSink allows to create events.
@@ -901,6 +913,10 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
901913
handler = genericfilters.WithMaxInFlightLimit(handler, c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight, c.LongRunningFunc)
902914
}
903915

916+
if c.SendRetryAfterWhileNotReadyOnce {
917+
handler = genericfilters.WithNotReady(handler, c.lifecycleSignals.HasBeenReady.Signaled())
918+
}
919+
904920
handler = filterlatency.TrackCompleted(handler)
905921
handler = genericapifilters.WithImpersonation(handler, c.Authorization.Authorizer, c.Serializer)
906922
handler = filterlatency.TrackStarted(handler, "impersonation")

‎staging/src/k8s.io/apiserver/pkg/server/filters/with_not_ready_patch.go

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
/*
2+
Copyright 2022 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package filters
18+
19+
import (
20+
"errors"
21+
"k8s.io/apiserver/pkg/warning"
22+
"net/http"
23+
24+
"k8s.io/apiserver/pkg/authentication/user"
25+
"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
26+
"k8s.io/apiserver/pkg/endpoints/request"
27+
)
28+
29+
const (
30+
// notReadyDebuggerGroup facilitates debugging if the apiserver takes longer
31+
// to initilize. All request(s) from this designated group will be allowed
32+
// while the apiserver is being initialized.
33+
// The apiserver will reject all incoming requests with a 'Retry-After'
34+
// response header until it has fully initialized, except for
35+
// requests from this special debugger group.
36+
notReadyDebuggerGroup = "system:openshift:risky-not-ready-microshift-debugging-group"
37+
)
38+
39+
// WithNotReady rejects any incoming new request(s) with a 'Retry-After'
40+
// response if the specified hasBeenReadyCh channel is still open, with
41+
// the following exceptions:
42+
// - all request(s) from the designated debugger group is exempt, this
43+
// helps debug the apiserver if it takes longer to initialize.
44+
// - local loopback requests (this exempts system:apiserver)
45+
// - /healthz, /livez, /readyz, /metrics are exempt
46+
//
47+
// It includes new request(s) on a new or an existing TCP connection
48+
// Any new request(s) arriving before hasBeenreadyCh is closed
49+
// are replied with a 503 and the following response headers:
50+
// - 'Retry-After: N` (so client can retry after N seconds)
51+
func WithNotReady(handler http.Handler, hasBeenReadyCh <-chan struct{}) http.Handler {
52+
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
53+
select {
54+
case <-hasBeenReadyCh:
55+
handler.ServeHTTP(w, req)
56+
return
57+
default:
58+
}
59+
60+
requestor, exists := request.UserFrom(req.Context())
61+
if !exists {
62+
responsewriters.InternalError(w, req, errors.New("no user found for request"))
63+
return
64+
}
65+
66+
// make sure we exempt:
67+
// - local loopback requests (this exempts system:apiserver)
68+
// - health probes and metric scraping
69+
// - requests from the exempt debugger group.
70+
if requestor.GetName() == user.APIServerUser ||
71+
hasExemptPathPrefix(req) ||
72+
matchesDebuggerGroup(requestor, notReadyDebuggerGroup) {
73+
warning.AddWarning(req.Context(), "", "The apiserver was still initializing, while this request was being served")
74+
handler.ServeHTTP(w, req)
75+
return
76+
}
77+
78+
// Return a 503 status asking the client to try again after 5 seconds
79+
w.Header().Set("Retry-After", "5")
80+
http.Error(w, "The apiserver hasn't been fully initialized yet, please try again later.",
81+
http.StatusServiceUnavailable)
82+
})
83+
}
84+
85+
func matchesDebuggerGroup(requestor user.Info, debugger string) bool {
86+
for _, group := range requestor.GetGroups() {
87+
if group == debugger {
88+
return true
89+
}
90+
}
91+
return false
92+
}

‎staging/src/k8s.io/apiserver/pkg/server/filters/with_not_ready_patch_test.go

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
package filters
2+
3+
import (
4+
"net/http"
5+
"net/http/httptest"
6+
"testing"
7+
8+
"k8s.io/apiserver/pkg/authentication/user"
9+
genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
10+
"k8s.io/apiserver/pkg/endpoints/request"
11+
)
12+
13+
func TestWithNotReady(t *testing.T) {
14+
const warning = `299 - "The apiserver was still initializing, while this request was being served"`
15+
16+
tests := []struct {
17+
name string
18+
requestURL string
19+
hasBeenReady bool
20+
user *user.DefaultInfo
21+
handlerInvoked int
22+
retryAfterExpected string
23+
warningExpected string
24+
statusCodeexpected int
25+
}{
26+
{
27+
name: "the apiserver is fully initialized",
28+
hasBeenReady: true,
29+
handlerInvoked: 1,
30+
statusCodeexpected: http.StatusOK,
31+
},
32+
{
33+
name: "the apiserver is initializing, local loopback",
34+
hasBeenReady: false,
35+
user: &user.DefaultInfo{Name: user.APIServerUser},
36+
handlerInvoked: 1,
37+
statusCodeexpected: http.StatusOK,
38+
warningExpected: warning,
39+
},
40+
{
41+
name: "the apiserver is initializing, exempt debugger group",
42+
hasBeenReady: false,
43+
user: &user.DefaultInfo{Groups: []string{"system:authenticated", notReadyDebuggerGroup}},
44+
handlerInvoked: 1,
45+
statusCodeexpected: http.StatusOK,
46+
warningExpected: warning,
47+
},
48+
{
49+
name: "the apiserver is initializing, readyz",
50+
requestURL: "/readyz?verbose=1",
51+
user: &user.DefaultInfo{},
52+
hasBeenReady: false,
53+
handlerInvoked: 1,
54+
statusCodeexpected: http.StatusOK,
55+
warningExpected: warning,
56+
},
57+
{
58+
name: "the apiserver is initializing, healthz",
59+
requestURL: "/healthz?verbose=1",
60+
user: &user.DefaultInfo{},
61+
hasBeenReady: false,
62+
handlerInvoked: 1,
63+
statusCodeexpected: http.StatusOK,
64+
warningExpected: warning,
65+
},
66+
{
67+
name: "the apiserver is initializing, livez",
68+
requestURL: "/livez?verbose=1",
69+
user: &user.DefaultInfo{},
70+
hasBeenReady: false,
71+
handlerInvoked: 1,
72+
statusCodeexpected: http.StatusOK,
73+
warningExpected: warning,
74+
},
75+
{
76+
name: "the apiserver is initializing, metrics",
77+
requestURL: "/metrics",
78+
user: &user.DefaultInfo{},
79+
hasBeenReady: false,
80+
handlerInvoked: 1,
81+
statusCodeexpected: http.StatusOK,
82+
warningExpected: warning,
83+
},
84+
{
85+
name: "the apiserver is initializing, non-exempt request",
86+
hasBeenReady: false,
87+
user: &user.DefaultInfo{Groups: []string{"system:authenticated", "system:masters"}},
88+
statusCodeexpected: http.StatusServiceUnavailable,
89+
retryAfterExpected: "5",
90+
},
91+
}
92+
93+
for _, test := range tests {
94+
t.Run(test.name, func(t *testing.T) {
95+
hasBeenReadyCh := make(chan struct{})
96+
if test.hasBeenReady {
97+
close(hasBeenReadyCh)
98+
} else {
99+
defer close(hasBeenReadyCh)
100+
}
101+
102+
var handlerInvoked int
103+
handler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
104+
handlerInvoked++
105+
w.WriteHeader(http.StatusOK)
106+
})
107+
108+
if len(test.requestURL) == 0 {
109+
test.requestURL = "/api/v1/namespaces"
110+
}
111+
req, err := http.NewRequest(http.MethodGet, test.requestURL, nil)
112+
if err != nil {
113+
t.Fatalf("failed to create new http request - %v", err)
114+
}
115+
if test.user != nil {
116+
req = req.WithContext(request.WithUser(req.Context(), test.user))
117+
}
118+
w := httptest.NewRecorder()
119+
120+
withNotReady := WithNotReady(handler, hasBeenReadyCh)
121+
withNotReady = genericapifilters.WithWarningRecorder(withNotReady)
122+
withNotReady.ServeHTTP(w, req)
123+
124+
if test.handlerInvoked != handlerInvoked {
125+
t.Errorf("expected the handler to be invoked: %d times, but got: %d", test.handlerInvoked, handlerInvoked)
126+
}
127+
if test.statusCodeexpected != w.Code {
128+
t.Errorf("expected Response Status Code: %d, but got: %d", test.statusCodeexpected, w.Code)
129+
}
130+
131+
retryAfterGot := w.Header().Get("Retry-After")
132+
if test.retryAfterExpected != retryAfterGot {
133+
t.Errorf("expected Retry-After: %q, but got: %q", test.retryAfterExpected, retryAfterGot)
134+
}
135+
136+
warningGot := w.Header().Get("Warning")
137+
if test.warningExpected != warningGot {
138+
t.Errorf("expected Warning: %s, but got: %s", test.warningExpected, warningGot)
139+
}
140+
141+
})
142+
}
143+
}

‎staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go

Lines changed: 31 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -63,21 +63,34 @@ type ServerRunOptions struct {
6363
// If enabled, after ShutdownDelayDuration elapses, any incoming request is
6464
// rejected with a 429 status code and a 'Retry-After' response.
6565
ShutdownSendRetryAfter bool
66+
67+
// SendRetryAfterWhileNotReadyOnce, if enabled, the apiserver will
68+
// reject all incoming requests with a 503 status code and a
69+
// 'Retry-After' response header until the apiserver has fully
70+
// initialized, except for requests from a designated debugger group.
71+
// This option ensures that the system stays consistent even when
72+
// requests are received before the server has been initialized.
73+
// In particular, it prevents child deletion in case of GC or/and
74+
// orphaned content in case of the namespaces controller.
75+
// NOTE: this option is applicable to Microshift only,
76+
// this should never be enabled for OCP.
77+
SendRetryAfterWhileNotReadyOnce bool
6678
}
6779

6880
func NewServerRunOptions() *ServerRunOptions {
6981
defaults := server.NewConfig(serializer.CodecFactory{})
7082
return &ServerRunOptions{
71-
MaxRequestsInFlight: defaults.MaxRequestsInFlight,
72-
MaxMutatingRequestsInFlight: defaults.MaxMutatingRequestsInFlight,
73-
RequestTimeout: defaults.RequestTimeout,
74-
LivezGracePeriod: defaults.LivezGracePeriod,
75-
MinRequestTimeout: defaults.MinRequestTimeout,
76-
ShutdownDelayDuration: defaults.ShutdownDelayDuration,
77-
JSONPatchMaxCopyBytes: defaults.JSONPatchMaxCopyBytes,
78-
MaxRequestBodyBytes: defaults.MaxRequestBodyBytes,
79-
EnablePriorityAndFairness: true,
80-
ShutdownSendRetryAfter: false,
83+
MaxRequestsInFlight: defaults.MaxRequestsInFlight,
84+
MaxMutatingRequestsInFlight: defaults.MaxMutatingRequestsInFlight,
85+
RequestTimeout: defaults.RequestTimeout,
86+
LivezGracePeriod: defaults.LivezGracePeriod,
87+
MinRequestTimeout: defaults.MinRequestTimeout,
88+
ShutdownDelayDuration: defaults.ShutdownDelayDuration,
89+
JSONPatchMaxCopyBytes: defaults.JSONPatchMaxCopyBytes,
90+
MaxRequestBodyBytes: defaults.MaxRequestBodyBytes,
91+
EnablePriorityAndFairness: true,
92+
ShutdownSendRetryAfter: false,
93+
SendRetryAfterWhileNotReadyOnce: false,
8194
}
8295
}
8396

@@ -97,6 +110,7 @@ func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
97110
c.MaxRequestBodyBytes = s.MaxRequestBodyBytes
98111
c.PublicAddress = s.AdvertiseAddress
99112
c.ShutdownSendRetryAfter = s.ShutdownSendRetryAfter
113+
c.SendRetryAfterWhileNotReadyOnce = s.SendRetryAfterWhileNotReadyOnce
100114

101115
return nil
102116
}
@@ -256,5 +270,12 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
256270
"during this window all incoming requests will be rejected with a status code 429 and a 'Retry-After' response header, "+
257271
"in addition 'Connection: close' response header is set in order to tear down the TCP connection when idle.")
258272

273+
// NOTE: this option is applicable to Microshift only, this should never be enabled for OCP.
274+
fs.BoolVar(&s.SendRetryAfterWhileNotReadyOnce, "send-retry-after-while-not-ready-once", s.SendRetryAfterWhileNotReadyOnce, ""+
275+
"If true, incoming request(s) will be rejected with a '503' status code and a 'Retry-After' response header "+
276+
"until the apiserver has initialized, except for requests from a certain group. This option ensures that the system stays "+
277+
"consistent even when requests arrive at the server before it has been initialized. "+
278+
"This option is applicable to Microshift only, this should never be enabled for OCP")
279+
259280
utilfeature.DefaultMutableFeatureGate.AddFlag(fs)
260281
}

0 commit comments

Comments
 (0)