Skip to content

Commit 4744012

Browse files
Provide layers referenced by an image stream as layers subresource
Adds a new GET endpoint to an image stream as a subresource `layers` that returns an array of every layer referenced by the image stream and the tags and images included by the image. The subresource is fed by a store driven informer that caches and indexes only the layers. Clients get a 500 retry error if the cache has not initialized yet (the client will silently retry). Turns the registry access check for a given layer into an O(1) check instead of O(N) where N is the number of images in the image stream.
1 parent b5b4a14 commit 4744012

File tree

4 files changed

+275
-4
lines changed

4 files changed

+275
-4
lines changed

pkg/cmd/server/origin/legacy.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,8 @@ func LegacyStorage(storage map[schema.GroupVersion]map[string]rest.Storage) map[
207207

208208
case *imagestreametcd.REST:
209209
legacyStorage[resource] = &imagestreametcd.LegacyREST{REST: storage}
210+
case *imagestreametcd.LayersREST:
211+
delete(legacyStorage, resource)
210212

211213
case *routeetcd.REST:
212214
store := *storage.Store

pkg/image/apiserver/apiserver.go

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ type ExtraConfig struct {
5353
makeV1Storage sync.Once
5454
v1Storage map[string]rest.Storage
5555
v1StorageErr error
56+
startFns []func(<-chan struct{})
5657
}
5758

5859
type ImageAPIServerConfig struct {
@@ -107,6 +108,15 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
107108
return nil, err
108109
}
109110

111+
if err := s.GenericAPIServer.AddPostStartHook("image.openshift.io-apiserver-caches", func(context genericapiserver.PostStartHookContext) error {
112+
for _, fn := range c.ExtraConfig.startFns {
113+
go fn(context.StopCh)
114+
}
115+
return nil
116+
}); err != nil {
117+
return nil, err
118+
}
119+
110120
return s, nil
111121
}
112122

@@ -168,10 +178,13 @@ func (c *completedConfig) newV1RESTStorage() (map[string]rest.Storage, error) {
168178
whitelister = whitelist.WhitelistAllRegistries()
169179
}
170180

181+
imageLayerIndex := imagestreametcd.NewImageLayerIndex(imageStorage)
182+
c.ExtraConfig.startFns = append(c.ExtraConfig.startFns, imageLayerIndex.Run)
183+
171184
imageRegistry := image.NewRegistry(imageStorage)
172185
imageSignatureStorage := imagesignature.NewREST(imageClient.Image())
173186
imageStreamSecretsStorage := imagesecret.NewREST(coreClient)
174-
imageStreamStorage, imageStreamStatusStorage, internalImageStreamStorage, err := imagestreametcd.NewREST(c.GenericConfig.RESTOptionsGetter, c.ExtraConfig.RegistryHostnameRetriever, authorizationClient.SubjectAccessReviews(), c.ExtraConfig.LimitVerifier, whitelister)
187+
imageStreamStorage, imageStreamLayersStorage, imageStreamStatusStorage, internalImageStreamStorage, err := imagestreametcd.NewREST(c.GenericConfig.RESTOptionsGetter, c.ExtraConfig.RegistryHostnameRetriever, authorizationClient.SubjectAccessReviews(), c.ExtraConfig.LimitVerifier, whitelister, imageLayerIndex)
175188
if err != nil {
176189
return nil, fmt.Errorf("error building REST storage: %v", err)
177190
}
@@ -206,6 +219,7 @@ func (c *completedConfig) newV1RESTStorage() (map[string]rest.Storage, error) {
206219
v1Storage["imagesignatures"] = imageSignatureStorage
207220
v1Storage["imageStreams/secrets"] = imageStreamSecretsStorage
208221
v1Storage["imageStreams"] = imageStreamStorage
222+
v1Storage["imageStreams/layers"] = imageStreamLayersStorage
209223
v1Storage["imageStreams/status"] = imageStreamStatusStorage
210224
v1Storage["imageStreamImports"] = imageStreamImportStorage
211225
v1Storage["imageStreamImages"] = imageStreamImageStorage

pkg/image/registry/imagestream/etcd/etcd.go

Lines changed: 72 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package etcd
22

33
import (
4+
"k8s.io/apimachinery/pkg/api/errors"
45
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
56
"k8s.io/apimachinery/pkg/runtime"
67
apirequest "k8s.io/apiserver/pkg/endpoints/request"
@@ -46,7 +47,8 @@ func NewREST(
4647
subjectAccessReviewRegistry authorizationclient.SubjectAccessReviewInterface,
4748
limitVerifier imageadmission.LimitVerifier,
4849
registryWhitelister whitelist.RegistryWhitelister,
49-
) (*REST, *StatusREST, *InternalREST, error) {
50+
imageLayerIndex ImageLayerIndex,
51+
) (*REST, *LayersREST, *StatusREST, *InternalREST, error) {
5052
store := registry.Store{
5153
NewFunc: func() runtime.Object { return &imageapi.ImageStream{} },
5254
NewListFunc: func() runtime.Object { return &imageapi.ImageStreamList{} },
@@ -71,9 +73,11 @@ func NewREST(
7173
AttrFunc: storage.AttrFunc(storage.DefaultNamespaceScopedAttr).WithFieldMutation(imageapi.ImageStreamSelector),
7274
}
7375
if err := store.CompleteWithOptions(options); err != nil {
74-
return nil, nil, nil, err
76+
return nil, nil, nil, nil, err
7577
}
7678

79+
layersREST := &LayersREST{index: imageLayerIndex, store: &store}
80+
7781
statusStrategy := imagestream.NewStatusStrategy(strategy)
7882
statusStore := store
7983
statusStore.Decorator = nil
@@ -88,7 +92,7 @@ func NewREST(
8892
internalStore.UpdateStrategy = internalStrategy
8993

9094
internalREST := &InternalREST{store: &internalStore}
91-
return rest, statusREST, internalREST, nil
95+
return rest, layersREST, statusREST, internalREST, nil
9296
}
9397

9498
// StatusREST implements the REST endpoint for changing the status of an image stream.
@@ -138,6 +142,71 @@ func (r *InternalREST) Update(ctx apirequest.Context, name string, objInfo rest.
138142
return r.store.Update(ctx, name, objInfo, createValidation, updateValidation)
139143
}
140144

145+
// LayersREST implements the REST endpoint for changing both the spec and status of an image stream.
146+
type LayersREST struct {
147+
store *registry.Store
148+
index ImageLayerIndex
149+
}
150+
151+
var _ rest.Getter = &LayersREST{}
152+
153+
func (r *LayersREST) New() runtime.Object {
154+
return &imageapi.ImageStreamLayers{}
155+
}
156+
157+
// Get returns the layers for an image stream.
158+
func (r *LayersREST) Get(ctx apirequest.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
159+
if !r.index.HasSynced() {
160+
return nil, errors.NewServerTimeout(r.store.DefaultQualifiedResource, "get", 2)
161+
}
162+
obj, err := r.store.Get(ctx, name, options)
163+
if err != nil {
164+
return nil, err
165+
}
166+
is := obj.(*imageapi.ImageStream)
167+
isl := &imageapi.ImageStreamLayers{
168+
ObjectMeta: is.ObjectMeta,
169+
Blobs: make(map[string]imageapi.ImageLayerData),
170+
Layers: make(map[string]imageapi.BlobReferences),
171+
Manifests: make(map[string]string),
172+
}
173+
174+
for _, status := range is.Status.Tags {
175+
for _, item := range status.Items {
176+
if len(item.Image) == 0 {
177+
continue
178+
}
179+
180+
obj, _, _ := r.index.GetIndexer().GetByKey(item.Image)
181+
entry, ok := obj.(*ImageLayers)
182+
if !ok {
183+
continue
184+
}
185+
186+
if _, ok := isl.Layers[item.Image]; !ok {
187+
names := make(imageapi.BlobReferences, 0, len(entry.Layers))
188+
for _, layer := range entry.Layers {
189+
names = append(names, layer.Name)
190+
if _, ok := isl.Blobs[layer.Name]; !ok {
191+
isl.Blobs[layer.Name] = imageapi.ImageLayerData{LayerSize: layer.LayerSize, MediaType: layer.MediaType}
192+
}
193+
}
194+
isl.Layers[item.Image] = names
195+
}
196+
197+
if blob := entry.Manifest; blob != nil {
198+
if _, ok := isl.Manifests[item.Image]; !ok {
199+
isl.Manifests[item.Image] = blob.Name
200+
if _, ok := isl.Blobs[blob.Name]; !ok {
201+
isl.Blobs[blob.Name] = imageapi.ImageLayerData{LayerSize: blob.LayerSize, MediaType: blob.MediaType}
202+
}
203+
}
204+
}
205+
}
206+
}
207+
return isl, nil
208+
}
209+
141210
// LegacyREST allows us to wrap and alter some behavior
142211
type LegacyREST struct {
143212
*REST
Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
package etcd
2+
3+
import (
4+
"fmt"
5+
6+
metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
7+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
8+
"k8s.io/apimachinery/pkg/runtime"
9+
"k8s.io/apimachinery/pkg/runtime/schema"
10+
"k8s.io/apimachinery/pkg/types"
11+
"k8s.io/apimachinery/pkg/watch"
12+
apirequest "k8s.io/apiserver/pkg/endpoints/request"
13+
"k8s.io/apiserver/pkg/registry/rest"
14+
"k8s.io/client-go/tools/cache"
15+
16+
imageapi "github.com/openshift/origin/pkg/image/apis/image"
17+
)
18+
19+
// ImageLayerIndex is a cache of image digests to the layers they contain.
20+
// Because a very large number of images can exist on a cluster, we only
21+
// hold in memory a small subset of the full image object.
22+
type ImageLayerIndex interface {
23+
cache.SharedIndexInformer
24+
}
25+
26+
type ImageStore interface {
27+
rest.Watcher
28+
rest.Lister
29+
}
30+
31+
// NewImageLayerIndex creates a new index over a store that must return
32+
// images.
33+
func NewImageLayerIndex(store ImageStore) ImageLayerIndex {
34+
ctx := apirequest.NewContext()
35+
informer := cache.NewSharedIndexInformer(&cache.ListWatch{
36+
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
37+
obj, err := store.List(ctx, &metainternalversion.ListOptions{
38+
ResourceVersion: options.ResourceVersion,
39+
Limit: options.Limit,
40+
Continue: options.Continue,
41+
})
42+
if err != nil {
43+
return nil, err
44+
}
45+
list, ok := obj.(*imageapi.ImageList)
46+
if !ok {
47+
return nil, fmt.Errorf("unexpected store type %T for layer index", obj)
48+
}
49+
// reduce the full image list to a smaller subset.
50+
out := &metainternalversion.List{
51+
Items: make([]runtime.Object, len(list.Items)),
52+
}
53+
out.Continue = list.Continue
54+
out.ResourceVersion = list.ResourceVersion
55+
for i, image := range list.Items {
56+
out.Items[i] = &ImageLayers{
57+
Name: image.Name,
58+
Layers: image.DockerImageLayers,
59+
Manifest: manifestFromImage(&image),
60+
}
61+
}
62+
return out, nil
63+
},
64+
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
65+
w, err := store.Watch(ctx, &metainternalversion.ListOptions{
66+
ResourceVersion: options.ResourceVersion,
67+
})
68+
if err != nil {
69+
return nil, err
70+
}
71+
return watch.Filter(w, func(in watch.Event) (out watch.Event, keep bool) {
72+
if in.Object == nil {
73+
return in, true
74+
}
75+
// reduce each object to the minimal subset we need for the cache
76+
image, ok := in.Object.(*imageapi.Image)
77+
if !ok {
78+
return in, true
79+
}
80+
in.Object = &ImageLayers{
81+
Name: image.Name,
82+
ResourceVersion: image.ResourceVersion,
83+
Layers: image.DockerImageLayers,
84+
Manifest: manifestFromImage(image),
85+
}
86+
return in, true
87+
}), nil
88+
},
89+
}, &ImageLayers{}, 0, cache.Indexers{
90+
// layers allows fast access to the images with a given layer
91+
"layers": func(obj interface{}) ([]string, error) {
92+
entry, ok := obj.(*ImageLayers)
93+
if !ok {
94+
return nil, fmt.Errorf("unexpected cache object %T", obj)
95+
}
96+
keys := make([]string, 0, len(entry.Layers))
97+
for _, layer := range entry.Layers {
98+
keys = append(keys, layer.Name)
99+
}
100+
return keys, nil
101+
},
102+
})
103+
return informer
104+
}
105+
106+
// manifestFromImage attempts to find a manifest blob description from
107+
// an image. Images older than schema2 in Docker do not have a manifest blob.
108+
func manifestFromImage(image *imageapi.Image) *imageapi.ImageLayer {
109+
if image.DockerImageManifestMediaType != "application/vnd.docker.distribution.manifest.v2+json" {
110+
return nil
111+
}
112+
return &imageapi.ImageLayer{
113+
Name: image.DockerImageMetadata.ID,
114+
MediaType: image.DockerImageManifestMediaType,
115+
}
116+
}
117+
118+
// ImageLayers is the minimal set of data we need to retain to provide the cache.
119+
// Unlike a more general informer cache, we do not retain the full object because of
120+
// the potential size of the objects being stored. Even a small cluster may have 20k
121+
// or more images in active use.
122+
type ImageLayers struct {
123+
Name string
124+
ResourceVersion string
125+
Manifest *imageapi.ImageLayer
126+
Layers []imageapi.ImageLayer
127+
}
128+
129+
var (
130+
_ runtime.Object = &ImageLayers{}
131+
_ metav1.Object = &ImageLayers{}
132+
)
133+
134+
func (l *ImageLayers) GetObjectKind() schema.ObjectKind { return &metav1.TypeMeta{} }
135+
func (l *ImageLayers) DeepCopyObject() runtime.Object {
136+
var layers []imageapi.ImageLayer
137+
if l.Layers != nil {
138+
layers = make([]imageapi.ImageLayer, len(l.Layers))
139+
copy(layers, l.Layers)
140+
}
141+
return &ImageLayers{
142+
Name: l.Name,
143+
Layers: layers,
144+
}
145+
}
146+
147+
// client-go/cache.SharedIndexInformer hardcodes the key function to assume ObjectMeta.
148+
// Here we implement the relevant accessors to allow a minimal index to be created.
149+
// SharedIndexInformer will be refactored to require a more minimal subset of actions
150+
// in the near future.
151+
152+
func (l *ImageLayers) GetName() string { return l.Name }
153+
func (l *ImageLayers) GetNamespace() string { return "" }
154+
func (l *ImageLayers) GetResourceVersion() string { return l.ResourceVersion }
155+
func (l *ImageLayers) SetResourceVersion(version string) { l.ResourceVersion = version }
156+
157+
// These methods are unused stubs to satisfy meta.Object.
158+
159+
func (l *ImageLayers) SetNamespace(namespace string) {}
160+
func (l *ImageLayers) SetName(name string) {}
161+
func (l *ImageLayers) GetGenerateName() string { return "" }
162+
func (l *ImageLayers) SetGenerateName(name string) {}
163+
func (l *ImageLayers) GetUID() types.UID { return "" }
164+
func (l *ImageLayers) SetUID(uid types.UID) {}
165+
func (l *ImageLayers) GetGeneration() int64 { return 0 }
166+
func (l *ImageLayers) SetGeneration(generation int64) {}
167+
func (l *ImageLayers) GetSelfLink() string { return "" }
168+
func (l *ImageLayers) SetSelfLink(selfLink string) {}
169+
func (l *ImageLayers) GetCreationTimestamp() metav1.Time { return metav1.Time{} }
170+
func (l *ImageLayers) SetCreationTimestamp(timestamp metav1.Time) {}
171+
func (l *ImageLayers) GetDeletionTimestamp() *metav1.Time { return nil }
172+
func (l *ImageLayers) SetDeletionTimestamp(timestamp *metav1.Time) {}
173+
func (l *ImageLayers) GetDeletionGracePeriodSeconds() *int64 { return nil }
174+
func (l *ImageLayers) SetDeletionGracePeriodSeconds(*int64) {}
175+
func (l *ImageLayers) GetLabels() map[string]string { return nil }
176+
func (l *ImageLayers) SetLabels(labels map[string]string) {}
177+
func (l *ImageLayers) GetAnnotations() map[string]string { return nil }
178+
func (l *ImageLayers) SetAnnotations(annotations map[string]string) {}
179+
func (l *ImageLayers) GetInitializers() *metav1.Initializers { return nil }
180+
func (l *ImageLayers) SetInitializers(initializers *metav1.Initializers) {}
181+
func (l *ImageLayers) GetFinalizers() []string { return nil }
182+
func (l *ImageLayers) SetFinalizers(finalizers []string) {}
183+
func (l *ImageLayers) GetOwnerReferences() []metav1.OwnerReference { return nil }
184+
func (l *ImageLayers) SetOwnerReferences([]metav1.OwnerReference) {}
185+
func (l *ImageLayers) GetClusterName() string { return "" }
186+
func (l *ImageLayers) SetClusterName(clusterName string) {}

0 commit comments

Comments
 (0)