1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
|
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package discovery
import (
"encoding/json"
"fmt"
"net/url"
"sort"
"strings"
"github.com/emicklei/go-restful-swagger12"
"github.com/go-openapi/loads"
"github.com/go-openapi/spec"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/version"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/pkg/api/v1"
restclient "k8s.io/client-go/rest"
)
// defaultRetries is the number of times a resource discovery is repeated if an api group disappears on the fly (e.g. ThirdPartyResources).
const defaultRetries = 2
// DiscoveryInterface holds the methods that discover server-supported API groups,
// versions and resources.
type DiscoveryInterface interface {
RESTClient() restclient.Interface
ServerGroupsInterface
ServerResourcesInterface
ServerVersionInterface
SwaggerSchemaInterface
OpenAPISchemaInterface
}
// CachedDiscoveryInterface is a DiscoveryInterface with cache invalidation and freshness.
type CachedDiscoveryInterface interface {
DiscoveryInterface
// Fresh returns true if no cached data was used that had been retrieved before the instantiation.
Fresh() bool
// Invalidate enforces that no cached data is used in the future that is older than the current time.
Invalidate()
}
// ServerGroupsInterface has methods for obtaining supported groups on the API server
type ServerGroupsInterface interface {
// ServerGroups returns the supported groups, with information like supported versions and the
// preferred version.
ServerGroups() (*metav1.APIGroupList, error)
}
// ServerResourcesInterface has methods for obtaining supported resources on the API server
type ServerResourcesInterface interface {
// ServerResourcesForGroupVersion returns the supported resources for a group and version.
ServerResourcesForGroupVersion(groupVersion string) (*metav1.APIResourceList, error)
// ServerResources returns the supported resources for all groups and versions.
ServerResources() ([]*metav1.APIResourceList, error)
// ServerPreferredResources returns the supported resources with the version preferred by the
// server.
ServerPreferredResources() ([]*metav1.APIResourceList, error)
// ServerPreferredNamespacedResources returns the supported namespaced resources with the
// version preferred by the server.
ServerPreferredNamespacedResources() ([]*metav1.APIResourceList, error)
}
// ServerVersionInterface has a method for retrieving the server's version.
type ServerVersionInterface interface {
// ServerVersion retrieves and parses the server's version (git version).
ServerVersion() (*version.Info, error)
}
// SwaggerSchemaInterface has a method to retrieve the swagger schema.
type SwaggerSchemaInterface interface {
// SwaggerSchema retrieves and parses the swagger API schema the server supports.
SwaggerSchema(version schema.GroupVersion) (*swagger.ApiDeclaration, error)
}
// OpenAPISchemaInterface has a method to retrieve the open API schema.
type OpenAPISchemaInterface interface {
// OpenAPISchema retrieves and parses the swagger API schema the server supports.
OpenAPISchema() (*spec.Swagger, error)
}
// DiscoveryClient implements the functions that discover server-supported API groups,
// versions and resources.
type DiscoveryClient struct {
restClient restclient.Interface
LegacyPrefix string
}
// Convert metav1.APIVersions to metav1.APIGroup. APIVersions is used by legacy v1, so
// group would be "".
func apiVersionsToAPIGroup(apiVersions *metav1.APIVersions) (apiGroup metav1.APIGroup) {
groupVersions := []metav1.GroupVersionForDiscovery{}
for _, version := range apiVersions.Versions {
groupVersion := metav1.GroupVersionForDiscovery{
GroupVersion: version,
Version: version,
}
groupVersions = append(groupVersions, groupVersion)
}
apiGroup.Versions = groupVersions
// There should be only one groupVersion returned at /api
apiGroup.PreferredVersion = groupVersions[0]
return
}
// ServerGroups returns the supported groups, with information like supported versions and the
// preferred version.
func (d *DiscoveryClient) ServerGroups() (apiGroupList *metav1.APIGroupList, err error) {
// Get the groupVersions exposed at /api
v := &metav1.APIVersions{}
err = d.restClient.Get().AbsPath(d.LegacyPrefix).Do().Into(v)
apiGroup := metav1.APIGroup{}
if err == nil && len(v.Versions) != 0 {
apiGroup = apiVersionsToAPIGroup(v)
}
if err != nil && !errors.IsNotFound(err) && !errors.IsForbidden(err) {
return nil, err
}
// Get the groupVersions exposed at /apis
apiGroupList = &metav1.APIGroupList{}
err = d.restClient.Get().AbsPath("/apis").Do().Into(apiGroupList)
if err != nil && !errors.IsNotFound(err) && !errors.IsForbidden(err) {
return nil, err
}
// to be compatible with a v1.0 server, if it's a 403 or 404, ignore and return whatever we got from /api
if err != nil && (errors.IsNotFound(err) || errors.IsForbidden(err)) {
apiGroupList = &metav1.APIGroupList{}
}
// append the group retrieved from /api to the list if not empty
if len(v.Versions) != 0 {
apiGroupList.Groups = append(apiGroupList.Groups, apiGroup)
}
return apiGroupList, nil
}
// ServerResourcesForGroupVersion returns the supported resources for a group and version.
func (d *DiscoveryClient) ServerResourcesForGroupVersion(groupVersion string) (resources *metav1.APIResourceList, err error) {
url := url.URL{}
if len(groupVersion) == 0 {
return nil, fmt.Errorf("groupVersion shouldn't be empty")
}
if len(d.LegacyPrefix) > 0 && groupVersion == "v1" {
url.Path = d.LegacyPrefix + "/" + groupVersion
} else {
url.Path = "/apis/" + groupVersion
}
resources = &metav1.APIResourceList{
GroupVersion: groupVersion,
}
err = d.restClient.Get().AbsPath(url.String()).Do().Into(resources)
if err != nil {
// ignore 403 or 404 error to be compatible with an v1.0 server.
if groupVersion == "v1" && (errors.IsNotFound(err) || errors.IsForbidden(err)) {
return resources, nil
}
return nil, err
}
return resources, nil
}
// serverResources returns the supported resources for all groups and versions.
func (d *DiscoveryClient) serverResources() ([]*metav1.APIResourceList, error) {
apiGroups, err := d.ServerGroups()
if err != nil {
return nil, err
}
result := []*metav1.APIResourceList{}
failedGroups := make(map[schema.GroupVersion]error)
for _, apiGroup := range apiGroups.Groups {
for _, version := range apiGroup.Versions {
gv := schema.GroupVersion{Group: apiGroup.Name, Version: version.Version}
resources, err := d.ServerResourcesForGroupVersion(version.GroupVersion)
if err != nil {
// TODO: maybe restrict this to NotFound errors
failedGroups[gv] = err
continue
}
result = append(result, resources)
}
}
if len(failedGroups) == 0 {
return result, nil
}
return result, &ErrGroupDiscoveryFailed{Groups: failedGroups}
}
// ServerResources returns the supported resources for all groups and versions.
func (d *DiscoveryClient) ServerResources() ([]*metav1.APIResourceList, error) {
return withRetries(defaultRetries, d.serverResources)
}
// ErrGroupDiscoveryFailed is returned if one or more API groups fail to load.
type ErrGroupDiscoveryFailed struct {
// Groups is a list of the groups that failed to load and the error cause
Groups map[schema.GroupVersion]error
}
// Error implements the error interface
func (e *ErrGroupDiscoveryFailed) Error() string {
var groups []string
for k, v := range e.Groups {
groups = append(groups, fmt.Sprintf("%s: %v", k, v))
}
sort.Strings(groups)
return fmt.Sprintf("unable to retrieve the complete list of server APIs: %s", strings.Join(groups, ", "))
}
// IsGroupDiscoveryFailedError returns true if the provided error indicates the server was unable to discover
// a complete list of APIs for the client to use.
func IsGroupDiscoveryFailedError(err error) bool {
_, ok := err.(*ErrGroupDiscoveryFailed)
return err != nil && ok
}
// serverPreferredResources returns the supported resources with the version preferred by the server.
func (d *DiscoveryClient) serverPreferredResources() ([]*metav1.APIResourceList, error) {
serverGroupList, err := d.ServerGroups()
if err != nil {
return nil, err
}
result := []*metav1.APIResourceList{}
failedGroups := make(map[schema.GroupVersion]error)
grVersions := map[schema.GroupResource]string{} // selected version of a GroupResource
grApiResources := map[schema.GroupResource]*metav1.APIResource{} // selected APIResource for a GroupResource
gvApiResourceLists := map[schema.GroupVersion]*metav1.APIResourceList{} // blueprint for a APIResourceList for later grouping
for _, apiGroup := range serverGroupList.Groups {
for _, version := range apiGroup.Versions {
groupVersion := schema.GroupVersion{Group: apiGroup.Name, Version: version.Version}
apiResourceList, err := d.ServerResourcesForGroupVersion(version.GroupVersion)
if err != nil {
// TODO: maybe restrict this to NotFound errors
failedGroups[groupVersion] = err
continue
}
// create empty list which is filled later in another loop
emptyApiResourceList := metav1.APIResourceList{
GroupVersion: version.GroupVersion,
}
gvApiResourceLists[groupVersion] = &emptyApiResourceList
result = append(result, &emptyApiResourceList)
for i := range apiResourceList.APIResources {
apiResource := &apiResourceList.APIResources[i]
if strings.Contains(apiResource.Name, "/") {
continue
}
gv := schema.GroupResource{Group: apiGroup.Name, Resource: apiResource.Name}
if _, ok := grApiResources[gv]; ok && version.Version != apiGroup.PreferredVersion.Version {
// only override with preferred version
continue
}
grVersions[gv] = version.Version
grApiResources[gv] = apiResource
}
}
}
// group selected APIResources according to GroupVersion into APIResourceLists
for groupResource, apiResource := range grApiResources {
version := grVersions[groupResource]
groupVersion := schema.GroupVersion{Group: groupResource.Group, Version: version}
apiResourceList := gvApiResourceLists[groupVersion]
apiResourceList.APIResources = append(apiResourceList.APIResources, *apiResource)
}
if len(failedGroups) == 0 {
return result, nil
}
return result, &ErrGroupDiscoveryFailed{Groups: failedGroups}
}
// ServerPreferredResources returns the supported resources with the version preferred by the
// server.
func (d *DiscoveryClient) ServerPreferredResources() ([]*metav1.APIResourceList, error) {
return withRetries(defaultRetries, d.serverPreferredResources)
}
// ServerPreferredNamespacedResources returns the supported namespaced resources with the
// version preferred by the server.
func (d *DiscoveryClient) ServerPreferredNamespacedResources() ([]*metav1.APIResourceList, error) {
all, err := d.ServerPreferredResources()
return FilteredBy(ResourcePredicateFunc(func(groupVersion string, r *metav1.APIResource) bool {
return r.Namespaced
}), all), err
}
// ServerVersion retrieves and parses the server's version (git version).
func (d *DiscoveryClient) ServerVersion() (*version.Info, error) {
body, err := d.restClient.Get().AbsPath("/version").Do().Raw()
if err != nil {
return nil, err
}
var info version.Info
err = json.Unmarshal(body, &info)
if err != nil {
return nil, fmt.Errorf("got '%s': %v", string(body), err)
}
return &info, nil
}
// SwaggerSchema retrieves and parses the swagger API schema the server supports.
// TODO: Replace usages with Open API. Tracked in https://github.com/kubernetes/kubernetes/issues/44589
func (d *DiscoveryClient) SwaggerSchema(version schema.GroupVersion) (*swagger.ApiDeclaration, error) {
if version.Empty() {
return nil, fmt.Errorf("groupVersion cannot be empty")
}
groupList, err := d.ServerGroups()
if err != nil {
return nil, err
}
groupVersions := metav1.ExtractGroupVersions(groupList)
// This check also takes care the case that kubectl is newer than the running endpoint
if stringDoesntExistIn(version.String(), groupVersions) {
return nil, fmt.Errorf("API version: %v is not supported by the server. Use one of: %v", version, groupVersions)
}
var path string
if len(d.LegacyPrefix) > 0 && version == v1.SchemeGroupVersion {
path = "/swaggerapi" + d.LegacyPrefix + "/" + version.Version
} else {
path = "/swaggerapi/apis/" + version.Group + "/" + version.Version
}
body, err := d.restClient.Get().AbsPath(path).Do().Raw()
if err != nil {
return nil, err
}
var schema swagger.ApiDeclaration
err = json.Unmarshal(body, &schema)
if err != nil {
return nil, fmt.Errorf("got '%s': %v", string(body), err)
}
return &schema, nil
}
// OpenAPISchema fetches the open api schema using a rest client and parses the json.
// Warning: this is very expensive (~1.2s)
func (d *DiscoveryClient) OpenAPISchema() (*spec.Swagger, error) {
data, err := d.restClient.Get().AbsPath("/swagger.json").Do().Raw()
if err != nil {
return nil, err
}
msg := json.RawMessage(data)
doc, err := loads.Analyzed(msg, "")
if err != nil {
return nil, err
}
return doc.Spec(), err
}
// withRetries retries the given recovery function in case the groups supported by the server change after ServerGroup() returns.
func withRetries(maxRetries int, f func() ([]*metav1.APIResourceList, error)) ([]*metav1.APIResourceList, error) {
var result []*metav1.APIResourceList
var err error
for i := 0; i < maxRetries; i++ {
result, err = f()
if err == nil {
return result, nil
}
if _, ok := err.(*ErrGroupDiscoveryFailed); !ok {
return nil, err
}
}
return result, err
}
func setDiscoveryDefaults(config *restclient.Config) error {
config.APIPath = ""
config.GroupVersion = nil
codec := runtime.NoopEncoder{Decoder: scheme.Codecs.UniversalDecoder()}
config.NegotiatedSerializer = serializer.NegotiatedSerializerWrapper(runtime.SerializerInfo{Serializer: codec})
if len(config.UserAgent) == 0 {
config.UserAgent = restclient.DefaultKubernetesUserAgent()
}
return nil
}
// NewDiscoveryClientForConfig creates a new DiscoveryClient for the given config. This client
// can be used to discover supported resources in the API server.
func NewDiscoveryClientForConfig(c *restclient.Config) (*DiscoveryClient, error) {
config := *c
if err := setDiscoveryDefaults(&config); err != nil {
return nil, err
}
client, err := restclient.UnversionedRESTClientFor(&config)
return &DiscoveryClient{restClient: client, LegacyPrefix: "/api"}, err
}
// NewDiscoveryClientForConfig creates a new DiscoveryClient for the given config. If
// there is an error, it panics.
func NewDiscoveryClientForConfigOrDie(c *restclient.Config) *DiscoveryClient {
client, err := NewDiscoveryClientForConfig(c)
if err != nil {
panic(err)
}
return client
}
// New creates a new DiscoveryClient for the given RESTClient.
func NewDiscoveryClient(c restclient.Interface) *DiscoveryClient {
return &DiscoveryClient{restClient: c, LegacyPrefix: "/api"}
}
func stringDoesntExistIn(str string, slice []string) bool {
for _, s := range slice {
if s == str {
return false
}
}
return true
}
// RESTClient returns a RESTClient that is used to communicate
// with API server by this client implementation.
func (c *DiscoveryClient) RESTClient() restclient.Interface {
if c == nil {
return nil
}
return c.restClient
}
|