From 8eb6ab4bcd247bcb89ea37e4ed06475bb222e338 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Mon, 17 Sep 2018 14:47:26 -0700 Subject: [PATCH 1/5] Remove intermediate API server objects Instead just generate HTTP handler functions directly. --- vkubelet/apiserver.go | 198 +++++++++++++++++++++--------------------- 1 file changed, 97 insertions(+), 101 deletions(-) diff --git a/vkubelet/apiserver.go b/vkubelet/apiserver.go index 0b971bae9..f69c18933 100644 --- a/vkubelet/apiserver.go +++ b/vkubelet/apiserver.go @@ -34,9 +34,11 @@ func NotFound(w http.ResponseWriter, r *http.Request) { http.Error(w, "404 request not found", http.StatusNotFound) } -// KubeletServer implements HTTP endpoints for serving kubelet API's -type KubeletServer struct { - p Provider +// NotImplemented provides a handler for cases where a provider does not implement a given API +func NotImplemented(w http.ResponseWriter, r *http.Request) { + logger := log.G(loggingContext(r)) + log.Trace(logger, "501 not implemented") + http.Error(w, "501 not implemented", http.StatusNotImplemented) } // KubeletServertStart starts the virtual kubelet HTTP server. @@ -47,9 +49,8 @@ func KubeletServerStart(p Provider) { addr := fmt.Sprintf(":%s", port) r := mux.NewRouter() - s := &KubeletServer{p: p} - r.HandleFunc("/containerLogs/{namespace}/{pod}/{container}", s.ApiServerHandler).Methods("GET") - r.HandleFunc("/exec/{namespace}/{pod}/{container}", s.ApiServerHandlerExec).Methods("POST") + r.HandleFunc("/containerLogs/{namespace}/{pod}/{container}", PodLogsHandlerFunc(p)).Methods("GET") + r.HandleFunc("/exec/{namespace}/{pod}/{container}", PodExecHandlerFunc(p)).Methods("POST") r.NotFoundHandler = http.HandlerFunc(NotFound) if err := http.ListenAndServeTLS(addr, certFilePath, keyFilePath, r); err != nil { @@ -61,120 +62,115 @@ func KubeletServerStart(p Provider) { // TLS is never enabled on this endpoint. func MetricsServerStart(p Provider, addr string) { r := mux.NewRouter() - s := &MetricsServer{p: p} - r.HandleFunc("/stats/summary", s.MetricsSummaryHandler).Methods("GET") - r.HandleFunc("/stats/summary/", s.MetricsSummaryHandler).Methods("GET") + + mp, ok := p.(MetricsProvider) + if !ok { + r.HandleFunc("/stats/summary", NotImplemented).Methods("GET") + r.HandleFunc("/stats/summary/", NotImplemented).Methods("GET") + } else { + r.HandleFunc("/stats/summary", PodMetricsHandlerFunc(mp)).Methods("GET") + r.HandleFunc("/stats/summary/", PodMetricsHandlerFunc(mp)).Methods("GET") + } r.NotFoundHandler = http.HandlerFunc(NotFound) if err := http.ListenAndServe(addr, r); err != nil { log.G(context.TODO()).WithError(err).Error("Error starting http server") } } -// MetricsServer provides an HTTP endpopint for accessing pod metrics -type MetricsServer struct { - p Provider -} +// PodMetricsHandlerFunc makes an HTTP handler for implementing the kubelet summary stats endpoint +func PodMetricsHandlerFunc(mp MetricsProvider) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + ctx := loggingContext(req) -// MetricsSummaryHandler is an HTTP handler for implementing the kubelet summary stats endpoint -func (s *MetricsServer) MetricsSummaryHandler(w http.ResponseWriter, req *http.Request) { - ctx := loggingContext(req) - - mp, ok := s.p.(MetricsProvider) - if !ok { - log.G(ctx).Debug("stats not implemented for provider") - http.Error(w, "not implememnted", http.StatusNotImplemented) - return - } - - stats, err := mp.GetStatsSummary(req.Context()) - if err != nil { - if errors.Cause(err) == context.Canceled { - return - } - log.G(ctx).Error("Error getting stats from provider:", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - b, err := json.Marshal(stats) - if err != nil { - log.G(ctx).WithError(err).Error("Could not marshal stats") - http.Error(w, "could not marshal stats: "+err.Error(), http.StatusInternalServerError) - return - } - - if _, err := w.Write(b); err != nil { - log.G(ctx).WithError(err).Debug("Could not write to client") - } -} - -func (s *KubeletServer) ApiServerHandler(w http.ResponseWriter, req *http.Request) { - vars := mux.Vars(req) - if len(vars) != 3 { - NotFound(w, req) - return - } - - ctx := loggingContext(req) - - namespace := vars["namespace"] - pod := vars["pod"] - container := vars["container"] - tail := 10 - q := req.URL.Query() - - if queryTail := q.Get("tailLines"); queryTail != "" { - t, err := strconv.Atoi(queryTail) + stats, err := mp.GetStatsSummary(req.Context()) if err != nil { - logger := log.G(context.TODO()).WithError(err) - log.Trace(logger, "could not parse tailLines") - http.Error(w, fmt.Sprintf("could not parse \"tailLines\": %v", err), http.StatusBadRequest) + if errors.Cause(err) == context.Canceled { + return + } + log.G(ctx).Error("Error getting stats from provider:", err) + http.Error(w, err.Error(), http.StatusInternalServerError) return } - tail = t - } - podsLogs, err := s.p.GetContainerLogs(ctx, namespace, pod, container, tail) - if err != nil { - log.G(ctx).WithError(err).Error("error getting container logs") - http.Error(w, fmt.Sprintf("error while getting container logs: %v", err), http.StatusInternalServerError) - return - } + b, err := json.Marshal(stats) + if err != nil { + log.G(ctx).WithError(err).Error("Could not marshal stats") + http.Error(w, "could not marshal stats: "+err.Error(), http.StatusInternalServerError) + return + } - if _, err := io.WriteString(w, podsLogs); err != nil { - log.G(ctx).WithError(err).Warn("error writing response to client") + if _, err := w.Write(b); err != nil { + log.G(ctx).WithError(err).Debug("Could not write to client") + } } } -func (s *KubeletServer) ApiServerHandlerExec(w http.ResponseWriter, req *http.Request) { - vars := mux.Vars(req) +// PodLogsHandlerFunc creates an http handler function from a provider to serve logs from a pod +func PodLogsHandlerFunc(p Provider) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + vars := mux.Vars(req) + if len(vars) != 3 { + NotFound(w, req) + return + } - namespace := vars["namespace"] - pod := vars["pod"] - container := vars["container"] + ctx := loggingContext(req) - supportedStreamProtocols := strings.Split(req.Header.Get("X-Stream-Protocol-Version"), ",") + namespace := vars["namespace"] + pod := vars["pod"] + container := vars["container"] + tail := 10 + q := req.URL.Query() - q := req.URL.Query() - command := q["command"] + if queryTail := q.Get("tailLines"); queryTail != "" { + t, err := strconv.Atoi(queryTail) + if err != nil { + logger := log.G(context.TODO()).WithError(err) + log.Trace(logger, "could not parse tailLines") + http.Error(w, fmt.Sprintf("could not parse \"tailLines\": %v", err), http.StatusBadRequest) + return + } + tail = t + } - // streamOpts := &remotecommand.Options{ - // Stdin: (q.Get("input") == "1"), - // Stdout: (q.Get("output") == "1"), - // Stderr: (q.Get("error") == "1"), - // TTY: (q.Get("tty") == "1"), - // } + podsLogs, err := p.GetContainerLogs(ctx, namespace, pod, container, tail) + if err != nil { + log.G(ctx).WithError(err).Error("error getting container logs") + http.Error(w, fmt.Sprintf("error while getting container logs: %v", err), http.StatusInternalServerError) + return + } - // TODO: tty flag causes remotecommand.createStreams to wait for the wrong number of streams - streamOpts := &remotecommand.Options{ - Stdin: true, - Stdout: true, - Stderr: true, - TTY: false, + if _, err := io.WriteString(w, podsLogs); err != nil { + log.G(ctx).WithError(err).Warn("error writing response to client") + } + } +} + +// PodExecHandlerFunc makes an http handler func from a Provider which execs a command in a pod's container +func PodExecHandlerFunc(p Provider) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + vars := mux.Vars(req) + + namespace := vars["namespace"] + pod := vars["pod"] + container := vars["container"] + + supportedStreamProtocols := strings.Split(req.Header.Get("X-Stream-Protocol-Version"), ",") + + q := req.URL.Query() + command := q["command"] + + // TODO: tty flag causes remotecommand.createStreams to wait for the wrong number of streams + streamOpts := &remotecommand.Options{ + Stdin: true, + Stdout: true, + Stderr: true, + TTY: false, + } + + idleTimeout := time.Second * 30 + streamCreationTimeout := time.Second * 30 + + remotecommand.ServeExec(w, req, p, fmt.Sprintf("%s-%s", namespace, pod), "", container, command, streamOpts, idleTimeout, streamCreationTimeout, supportedStreamProtocols) } - - idleTimeout := time.Second * 30 - streamCreationTimeout := time.Second * 30 - - remotecommand.ServeExec(w, req, s.p, fmt.Sprintf("%s-%s", namespace, pod), "", container, command, streamOpts, idleTimeout, streamCreationTimeout, supportedStreamProtocols) } From 74f76c75d5c80827609084a6bdc02156a95dd890 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Mon, 17 Sep 2018 16:35:17 -0700 Subject: [PATCH 2/5] Instrustment handlers for logging/error handling This refactors a bit of the http handler code. Moves error handling for handler functions to a generic handler. This also has a side-effect of being able to propagate errors from the provider to send the correct status code, provided the error type matches a pre-defined interface. --- Gopkg.lock | 19 +- .../github.com/cpuguy83/strongerrors/LICENSE | 201 +++++++++++++ .../github.com/cpuguy83/strongerrors/defs.go | 84 ++++++ .../github.com/cpuguy83/strongerrors/doc.go | 12 + .../cpuguy83/strongerrors/helpers.go | 272 ++++++++++++++++++ vendor/github.com/cpuguy83/strongerrors/is.go | 128 +++++++++ .../cpuguy83/strongerrors/status/grpc.go | 85 ++++++ .../cpuguy83/strongerrors/status/http.go | 37 +++ vkubelet/apiserver.go | 85 ++++-- 9 files changed, 890 insertions(+), 33 deletions(-) create mode 100644 vendor/github.com/cpuguy83/strongerrors/LICENSE create mode 100644 vendor/github.com/cpuguy83/strongerrors/defs.go create mode 100644 vendor/github.com/cpuguy83/strongerrors/doc.go create mode 100644 vendor/github.com/cpuguy83/strongerrors/helpers.go create mode 100644 vendor/github.com/cpuguy83/strongerrors/is.go create mode 100644 vendor/github.com/cpuguy83/strongerrors/status/grpc.go create mode 100644 vendor/github.com/cpuguy83/strongerrors/status/http.go diff --git a/Gopkg.lock b/Gopkg.lock index 4fd54bdb9..e100df6d2 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -2,13 +2,13 @@ [[projects]] - digest = "1:ef14b1839bd43a923896a762b31d000c12f0d21506cde7251d9342bb6acb4c05" + digest = "1:6453651bd5fc38b3a07d133c295a881c1bce87d4be4ccefe61eee26d22d6b15c" name = "github.com/Azure/azure-sdk-for-go" packages = [ "profiles/preview/preview/servicefabricmesh/mgmt/servicefabricmesh", "services/batch/2017-09-01.6.0/batch", - "services/preview/servicefabricmesh/mgmt/2018-07-01-preview/servicefabricmesh", "services/network/mgmt/2018-05-01/network", + "services/preview/servicefabricmesh/mgmt/2018-07-01-preview/servicefabricmesh", "version", ] pruneopts = "NUT" @@ -151,6 +151,17 @@ pruneopts = "NUT" revision = "a41693b7b7afb422c7ecb1028458ab27da047bbb" +[[projects]] + digest = "1:7393d591b1707b6bd40171878d13aada2f7fa921cd0a500406d65fd3966ca2f4" + name = "github.com/cpuguy83/strongerrors" + packages = [ + ".", + "status", + ] + pruneopts = "NUT" + revision = "d3f3ceac8165d8532efb6aa406399545de78d967" + version = "v0.2.0" + [[projects]] digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39" name = "github.com/davecgh/go-spew" @@ -1352,6 +1363,7 @@ input-imports = [ "github.com/Azure/azure-sdk-for-go/profiles/preview/preview/servicefabricmesh/mgmt/servicefabricmesh", "github.com/Azure/azure-sdk-for-go/services/batch/2017-09-01.6.0/batch", + "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-05-01/network", "github.com/Azure/go-autorest/autorest", "github.com/Azure/go-autorest/autorest/adal", "github.com/Azure/go-autorest/autorest/azure", @@ -1368,6 +1380,8 @@ "github.com/aws/aws-sdk-go/service/ecs/ecsiface", "github.com/aws/aws-sdk-go/service/iam", "github.com/cenkalti/backoff", + "github.com/cpuguy83/strongerrors", + "github.com/cpuguy83/strongerrors/status", "github.com/dimchansky/utfbom", "github.com/docker/docker/api/types/strslice", "github.com/docker/go-connections/nat", @@ -1432,6 +1446,7 @@ "k8s.io/client-go/kubernetes/fake", "k8s.io/client-go/rest", "k8s.io/client-go/tools/clientcmd", + "k8s.io/client-go/tools/clientcmd/api/v1", "k8s.io/client-go/tools/remotecommand", "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2", "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1", diff --git a/vendor/github.com/cpuguy83/strongerrors/LICENSE b/vendor/github.com/cpuguy83/strongerrors/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/vendor/github.com/cpuguy83/strongerrors/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/cpuguy83/strongerrors/defs.go b/vendor/github.com/cpuguy83/strongerrors/defs.go new file mode 100644 index 000000000..e67ced8fd --- /dev/null +++ b/vendor/github.com/cpuguy83/strongerrors/defs.go @@ -0,0 +1,84 @@ +package strongerrors + +// ErrNotFound signals that the requested object doesn't exist +type ErrNotFound interface { + NotFound() +} + +// ErrInvalidArgument signals that the user input is invalid +type ErrInvalidArgument interface { + InvalidArgument() +} + +// ErrConflict signals that some internal state conflicts with the requested action and can't be performed. +// A change in state should be able to clear this error. +type ErrConflict interface { + Conflict() +} + +// ErrUnauthorized is used to signify that the user is not authorized to perform a specific action +type ErrUnauthorized interface { + Unauthorized() +} + +// ErrUnauthenticated is used to indicate that the caller cannot be identified. +type ErrUnauthenticated interface { + Unauthenticated() +} + +// ErrUnavailable signals that the requested action/subsystem is not available. +type ErrUnavailable interface { + Unavailable() +} + +// ErrForbidden signals that the requested action cannot be performed under any circumstances. +// When a ErrForbidden is returned, the caller should never retry the action. +type ErrForbidden interface { + Forbidden() +} + +// ErrSystem signals that some internal error occurred. +// An example of this would be a failed mount request. +type ErrSystem interface { + System() +} + +// ErrNotModified signals that an action can't be performed because it's already in the desired state +type ErrNotModified interface { + NotModified() +} + +// ErrAlreadyExists is a special case of ErrNotModified which signals that the desired object already exists +type ErrAlreadyExists interface { + AlreadyExists() +} + +// ErrNotImplemented signals that the requested action/feature is not implemented on the system as configured. +type ErrNotImplemented interface { + NotImplemented() +} + +// ErrUnknown signals that the kind of error that occurred is not known. +type ErrUnknown interface { + Unknown() +} + +// ErrCancelled signals that the action was cancelled. +type ErrCancelled interface { + Cancelled() +} + +// ErrDeadline signals that the deadline was reached before the action completed. +type ErrDeadline interface { + DeadlineExceeded() +} + +// ErrExhausted indicates that the action cannot be performed because some resource is exhausted. +type ErrExhausted interface { + Exhausted() +} + +// ErrDataLoss indicates that data was lost or there is data corruption. +type ErrDataLoss interface { + DataLoss() +} diff --git a/vendor/github.com/cpuguy83/strongerrors/doc.go b/vendor/github.com/cpuguy83/strongerrors/doc.go new file mode 100644 index 000000000..f262bf98e --- /dev/null +++ b/vendor/github.com/cpuguy83/strongerrors/doc.go @@ -0,0 +1,12 @@ +// Package strongerrors defines a set of error interfaces that packages should use for communicating classes of errors. +// Errors that cross the package boundary should implement one (and only one) of these interfaces. +// +// Packages should not reference these interfaces directly, only implement them. +// To check if a particular error implements one of these interfaces, there are helper +// functions provided (e.g. `Is`) which can be used rather than asserting the interfaces directly. +// If you must assert on these interfaces, be sure to check the causal chain (`err.Cause()`). +// +// A set of helper functions are provided to take any error and turn it into a specific error class. +// This frees you from defining the same error classes all over your code. However, you can still +// implement the error classes ony our own if you desire. +package strongerrors diff --git a/vendor/github.com/cpuguy83/strongerrors/helpers.go b/vendor/github.com/cpuguy83/strongerrors/helpers.go new file mode 100644 index 000000000..9efe54e25 --- /dev/null +++ b/vendor/github.com/cpuguy83/strongerrors/helpers.go @@ -0,0 +1,272 @@ +package strongerrors + +import "context" + +type errNotFound struct{ error } + +func (errNotFound) NotFound() {} + +func (e errNotFound) Cause() error { + return e.error +} + +// NotFound is a helper to create an error of the class with the same name from any error type +func NotFound(err error) error { + if err == nil { + return nil + } + return errNotFound{err} +} + +type errInvalidArg struct{ error } + +func (errInvalidArg) InvalidArgument() {} + +func (e errInvalidArg) Cause() error { + return e.error +} + +// InvalidArgument is a helper to create an error of the class with the same name from any error type +func InvalidArgument(err error) error { + if err == nil { + return nil + } + return errInvalidArg{err} +} + +type errConflict struct{ error } + +func (errConflict) Conflict() {} + +func (e errConflict) Cause() error { + return e.error +} + +// Conflict is a helper to create an error of the class with the same name from any error type +func Conflict(err error) error { + if err == nil { + return nil + } + return errConflict{err} +} + +type errUnauthorized struct{ error } + +func (errUnauthorized) Unauthorized() {} + +func (e errUnauthorized) Cause() error { + return e.error +} + +// Unauthorized is a helper to create an error of the class with the same name from any error type +func Unauthorized(err error) error { + if err == nil { + return nil + } + return errUnauthorized{err} +} + +type errUnauthenticated struct{ error } + +func (errUnauthenticated) Unauthenticated() {} + +func (e errUnauthenticated) Cause() error { + return e.error +} + +// Unauthenticated is a helper to create an error of the class with the same name from any error type +func Unauthenticated(err error) error { + if err == nil { + return nil + } + return errUnauthenticated{err} +} + +type errUnavailable struct{ error } + +func (errUnavailable) Unavailable() {} + +func (e errUnavailable) Cause() error { + return e.error +} + +// Unavailable is a helper to create an error of the class with the same name from any error type +func Unavailable(err error) error { + return errUnavailable{err} +} + +type errForbidden struct{ error } + +func (errForbidden) Forbidden() {} + +func (e errForbidden) Cause() error { + return e.error +} + +// Forbidden is a helper to create an error of the class with the same name from any error type +func Forbidden(err error) error { + if err == nil { + return nil + } + return errForbidden{err} +} + +type errSystem struct{ error } + +func (errSystem) System() {} + +func (e errSystem) Cause() error { + return e.error +} + +// System is a helper to create an error of the class with the same name from any error type +func System(err error) error { + if err == nil { + return nil + } + return errSystem{err} +} + +type errNotModified struct{ error } + +func (errNotModified) NotModified() {} + +func (e errNotModified) Cause() error { + return e.error +} + +// NotModified is a helper to create an error of the class with the same name from any error type +func NotModified(err error) error { + if err == nil { + return nil + } + return errNotModified{err} +} + +type errAlreadyExists struct{ error } + +func (errAlreadyExists) AlreadyExists() {} + +func (e errAlreadyExists) Cause() error { + return e.error +} + +// AlreadyExists is a helper to create an error of the class with the same name from any error type +func AlreadyExists(err error) error { + if err == nil { + return nil + } + return errAlreadyExists{err} +} + +type errNotImplemented struct{ error } + +func (errNotImplemented) NotImplemented() {} + +func (e errNotImplemented) Cause() error { + return e.error +} + +// NotImplemented is a helper to create an error of the class with the same name from any error type +func NotImplemented(err error) error { + if err == nil { + return nil + } + return errNotImplemented{err} +} + +type errUnknown struct{ error } + +func (errUnknown) Unknown() {} + +func (e errUnknown) Cause() error { + return e.error +} + +// Unknown is a helper to create an error of the class with the same name from any error type +func Unknown(err error) error { + if err == nil { + return nil + } + return errUnknown{err} +} + +type errCancelled struct{ error } + +func (errCancelled) Cancelled() {} + +func (e errCancelled) Cause() error { + return e.error +} + +// Cancelled is a helper to create an error of the class with the same name from any error type +func Cancelled(err error) error { + if err == nil { + return nil + } + return errCancelled{err} +} + +type errDeadline struct{ error } + +func (errDeadline) DeadlineExceeded() {} + +func (e errDeadline) Cause() error { + return e.error +} + +// Deadline is a helper to create an error of the class with the same name from any error type +func Deadline(err error) error { + if err == nil { + return nil + } + return errDeadline{err} +} + +type errExhausted struct{ error } + +func (errExhausted) Exhausted() {} + +func (e errExhausted) Cause() error { + return e.error +} + +// Exhausted is a helper to create an error of the class with the same name from any error type +func Exhausted(err error) error { + if err == nil { + return nil + } + return errExhausted{err} +} + +type errDataLoss struct{ error } + +func (errDataLoss) DataLoss() {} + +func (e errDataLoss) Cause() error { + return e.error +} + +// DataLoss is a helper to create an error of the class with the same name from any error type +func DataLoss(err error) error { + if err == nil { + return nil + } + return errDataLoss{err} +} + +// FromContext returns the error class from the passed in context +func FromContext(ctx context.Context) error { + e := ctx.Err() + if e == nil { + return nil + } + + if e == context.Canceled { + return Cancelled(e) + } + if e == context.DeadlineExceeded { + return Deadline(e) + } + return Unknown(e) +} diff --git a/vendor/github.com/cpuguy83/strongerrors/is.go b/vendor/github.com/cpuguy83/strongerrors/is.go new file mode 100644 index 000000000..2d94546d7 --- /dev/null +++ b/vendor/github.com/cpuguy83/strongerrors/is.go @@ -0,0 +1,128 @@ +package strongerrors + +type causer interface { + Cause() error +} + +func getImplementer(err error) error { + switch e := err.(type) { + case + ErrNotFound, + ErrInvalidArgument, + ErrConflict, + ErrUnauthorized, + ErrUnauthenticated, + ErrUnavailable, + ErrForbidden, + ErrSystem, + ErrNotModified, + ErrAlreadyExists, + ErrNotImplemented, + ErrCancelled, + ErrDeadline, + ErrDataLoss, + ErrExhausted, + ErrUnknown: + return err + case causer: + return getImplementer(e.Cause()) + default: + return err + } +} + +// IsNotFound returns if the passed in error is an ErrNotFound +func IsNotFound(err error) bool { + _, ok := getImplementer(err).(ErrNotFound) + return ok +} + +// IsInvalidArgument returns if the passed in error is an ErrInvalidParameter +func IsInvalidArgument(err error) bool { + _, ok := getImplementer(err).(ErrInvalidArgument) + return ok +} + +// IsConflict returns if the passed in error is an ErrConflict +func IsConflict(err error) bool { + _, ok := getImplementer(err).(ErrConflict) + return ok +} + +// IsUnauthorized returns if the the passed in error is an ErrUnauthorized +func IsUnauthorized(err error) bool { + _, ok := getImplementer(err).(ErrUnauthorized) + return ok +} + +// IsUnauthenticated returns if the the passed in error is an ErrUnauthenticated +func IsUnauthenticated(err error) bool { + _, ok := getImplementer(err).(ErrUnauthenticated) + return ok +} + +// IsUnavailable returns if the passed in error is an ErrUnavailable +func IsUnavailable(err error) bool { + _, ok := getImplementer(err).(ErrUnavailable) + return ok +} + +// IsForbidden returns if the passed in error is an ErrForbidden +func IsForbidden(err error) bool { + _, ok := getImplementer(err).(ErrForbidden) + return ok +} + +// IsSystem returns if the passed in error is an ErrSystem +func IsSystem(err error) bool { + _, ok := getImplementer(err).(ErrSystem) + return ok +} + +// IsNotModified returns if the passed in error is a NotModified error +func IsNotModified(err error) bool { + _, ok := getImplementer(err).(ErrNotModified) + return ok +} + +// IsAlreadyExists returns if the passed in error is a AlreadyExists error +func IsAlreadyExists(err error) bool { + _, ok := getImplementer(err).(ErrAlreadyExists) + return ok +} + +// IsNotImplemented returns if the passed in error is an ErrNotImplemented +func IsNotImplemented(err error) bool { + _, ok := getImplementer(err).(ErrNotImplemented) + return ok +} + +// IsUnknown returns if the passed in error is an ErrUnknown +func IsUnknown(err error) bool { + _, ok := getImplementer(err).(ErrUnknown) + return ok +} + +// IsCancelled returns if the passed in error is an ErrCancelled +func IsCancelled(err error) bool { + _, ok := getImplementer(err).(ErrCancelled) + return ok +} + +// IsDeadline returns if the passed in error is an ErrDeadline +func IsDeadline(err error) bool { + _, ok := getImplementer(err).(ErrDeadline) + return ok +} + +// IsExhausted returns if the passed in error is an ErrDeadline +func IsExhausted(err error) bool { + _, ok := getImplementer(err).(ErrExhausted) + return ok +} + +// IsDataLoss returns if the passed in error is an ErrDataLoss +func IsDataLoss(err error) bool { + _, ok := getImplementer(err).(ErrDataLoss) + return ok +} diff --git a/vendor/github.com/cpuguy83/strongerrors/status/grpc.go b/vendor/github.com/cpuguy83/strongerrors/status/grpc.go new file mode 100644 index 000000000..c446a86a5 --- /dev/null +++ b/vendor/github.com/cpuguy83/strongerrors/status/grpc.go @@ -0,0 +1,85 @@ +package status + +import ( + "github.com/cpuguy83/strongerrors" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// FromGRPC returns an error class from the provided GPRC status +// If the status is nil or OK, this will return nil +// nolint: gocyclo +func FromGRPC(s *status.Status) error { + if s == nil || s.Code() == codes.OK { + return nil + } + + switch s.Code() { + case codes.InvalidArgument: + return strongerrors.InvalidArgument(s.Err()) + case codes.NotFound: + return strongerrors.NotFound(s.Err()) + case codes.Unimplemented: + return strongerrors.NotImplemented(s.Err()) + case codes.DeadlineExceeded: + return strongerrors.Deadline(s.Err()) + case codes.Canceled: + return strongerrors.Cancelled(s.Err()) + case codes.AlreadyExists: + return strongerrors.AlreadyExists(s.Err()) + case codes.PermissionDenied: + return strongerrors.Unauthorized(s.Err()) + case codes.Unauthenticated: + return strongerrors.Unauthenticated(s.Err()) + // TODO(cpuguy83): consider more granular errors for these cases + case codes.FailedPrecondition, codes.Aborted, codes.Unavailable, codes.OutOfRange: + return strongerrors.Conflict(s.Err()) + case codes.ResourceExhausted: + return strongerrors.Exhausted(s.Err()) + case codes.DataLoss: + return strongerrors.DataLoss(s.Err()) + default: + return strongerrors.Unknown(s.Err()) + } +} + +// ToGRPC takes the passed in error and converts it to a GRPC status error +// If the passed in error is already a gprc status error, then it is returned unmodified +// If the passed in error is nil, then a nil error is returned. +// nolint: gocyclo +func ToGRPC(err error) error { + if _, ok := status.FromError(err); ok { + return err + } + + switch { + case strongerrors.IsNotFound(err): + return status.Error(codes.NotFound, err.Error()) + case strongerrors.IsConflict(err), strongerrors.IsNotModified(err): + return status.Error(codes.FailedPrecondition, err.Error()) + case strongerrors.IsInvalidArgument(err): + return status.Error(codes.InvalidArgument, err.Error()) + case strongerrors.IsAlreadyExists(err): + return status.Error(codes.AlreadyExists, err.Error()) + case strongerrors.IsCancelled(err): + return status.Error(codes.Canceled, err.Error()) + case strongerrors.IsDeadline(err): + return status.Error(codes.DeadlineExceeded, err.Error()) + case strongerrors.IsUnauthorized(err): + return status.Error(codes.PermissionDenied, err.Error()) + case strongerrors.IsUnauthenticated(err): + return status.Error(codes.Unauthenticated, err.Error()) + case strongerrors.IsForbidden(err), strongerrors.IsNotImplemented(err): + return status.Error(codes.Unimplemented, err.Error()) + case strongerrors.IsExhausted(err): + return status.Error(codes.ResourceExhausted, err.Error()) + case strongerrors.IsDataLoss(err): + return status.Error(codes.DataLoss, err.Error()) + case strongerrors.IsSystem(err): + return status.Error(codes.Internal, err.Error()) + case strongerrors.IsUnavailable(err): + return status.Error(codes.Unavailable, err.Error()) + default: + return status.Error(codes.Unknown, err.Error()) + } +} diff --git a/vendor/github.com/cpuguy83/strongerrors/status/http.go b/vendor/github.com/cpuguy83/strongerrors/status/http.go new file mode 100644 index 000000000..fa970ce61 --- /dev/null +++ b/vendor/github.com/cpuguy83/strongerrors/status/http.go @@ -0,0 +1,37 @@ +package status + +import ( + "net/http" + + "github.com/cpuguy83/strongerrors" +) + +// HTTPCode takes an error and returns the HTTP status code for the given error +// If a match is found then the second return argument will be true, otherwise it will be false. +// nolint: gocyclo +func HTTPCode(err error) (int, bool) { + switch { + case strongerrors.IsNotFound(err): + return http.StatusNotFound, true + case strongerrors.IsInvalidArgument(err): + return http.StatusBadRequest, true + case strongerrors.IsConflict(err): + return http.StatusConflict, true + case strongerrors.IsUnauthenticated(err), strongerrors.IsForbidden(err): + return http.StatusForbidden, true + case strongerrors.IsUnauthorized(err): + return http.StatusUnauthorized, true + case strongerrors.IsUnavailable(err): + return http.StatusServiceUnavailable, true + case strongerrors.IsForbidden(err): + return http.StatusForbidden, true + case strongerrors.IsAlreadyExists(err), strongerrors.IsNotModified(err): + return http.StatusNotModified, true + case strongerrors.IsNotImplemented(err): + return http.StatusNotImplemented, true + case strongerrors.IsSystem(err) || strongerrors.IsUnknown(err) || strongerrors.IsDataLoss(err) || strongerrors.IsExhausted(err): + return http.StatusInternalServerError, true + default: + return http.StatusInternalServerError, false + } +} diff --git a/vkubelet/apiserver.go b/vkubelet/apiserver.go index f69c18933..59bc280c8 100644 --- a/vkubelet/apiserver.go +++ b/vkubelet/apiserver.go @@ -12,13 +12,15 @@ import ( "time" "github.com/Sirupsen/logrus" + "github.com/cpuguy83/strongerrors" + "github.com/cpuguy83/strongerrors/status" "github.com/gorilla/mux" "github.com/pkg/errors" "github.com/virtual-kubelet/virtual-kubelet/log" "k8s.io/kubernetes/pkg/kubelet/server/remotecommand" ) -func loggingContext(r *http.Request) context.Context { +func instrumentContext(r *http.Request) context.Context { ctx := r.Context() logger := log.G(ctx).WithFields(logrus.Fields{ "uri": r.RequestURI, @@ -29,19 +31,50 @@ func loggingContext(r *http.Request) context.Context { // NotFound provides a handler for cases where the requested endpoint doesn't exist func NotFound(w http.ResponseWriter, r *http.Request) { - logger := log.G(loggingContext(r)) + logger := log.G(instrumentContext(r)) log.Trace(logger, "404 request not found") http.Error(w, "404 request not found", http.StatusNotFound) } // NotImplemented provides a handler for cases where a provider does not implement a given API func NotImplemented(w http.ResponseWriter, r *http.Request) { - logger := log.G(loggingContext(r)) + logger := log.G(instrumentContext(r)) log.Trace(logger, "501 not implemented") http.Error(w, "501 not implemented", http.StatusNotImplemented) } -// KubeletServertStart starts the virtual kubelet HTTP server. +type handlerFunc func(http.ResponseWriter, *http.Request) error + +// InstrumentHandler wraps an http.Handler and injects instrumentation into the request context. +func InstrumentHandler(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + ctx := instrumentContext(req) + req = req.WithContext(ctx) + h.ServeHTTP(w, req) + }) +} + +func handleError(f handlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + err := f(w, req) + if err == nil { + return + } + + code, _ := status.HTTPCode(err) + w.WriteHeader(code) + io.WriteString(w, err.Error()) + logger := log.G(req.Context()).WithError(err).WithField("httpStatusCode", code) + + if code >= 500 { + logger.Error("Internal server error on request") + } else { + log.Trace(logger, "Error on request") + } + } +} + +// KubeletServerStart starts the virtual kubelet HTTP server. func KubeletServerStart(p Provider) { certFilePath := os.Getenv("APISERVER_CERT_LOCATION") keyFilePath := os.Getenv("APISERVER_KEY_LOCATION") @@ -53,7 +86,7 @@ func KubeletServerStart(p Provider) { r.HandleFunc("/exec/{namespace}/{pod}/{container}", PodExecHandlerFunc(p)).Methods("POST") r.NotFoundHandler = http.HandlerFunc(NotFound) - if err := http.ListenAndServeTLS(addr, certFilePath, keyFilePath, r); err != nil { + if err := http.ListenAndServeTLS(addr, certFilePath, keyFilePath, InstrumentHandler(r)); err != nil { log.G(context.TODO()).WithError(err).Error("error setting up http server") } } @@ -72,49 +105,43 @@ func MetricsServerStart(p Provider, addr string) { r.HandleFunc("/stats/summary/", PodMetricsHandlerFunc(mp)).Methods("GET") } r.NotFoundHandler = http.HandlerFunc(NotFound) - if err := http.ListenAndServe(addr, r); err != nil { + if err := http.ListenAndServe(addr, InstrumentHandler(r)); err != nil { log.G(context.TODO()).WithError(err).Error("Error starting http server") } } // PodMetricsHandlerFunc makes an HTTP handler for implementing the kubelet summary stats endpoint func PodMetricsHandlerFunc(mp MetricsProvider) http.HandlerFunc { - return func(w http.ResponseWriter, req *http.Request) { - ctx := loggingContext(req) - + return handleError(func(w http.ResponseWriter, req *http.Request) error { stats, err := mp.GetStatsSummary(req.Context()) if err != nil { if errors.Cause(err) == context.Canceled { - return + return strongerrors.Cancelled(err) } - log.G(ctx).Error("Error getting stats from provider:", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return + return strongerrors.Unknown(errors.Wrap(err, "error getting status from provider")) } b, err := json.Marshal(stats) if err != nil { - log.G(ctx).WithError(err).Error("Could not marshal stats") - http.Error(w, "could not marshal stats: "+err.Error(), http.StatusInternalServerError) - return + return strongerrors.Unknown(errors.Wrap(err, "error marshalling stats")) } if _, err := w.Write(b); err != nil { - log.G(ctx).WithError(err).Debug("Could not write to client") + return strongerrors.Unknown(errors.Wrap(err, "could not write to client")) } - } + return nil + }) } // PodLogsHandlerFunc creates an http handler function from a provider to serve logs from a pod func PodLogsHandlerFunc(p Provider) http.HandlerFunc { - return func(w http.ResponseWriter, req *http.Request) { + return handleError(func(w http.ResponseWriter, req *http.Request) error { vars := mux.Vars(req) if len(vars) != 3 { - NotFound(w, req) - return + return strongerrors.NotFound(errors.New("not found")) } - ctx := loggingContext(req) + ctx := req.Context() namespace := vars["namespace"] pod := vars["pod"] @@ -125,25 +152,21 @@ func PodLogsHandlerFunc(p Provider) http.HandlerFunc { if queryTail := q.Get("tailLines"); queryTail != "" { t, err := strconv.Atoi(queryTail) if err != nil { - logger := log.G(context.TODO()).WithError(err) - log.Trace(logger, "could not parse tailLines") - http.Error(w, fmt.Sprintf("could not parse \"tailLines\": %v", err), http.StatusBadRequest) - return + return strongerrors.InvalidArgument(errors.Wrap(err, "could not parse \"tailLines\"")) } tail = t } podsLogs, err := p.GetContainerLogs(ctx, namespace, pod, container, tail) if err != nil { - log.G(ctx).WithError(err).Error("error getting container logs") - http.Error(w, fmt.Sprintf("error while getting container logs: %v", err), http.StatusInternalServerError) - return + return errors.Wrap(err, "error getting container logs?)") } if _, err := io.WriteString(w, podsLogs); err != nil { - log.G(ctx).WithError(err).Warn("error writing response to client") + return strongerrors.Unknown(errors.Wrap(err, "error writing response to client")) } - } + return nil + }) } // PodExecHandlerFunc makes an http handler func from a Provider which execs a command in a pod's container From da5e24ef4db3e94b6348d2611348c29e0e549d44 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Tue, 18 Sep 2018 10:05:54 -0700 Subject: [PATCH 3/5] Move API handlers to separate package This makes the package split a little cleaner and easier to import the HTTP handlers for other consumers. --- vkubelet/api/exec.go | 42 ++++++++++ vkubelet/api/helpers.go | 31 +++++++ vkubelet/api/logs.go | 53 ++++++++++++ vkubelet/api/stats.go | 39 +++++++++ vkubelet/apiserver.go | 178 +++++++--------------------------------- vkubelet/provider.go | 4 +- 6 files changed, 197 insertions(+), 150 deletions(-) create mode 100644 vkubelet/api/exec.go create mode 100644 vkubelet/api/helpers.go create mode 100644 vkubelet/api/logs.go create mode 100644 vkubelet/api/stats.go diff --git a/vkubelet/api/exec.go b/vkubelet/api/exec.go new file mode 100644 index 000000000..c52816443 --- /dev/null +++ b/vkubelet/api/exec.go @@ -0,0 +1,42 @@ +package api + +import ( + "fmt" + "net/http" + "strings" + "time" + + "github.com/gorilla/mux" + "k8s.io/kubernetes/pkg/kubelet/server/remotecommand" +) + +// PodExecHandlerFunc makes an http handler func from a Provider which execs a command in a pod's container +// Note that this handler currently depends on gorrilla/mux to get url parts as variables. +// TODO(@cpuguy83): don't force gorilla/mux on consumers of this function +func PodExecHandlerFunc(backend remotecommand.Executor) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + vars := mux.Vars(req) + + namespace := vars["namespace"] + pod := vars["pod"] + container := vars["container"] + + supportedStreamProtocols := strings.Split(req.Header.Get("X-Stream-Protocol-Version"), ",") + + q := req.URL.Query() + command := q["command"] + + // TODO: tty flag causes remotecommand.createStreams to wait for the wrong number of streams + streamOpts := &remotecommand.Options{ + Stdin: true, + Stdout: true, + Stderr: true, + TTY: false, + } + + idleTimeout := time.Second * 30 + streamCreationTimeout := time.Second * 30 + + remotecommand.ServeExec(w, req, backend, fmt.Sprintf("%s-%s", namespace, pod), "", container, command, streamOpts, idleTimeout, streamCreationTimeout, supportedStreamProtocols) + } +} diff --git a/vkubelet/api/helpers.go b/vkubelet/api/helpers.go new file mode 100644 index 000000000..95e651b56 --- /dev/null +++ b/vkubelet/api/helpers.go @@ -0,0 +1,31 @@ +package api + +import ( + "io" + "net/http" + + "github.com/cpuguy83/strongerrors/status" + "github.com/virtual-kubelet/virtual-kubelet/log" +) + +type handlerFunc func(http.ResponseWriter, *http.Request) error + +func handleError(f handlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + err := f(w, req) + if err == nil { + return + } + + code, _ := status.HTTPCode(err) + w.WriteHeader(code) + io.WriteString(w, err.Error()) + logger := log.G(req.Context()).WithError(err).WithField("httpStatusCode", code) + + if code >= 500 { + logger.Error("Internal server error on request") + } else { + log.Trace(logger, "Error on request") + } + } +} diff --git a/vkubelet/api/logs.go b/vkubelet/api/logs.go new file mode 100644 index 000000000..d7eec9607 --- /dev/null +++ b/vkubelet/api/logs.go @@ -0,0 +1,53 @@ +package api + +import ( + "context" + "io" + "net/http" + "strconv" + + "github.com/cpuguy83/strongerrors" + "github.com/gorilla/mux" + "github.com/pkg/errors" +) + +// ContainerLogsBackend is used in place of backend implementations for getting container logs +type ContainerLogsBackend interface { + GetContainerLogs(ctx context.Context, namespace, podName, containerName string, tail int) (string, error) +} + +// PodLogsHandlerFunc creates an http handler function from a provider to serve logs from a pod +func PodLogsHandlerFunc(p ContainerLogsBackend) http.HandlerFunc { + return handleError(func(w http.ResponseWriter, req *http.Request) error { + vars := mux.Vars(req) + if len(vars) != 3 { + return strongerrors.NotFound(errors.New("not found")) + } + + ctx := req.Context() + + namespace := vars["namespace"] + pod := vars["pod"] + container := vars["container"] + tail := 10 + q := req.URL.Query() + + if queryTail := q.Get("tailLines"); queryTail != "" { + t, err := strconv.Atoi(queryTail) + if err != nil { + return strongerrors.InvalidArgument(errors.Wrap(err, "could not parse \"tailLines\"")) + } + tail = t + } + + podsLogs, err := p.GetContainerLogs(ctx, namespace, pod, container, tail) + if err != nil { + return errors.Wrap(err, "error getting container logs?)") + } + + if _, err := io.WriteString(w, podsLogs); err != nil { + return strongerrors.Unknown(errors.Wrap(err, "error writing response to client")) + } + return nil + }) +} diff --git a/vkubelet/api/stats.go b/vkubelet/api/stats.go new file mode 100644 index 000000000..afa7b23a7 --- /dev/null +++ b/vkubelet/api/stats.go @@ -0,0 +1,39 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + + "github.com/cpuguy83/strongerrors" + "github.com/pkg/errors" + stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" +) + +// PodMetricsBackend is used in place of backend implementations to get k8s pod metrics. +type PodMetricsBackend interface { + GetStatsSummary(context.Context) (*stats.Summary, error) +} + +// PodMetricsHandlerFunc makes an HTTP handler for implementing the kubelet summary stats endpoint +func PodMetricsHandlerFunc(b PodMetricsBackend) http.HandlerFunc { + return handleError(func(w http.ResponseWriter, req *http.Request) error { + stats, err := b.GetStatsSummary(req.Context()) + if err != nil { + if errors.Cause(err) == context.Canceled { + return strongerrors.Cancelled(err) + } + return errors.Wrap(err, "error getting status from provider") + } + + b, err := json.Marshal(stats) + if err != nil { + return strongerrors.Unknown(errors.Wrap(err, "error marshalling stats")) + } + + if _, err := w.Write(b); err != nil { + return strongerrors.Unknown(errors.Wrap(err, "could not write to client")) + } + return nil + }) +} diff --git a/vkubelet/apiserver.go b/vkubelet/apiserver.go index 59bc280c8..539f837d2 100644 --- a/vkubelet/apiserver.go +++ b/vkubelet/apiserver.go @@ -2,78 +2,16 @@ package vkubelet import ( "context" - "encoding/json" "fmt" - "io" "net/http" "os" - "strconv" - "strings" - "time" "github.com/Sirupsen/logrus" - "github.com/cpuguy83/strongerrors" - "github.com/cpuguy83/strongerrors/status" "github.com/gorilla/mux" - "github.com/pkg/errors" "github.com/virtual-kubelet/virtual-kubelet/log" - "k8s.io/kubernetes/pkg/kubelet/server/remotecommand" + "github.com/virtual-kubelet/virtual-kubelet/vkubelet/api" ) -func instrumentContext(r *http.Request) context.Context { - ctx := r.Context() - logger := log.G(ctx).WithFields(logrus.Fields{ - "uri": r.RequestURI, - "vars": mux.Vars(r), - }) - return log.WithLogger(ctx, logger) -} - -// NotFound provides a handler for cases where the requested endpoint doesn't exist -func NotFound(w http.ResponseWriter, r *http.Request) { - logger := log.G(instrumentContext(r)) - log.Trace(logger, "404 request not found") - http.Error(w, "404 request not found", http.StatusNotFound) -} - -// NotImplemented provides a handler for cases where a provider does not implement a given API -func NotImplemented(w http.ResponseWriter, r *http.Request) { - logger := log.G(instrumentContext(r)) - log.Trace(logger, "501 not implemented") - http.Error(w, "501 not implemented", http.StatusNotImplemented) -} - -type handlerFunc func(http.ResponseWriter, *http.Request) error - -// InstrumentHandler wraps an http.Handler and injects instrumentation into the request context. -func InstrumentHandler(h http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - ctx := instrumentContext(req) - req = req.WithContext(ctx) - h.ServeHTTP(w, req) - }) -} - -func handleError(f handlerFunc) http.HandlerFunc { - return func(w http.ResponseWriter, req *http.Request) { - err := f(w, req) - if err == nil { - return - } - - code, _ := status.HTTPCode(err) - w.WriteHeader(code) - io.WriteString(w, err.Error()) - logger := log.G(req.Context()).WithError(err).WithField("httpStatusCode", code) - - if code >= 500 { - logger.Error("Internal server error on request") - } else { - log.Trace(logger, "Error on request") - } - } -} - // KubeletServerStart starts the virtual kubelet HTTP server. func KubeletServerStart(p Provider) { certFilePath := os.Getenv("APISERVER_CERT_LOCATION") @@ -82,8 +20,8 @@ func KubeletServerStart(p Provider) { addr := fmt.Sprintf(":%s", port) r := mux.NewRouter() - r.HandleFunc("/containerLogs/{namespace}/{pod}/{container}", PodLogsHandlerFunc(p)).Methods("GET") - r.HandleFunc("/exec/{namespace}/{pod}/{container}", PodExecHandlerFunc(p)).Methods("POST") + r.HandleFunc("/containerLogs/{namespace}/{pod}/{container}", api.PodLogsHandlerFunc(p)).Methods("GET") + r.HandleFunc("/exec/{namespace}/{pod}/{container}", api.PodExecHandlerFunc(p)).Methods("POST") r.NotFoundHandler = http.HandlerFunc(NotFound) if err := http.ListenAndServeTLS(addr, certFilePath, keyFilePath, InstrumentHandler(r)); err != nil { @@ -96,13 +34,13 @@ func KubeletServerStart(p Provider) { func MetricsServerStart(p Provider, addr string) { r := mux.NewRouter() - mp, ok := p.(MetricsProvider) + mp, ok := p.(PodMetricsProvider) if !ok { r.HandleFunc("/stats/summary", NotImplemented).Methods("GET") r.HandleFunc("/stats/summary/", NotImplemented).Methods("GET") } else { - r.HandleFunc("/stats/summary", PodMetricsHandlerFunc(mp)).Methods("GET") - r.HandleFunc("/stats/summary/", PodMetricsHandlerFunc(mp)).Methods("GET") + r.HandleFunc("/stats/summary", api.PodMetricsHandlerFunc(mp)).Methods("GET") + r.HandleFunc("/stats/summary/", api.PodMetricsHandlerFunc(mp)).Methods("GET") } r.NotFoundHandler = http.HandlerFunc(NotFound) if err := http.ListenAndServe(addr, InstrumentHandler(r)); err != nil { @@ -110,90 +48,34 @@ func MetricsServerStart(p Provider, addr string) { } } -// PodMetricsHandlerFunc makes an HTTP handler for implementing the kubelet summary stats endpoint -func PodMetricsHandlerFunc(mp MetricsProvider) http.HandlerFunc { - return handleError(func(w http.ResponseWriter, req *http.Request) error { - stats, err := mp.GetStatsSummary(req.Context()) - if err != nil { - if errors.Cause(err) == context.Canceled { - return strongerrors.Cancelled(err) - } - return strongerrors.Unknown(errors.Wrap(err, "error getting status from provider")) - } +func instrumentRequest(r *http.Request) context.Context { + ctx := r.Context() + logger := log.G(ctx).WithFields(logrus.Fields{ + "uri": r.RequestURI, + "vars": mux.Vars(r), + }) + return log.WithLogger(ctx, logger) +} - b, err := json.Marshal(stats) - if err != nil { - return strongerrors.Unknown(errors.Wrap(err, "error marshalling stats")) - } - - if _, err := w.Write(b); err != nil { - return strongerrors.Unknown(errors.Wrap(err, "could not write to client")) - } - return nil +// InstrumentHandler wraps an http.Handler and injects instrumentation into the request context. +func InstrumentHandler(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + ctx := instrumentRequest(req) + req = req.WithContext(ctx) + h.ServeHTTP(w, req) }) } -// PodLogsHandlerFunc creates an http handler function from a provider to serve logs from a pod -func PodLogsHandlerFunc(p Provider) http.HandlerFunc { - return handleError(func(w http.ResponseWriter, req *http.Request) error { - vars := mux.Vars(req) - if len(vars) != 3 { - return strongerrors.NotFound(errors.New("not found")) - } - - ctx := req.Context() - - namespace := vars["namespace"] - pod := vars["pod"] - container := vars["container"] - tail := 10 - q := req.URL.Query() - - if queryTail := q.Get("tailLines"); queryTail != "" { - t, err := strconv.Atoi(queryTail) - if err != nil { - return strongerrors.InvalidArgument(errors.Wrap(err, "could not parse \"tailLines\"")) - } - tail = t - } - - podsLogs, err := p.GetContainerLogs(ctx, namespace, pod, container, tail) - if err != nil { - return errors.Wrap(err, "error getting container logs?)") - } - - if _, err := io.WriteString(w, podsLogs); err != nil { - return strongerrors.Unknown(errors.Wrap(err, "error writing response to client")) - } - return nil - }) +// NotFound provides a handler for cases where the requested endpoint doesn't exist +func NotFound(w http.ResponseWriter, r *http.Request) { + logger := log.G(r.Context()) + log.Trace(logger, "404 request not found") + http.Error(w, "404 request not found", http.StatusNotFound) } -// PodExecHandlerFunc makes an http handler func from a Provider which execs a command in a pod's container -func PodExecHandlerFunc(p Provider) http.HandlerFunc { - return func(w http.ResponseWriter, req *http.Request) { - vars := mux.Vars(req) - - namespace := vars["namespace"] - pod := vars["pod"] - container := vars["container"] - - supportedStreamProtocols := strings.Split(req.Header.Get("X-Stream-Protocol-Version"), ",") - - q := req.URL.Query() - command := q["command"] - - // TODO: tty flag causes remotecommand.createStreams to wait for the wrong number of streams - streamOpts := &remotecommand.Options{ - Stdin: true, - Stdout: true, - Stderr: true, - TTY: false, - } - - idleTimeout := time.Second * 30 - streamCreationTimeout := time.Second * 30 - - remotecommand.ServeExec(w, req, p, fmt.Sprintf("%s-%s", namespace, pod), "", container, command, streamOpts, idleTimeout, streamCreationTimeout, supportedStreamProtocols) - } +// NotImplemented provides a handler for cases where a provider does not implement a given API +func NotImplemented(w http.ResponseWriter, r *http.Request) { + logger := log.G(r.Context()) + log.Trace(logger, "501 not implemented") + http.Error(w, "501 not implemented", http.StatusNotImplemented) } diff --git a/vkubelet/provider.go b/vkubelet/provider.go index 6ace5e839..873ba84a1 100644 --- a/vkubelet/provider.go +++ b/vkubelet/provider.go @@ -57,7 +57,7 @@ type Provider interface { OperatingSystem() string } -// MetricsProvider is an optional interface that providers can implement to expose pod stats -type MetricsProvider interface { +// PodMetricsProvider is an optional interface that providers can implement to expose pod stats +type PodMetricsProvider interface { GetStatsSummary(context.Context) (*stats.Summary, error) } From ddf955e6e12539d701e384c78472563d7377a2d8 Mon Sep 17 00:00:00 2001 From: Anubhav Mishra Date: Sun, 23 Sep 2018 13:50:36 -0700 Subject: [PATCH 4/5] Fix typo. (#357) Thanks for the contribution :) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 95cf4d659..52e03ed91 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ leveraging the portable Kubernetes API. Each provider may have its own configuration file, and required environmental variables. Providers must provide the following functionality to be considered a supported integration with Virtual Kubelet. -1. Provides the back-end plumbing necessary to support the lifecycle management of pods, containers and supporting resources in the context of Kubernete. +1. Provides the back-end plumbing necessary to support the lifecycle management of pods, containers and supporting resources in the context of Kubernetes. 2. Conforms to the current API provided by Virtual Kubelet. 3. Does not have access to the Kubernetes API Server and has a well-defined callback mechanism for getting data like secrets or configmaps. From f7eb898a000c98ffe63a495f3a2f70e63959f833 Mon Sep 17 00:00:00 2001 From: Robbie Zhang Date: Mon, 24 Sep 2018 12:32:49 -0700 Subject: [PATCH 5/5] Update virtual-kubelet helm chart to support ACI VNet (#356) * Update the virtual-kubelet helm chart for ACI VNet * Fix the typo * version * secret encoding * Update the version to 0.4.0 * Add ClusterResourceId for LogAnalytics --- charts/virtual-kubelet-0.4.0.tgz | Bin 0 -> 3509 bytes charts/virtual-kubelet-latest.tgz | Bin 3280 -> 3509 bytes charts/virtual-kubelet/Chart.yaml | 4 ++-- .../virtual-kubelet/templates/deployment.yaml | 17 +++++++++++++++++ charts/virtual-kubelet/templates/secret.yaml | 2 +- charts/virtual-kubelet/values.yaml | 10 +++++++++- 6 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 charts/virtual-kubelet-0.4.0.tgz diff --git a/charts/virtual-kubelet-0.4.0.tgz b/charts/virtual-kubelet-0.4.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..6bb05c074358d81c514259c96a918ce524eea903 GIT binary patch literal 3509 zcmV;m4NCGKiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PH(|ZyPzT`?rC72V%MlG>3TN{Op5)1)l4;T_dRz*iIkrt`CVk zL)+SDq^YEF>dof6AEfy`9>-~$=1?H|Cyqps6h%@L#aU3PBf^?D(F7Ul-R_)7?Jh~c zetELiYPDK#j*jf#R;yM0+d3-zw*7mDd&h5nfv4l(ffgxE_RZ=E!)sxw`8fd8e5H5ui!eAVnKbB*P!Mi zVm?d|1V<24Ah&7<$X}7fabPXksc_@OgXe0z49)Nv z<ss}Trp7J|N+rb6PGL1(K$LaIC7*m9>VL+>1cZggr1|kr3zt6n8FyRCUST=#t= z(ER^I+IALlQUp-ZBs+fcULz;s(ILjZ`@IxVcnyU6N%xEjehow-NcX%%5gEW?;~1jL+Umy)I(Ow0%;Y^kZI5{VXcq#zT3TOn_(2Ih+? zbQS~w>3Ny*luuZ2)EmgCXu?s)R#NWkQeUpnD0dqDJ>jY!#%llS74(y1{sgBaVj6~T zJ6({1p|V1sQ0`McQ(Fx=A)aL|2@LYRSFy@;JWoVC;jG4)ym}QQCn1I8Xtl~3(kv`l z>`hRUy#%@&ufTEx);5rPmZIqy;?pvk}?VQ3&z z6$NK;;~E?exbcCy)wsKBn$zqx7dkt1OQ-&}bzt@5r?}jP`+I|DAhh+Qmel6 zoFp2OGu5|8%g0mrV?4e9MwLd6(uGbAeM^iOIZC1v@TG`c7|&_3KLe-A;#U#X#=+YO zp%i}PnWGRQg>Je)-qil$Tg1)OF9o3VsuY_Tzi zU5Umskc`eWVr&VYBf{vEq7NhtE${ADc_YB z{y#bCO;nx4``?-s+7$o2Y3-N&|Nh?IaqGqZKS#T}Gx1XDTFiCMO>?|Pk&E{*$BvRn z&xNFa5fhfUH~*ttDs~Ggju7OX=fD#&Ag{X+L_!mwazF_+k6sYaJa_?0@gg;o(dE_gPxe!10Sl zZhorWEawvI=SijuKh$DvhDrpX;K+3wvNVwiqtYBTqaIOhu<4+bXp$T&aJHG^aYk_+ z)p&$(F0{4UJk@8y2J~)J6znSWBx);yl;CR{iovh$IAPCyOB7xZJqIU&w-)7WTYl7> z>>nMgD99MfmK$@Hy@91&%}0B!xWzgCQX<-Rxp~ST?(f?x!0@2oezx;P(>^U*wUzSr76YB{U(Iv4#>Z}_n{yz2Ibz?&{4{?;fW^G0>>UD>`RYk6`^!PwH#buqfZ}RhHql%k- znv6LeS=@Eo+0Rw!{qX#9)hERZu0D~iQ4WLceZ4+pSZ48MAkjXi@BtxylNosu&B0Cy zTW3X&U3T_{=&Y=ts%+GGBJotHSvl2^Da)^7y$g=|K{sWX7+t;{^u||%&WAOwJLe>S zPOPwE2^B^+(CYS2hHI3ffT>G;S&QjI$9lcXVSf|D*%NJr-@LyLf9h7(vzb#Uyqwm_ zq&Qqh%A9joCxcPl`Cp-wJ;&D3$p<&u(kAIugp=!44!o*u>F*4kU{Qb{%ynATgm?v` zEHV)o_&ADj>Wta>s#Cp?z^I(*8;&*NsbnOfUC4o+jRRcK9rcad_X<9I62OKRUbav{wu`^QP7~qn|Ws)r`Tusr+P6cI_ z7D;fXb5<*GYG(@{?Lk&$C@uM^XDA!3qO2%GTFtX62_|MuW`<(Q)d_WYYb7dFW!-r> zQu*eTZFy`jt9Q9tLrE@|MWS)n;E~tvA7w(D3}`M?Y&}KRLu*dfnr^aJmD)HFK-+?@ zca-5KORK9hc`&JI_v-<~LhG{a`KWpaFo+>4RB z7GVsOq{~)ePLmu|CAcji{TvC6^(s<>ehqYqQ29pnT*;)B+>hlq-QK%TPF_EH^2uq# zC+9cy$=NCNpW-qfxOd=n9z_*uPm6(+eZemhb}B3u=WW*_G?M!!jAr6C{GD$2-^tcI zHY-(3YgkF8brOz~W@ikA_4o_wR7n?<1rMK`G}%)Sd)6CsP#!5~naX2nDy!T}JWUsP zLZuBXmObu=f^vN+Sqztcox;S7ayP5V7o1U!_G6V8Pg;oafp9WIxfV}g$L)(=9ChU= zDy+PhB%J42!!nwg8FX(c_rP4)!~trQ1|<(&i{wvKO0z+ zII#YnIB-+_$No{}`=9;RLF=Xd<5}8Q)PEGi!*UtL4-6BZtkB`5)wcEauTg3BX^xDE zTOpbM?H*u*{qG$dy{X#&@$tcn{Xa*mH}DJL*IIR5EoO5~@nzE}kn_#OAL-|XtMwPo z99iJ1`R-hZ8z+Wrs>t(DsY0ztrEsE|mO)sJZd#RYC#f2!t8GZXWrXYu5(OC8Pb!_3yZ6435%j{S4STJuT)BdHVEP9N^B4~aZO z+uCTPsibl0&E~rwr1?D_$7!49P$2pzjzp0ZMN$;SSx~7X!kRbH1R3hx?wm;NE=jU;i6WPYNZbc3J~~lK|T=6q5->_}859StIVMsCYw(NP1{draZ;0 zSo&3isLo!b(rUSfZmaR%-oNi{E6XJ;LiMx`V1xbd9~>T3?f>xL@WuY0qtSrOu)Wm) zkQfR@wUEm;WVTgRe*>t=tPPB4RQBI6V(daN>Mh&QVSl$|skRzhjaRSWI+S8TeU#Ur z<|1M~Ob`S|5K|zxY6r+)k;LTBglKQR1E@XMb+;Pn)HZB2xbSgcE!nAXo~=w`Jai|87`2S-&+mid$g4e zax!7)w|_y)2u;V0fv<4mQn8!zZD6b^wZXZ&1NUQWR_QR@-#Z4xNC~G?^z=0Zk(I@D zfKI{3kclPbGt=2IphZ_cLb;$G!t2iIXnO~Qo`0jfr3zt6n8FyRCUS zT=#t=(ER^I+IALlQUp-ZBs+fcULz;s(ILjZ`@IxVcnyU6N%xEjehow-NcX%%5gEW?;~1jL+Umy)I(Ow0%;Y^kZI5{VXcq#zT3TOn_( z27l&@DRdSD0qJ>}@{~_laMT;fsA$4b$5vAA>r!8?&?t8r{XOBTAI56`>J{{pWBvrE zBw`wdZ#!L(f}yfPpHS{oK2uu_IU$~9EeQdzPZ<8Gqumoi4IT{DAVUpZoo@UT&^7M56-5ilaWJnac*_jRZ_l6G$g@3Q0 z$-p6DXdqJ+1!r;N8XOL|@qxP4xVvkb)9f`DIy-brr~bBeVD;mtxZH;OdxK{nwDqLp zvynOLAtcnCBpQ-4)wf5>$5Z%YJiY)%l}3)zg-#BAON-qihrh4`URY3 z0h_UeW^A!Bhh2%rGmwnVG-7NCpCiKPl%fwL3@z{OR^$ILCSfON*gv!jQ7}P|L<&KP zpcDk#B~K|bU)fNT(SRBgHc_-O>erK{701b7WMGZ^R3UoX>m*}Xt4QZpOQZe!fBo1}q0k`cgO&hHY4X z_*7)~-?r+{vg`|HG6PCC)|D#+gb_*$v7$}<-+s_Te=WDB0mix)n(tvW0MSj`MxbQbn?jzT*Z{bul`4c^DoK6#B4o0O*>ElVpa7iX< zU9h^+L_!mwazF_+k6sYaJa_?0@gg z;o(dE_gPxe!10SlZhorWEawvI=SijuKh$DvhDrpX;K+3wvNVwiqtYBTqaIOhu<4+b zXp$T&aJHG^aYk_+)p&$(F0{4UJk@8y2J~)J6znSWBx);ylz-rB8;ZfN?l@u3eM=Nx z5IqMcfwvarY+HWRoa`SRt0>4A%9a~*mc4(t+>TG{!${^cDZ@VAMWqlE5n-1 z((tE%=%SGCUzw6W29!8Jz9@Gpx8z2t-R=YD7716@DSqD-ao$@?+q^N(SFPqdUDmjSi$#B zilANWh4`A09wjRSMwi;4G|`cAL*W;3t`Hq}`h&4qP|Z3D`)40J+#4)!+!|~jJ7<@@ zdVD%$6`Xbqetity-n)0bZk;ZyFdDyjWRW+W0)Ae7W`A+)qlc4$-BLYYfl{u8D&lgB z8GbiOxz{e>?EL*zXV5wO=eXY;U3D(Ue|)vxlxY2_;skh^O84w?H0}+rhP~1G<*?hk z>YvnLw^Ax$TX|9_T~&$;&R%b7SJ~-~uDZkCNpCRjcg{Ahka%j>%5=?gI{&&H_O8ag zL1!?oUw?{ON%}NY&yhtjx_mq84*M5|ne~XuZ%PlK>z?&{4{?;fW^G0>>UD>`RYk6` z^!PwH#buqfZ}RhHql%k-nv6LeS=@Eo+0Rw!{qX#9)hERZu0D~iQ4WLceZ4+pSZ48M zAkjXi@BtxylNosu&B0CyTW3X&U3T_{=&Y=ts()6BY9rVUmgU*LFt~=)>e@?8hVhI&SH_+<#PljuhqJXJOeOZg?L&ti(%VB>L!`TyU zh2Olt4u9%a*Rz>ZD7>83$)q@3N6MUYS0{r}-T7akls(7R(a8rl+R`TJRfLo4RSvwW zZGY+S44q(6fFI0tTGfPj1*0r75g7P5igD_U+4-tdy^z4Doaq~mHR7pe&f$X$mX9=N zvtT?C1jO8({=HJQ^8Yy$ku%kXe`o6`zDp&6PK{LqIL)R{l^JPTFlPrGEhBVx>bbEq zQsEfjlE!6{Eh}72(aKH*WtJ96aHexsD}QimXA2+gK~`lbE%~WuC>yP!tSCcT&9f>A zCT2}$hGNRq33YgDB`Q>9-FZ1u`R0^ud2BDMcez?aNiLT~qH)*Yk=O1YWkQ<_Xf9N2 zJw?_-YfjdhZn9UE+BgwF+k&oll;I{ztE)44FsW(x>jAkrP)M%DG|aA!kfEoAEPpI% zf||JbQd}x_g=zz%Jo@spyaBAw{}nP7-$`Rb{{P_kO||~}_~_{PCI9yvtz2l8=KIo% z!w<~$#TI@ZlFt!tHL|qbQC&r3=f< zINFY7X+$3T%rwJhwPkXAh}?^jyB1*#l%&g6VNR1AR3*4AA^jW)jrA&0gMJNkiBS1Q z^<2rMmE4czH{IU5PflJxdh*F>!zbrA^~u>O^q=A~AGmkmbsj|(YEO%Slz)A}FA{bt zEEeZ&*CI5M`zDNL;x_!9ZusBH);u;VRZMGGNu_lXj+16*42AXh3+hx!7nB7LpPV$= zQxJRB8*@+|DQB6=V`(a@+)F%77kEOY4J?*D?uUYMeJNQCmwuhX#Efz`tH~FfQI7Uw zl^9Q2i1C4NGDEo*PhiLGi+^4mb>%23th|>boab4?GMbqgbZ;s5#jSEV>I8@S%sylk zR+umzeq#JiVPZUQMi|2fmYPJna8v!q{!!)opZ(TB>wl&G<5}8Q)PEGi z!*UtL4-6BZtkB`5)wcEauTg3BX^xDETOpbM?H*u*{qG$dy{X#&@$tcn{Xa*mH}DJL z*IIR5EoO5~@nzE}kn_#OAL-|XtMwPo99iJ1`R-hZ8z+Wrs>t(DsY0ztrEsE|mO)sJ zZd#RYC#f2!t8GZXWq*Y19WJ+pxHl_+&2mB}AU0-GND>7Y*iR~*mb>@9kP-C4s119q ztX#lW6MyYQpR1JJq=f7pm9gAva#rP3)m$iD*wvccYFu$+KjBEJ3~&*D+^7XZnQcoY zg;v)aX-Y63=%~jP)N#| z3wF=hvn?N`CupS1)DV8fAOufK1t<=bNkxJHQn7(lZ1gHvm> z1}^*&vp*>Q`G4Ob^#qrEVYXQSjD+aIgwN*`;XfuOpx)y-Ih)b%t z*tn3ZRzvy$6j{0u5QS3z7sWwv&I9UgTo?v7WFsx>$A2H;DiVB6eH2$9S3C}Um?Ln8 zz!yMns?z{}?weSe7<1O0xhNnFr{*phS!oz_nhlot#=Nhw^Sl zsRdxP#(xF-_7aCc{00GcVdSfHn_AdIEb47obxQ*VWS9N^V1T%8MCpeW4& z7aC^#8NFEHm7_^CC|oF@tej&~-@$NHKbRw@Xsour56}hp7zKPo*-}pqHDuXdPf@I? zhwy56Jl)v^u2v{+s6<$hK%zmZh*+W;9#3-~0e^S_MW&8?7fSyotTkJZk{^z&Ps+kEiL~LS#xBoOOKZ zQ8~W!|9~$EBf&;dPbO?D$w6^(fEzBZjZ7WAO}CcjQc3#pM>t9yjI8ZQoGaP~7 z0e?z!pE+edWlL#^Iq~#)sYA{ADU+8l^gJFjb=vB{48sg!5>ZGUl79CQ&3*a0KSxFS z3G68G#D)$qQijaar|F3aF5JvlcZeTS_K$aJFXPdp?*A2rk+JUG)3e!l>Zs4^$!9zi z{~x}t$NvYfdoTX~S;lOI&0t=_5|z?^B7d=}#@@n)$D#=8tr^#*hb}gtRzx+BtcV27 zT7~{Dh?u#S_Neh3JrzKw!!j5Bi}sll43J24a>osKcbW!>xG$j-fK8%ELuTR$_WIh< zIBal?hqZ2RJKDv)VqE1u7}?sXc>f(^gtx;S<7Nl$?kww+D|Eq{olJEU01=_u(|;&P zeXVa?&xd39bv8SPfJ%i7g#*I^`i5vOG89Cm;2R!0Fk8{`d=5^f!7pQ~wDU0=BEkLG z)1DGyiB57yWWJ{GTfP77`hU5?AVMJ>6+JN(ctHRA`>*>o{XaZ-J$TXo=NPxQ-G9NF zhAv1{u%H2|O@zPb5S4f(tiJYXyq;);lLJfJgMj@z=@RtsTmNE=gtU=BZ4^j; z#v%@a4Sb48Ko=B!AW>w9cC7!w1iOJ(FxWR2kT6GcXBuo4xUt1c)8zJD|kXh1`% zHCKn#T9;qVH--&wPp2AI8%-IZx6{rMO;?pGKbZsEUmnk%Y-c1WQwy+vmYtuaT{WU9 z+ArtiR1xMA;X^>?7)WD>Rv3gJ(qLM)-YbDlkS5PMxLu*3O6QkJW|R?_B^pU)B$C-0 zjJZSxu%>B&sthqnr&A~+hJPtYiG!uZqRv&oq;n}4i6R%adzlKm{%*znMq|7EPd&X( z5z?&XYpX&J#(#%NxNhQ0603FwkH!p8tWKG9}*R@M$Hje(Z4xSXxQi7N=TS5}jn=X|Z7|vvd?(=SP{I z3L{h0>D9k+=x4@u{VxhCPpt+&p#O)jd;7Kc@AclG_oDyLG0GZ_pA|CwD7#ttC=u1S zxhQ-$l{KY99!8uYQ-3bxy_yKUv^pJ7k4P8#7Ubt81-pWzhtE39&f0QbvOpN)$|!AK z6Vx#Tde<@zyAnNt%3M4ZRft|t|C9-!6F>*x)2Zu5aGitJH zCR7zouVkk4SZkGxq@-UdMw=<8DDlJHo!eMeWSPc6B|PV)0)KyFM^XJRpO0)^a4n<# z_~Om@WIVe(Kby?*#iDTD?+p%mc4OD@{AfC!d>BtIN8`!t^5kqZoE@E=HW74Dd5zYc zheA0KhWDU79{>AGv0me*73qiKif=EE&gb2!&nc~@I-eHflxj9am(D5zmHYw*(}u#Vro8;@GveuK!Q z%aKLgbOHFO*;lbmtHaKqZm3!{fTUx9N|>B-!QV+M{cW_LoV~vso(@m`Jv$mrFNYVi zU%ymPDkuFRIV;z1GIH{cN*fKs(e!dO8NVH$&W?sB4}YCcJlQqZO%7`y`_IK>d^sDR z4o_#7M{nC-L)NWnrN$tbUc8x(CP(L*kgaH{U7qd8Mkhz(d#K4;OAjKLjz^PmL$aDG zmzd&X+_b3OLpHV|;mI%!!i31F!QAo&k1i+U>Dk3(G`@U4IlE}sh2*C7ZdJQ9Xk71G z1t7<< zpNMNCrfiebG{7yLm&}|#F5dq>0{npZuDDEOw?uVhWZ8)FbzabIbCCebd`R++26(TO zw|{l#@2#MiH*^oXY$)>j88xF{~8voWW*ghP3kOBVvl;?GYi2yD;)OKHm! zbIb?w|GmS0z5jc--#>iG|DR)2zZtQzYMBXUzE`fFDBD>?KE-HdnR~N)`%ve$(X}=0 z6pw)W3oMb1hK~kKr;

VKrvcjSlE8!Q`VGMc!z0GCgxBw^2zT{f~oLu&R;Z$H|l56nK=E`NNq zf0iHZ-O~IqZi8gA4ydvHl?lyX=^Ji}Of5aBvKT(vSwdN2a6X%yUtOu@vDLLKRj$qwPti5LrNTJy z&6xQSr%YXlAc4!iI1l)eGAIAgmVdK=GBh8cCwQ`U>u-dSC5o+hLbGnF^hsb-ps11Z zUXW;&B~uyM`3LZN(Yc|_=Qq+}s8SsAQ}A1ULW2lxyXV@hlr|>&mLv%9p6jRU4VuI4 zhqc_;;{t0$gI0`OettWePLEFC+eIm1=blyLN7|OFV$cqgiNnrzIlV8d{C~Ur|9$bF ziFHX5>pmq8e5n6B*st|}2EG3NOZ@jNF++;36qw(qk#TfWHZXz8)PviY0-wy4pC zwO1H~j$CzDoL}1sTt~*9hksJ$o#JZyw3Bs1jU0E3AiGJ`1ln4h>2H`I?*|m}Zzl7v z;Ih(;4ve*;6s4LFL-QjlyXW-Z7j-x