diff --git a/vkubelet/env.go b/vkubelet/env.go index 28a075cb1..fb5cda8fe 100644 --- a/vkubelet/env.go +++ b/vkubelet/env.go @@ -3,20 +3,24 @@ package vkubelet import ( "context" "fmt" + "sort" + "strings" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + apivalidation "k8s.io/apimachinery/pkg/util/validation" "k8s.io/client-go/tools/record" "github.com/virtual-kubelet/virtual-kubelet/log" + "github.com/virtual-kubelet/virtual-kubelet/manager" ) const ( - // ReasonOptionalConfigMapNotFound is the reason used in events emitted when an optional config map is not found. + // ReasonOptionalConfigMapNotFound is the reason used in events emitted when an optional configmap is not found. ReasonOptionalConfigMapNotFound = "OptionalConfigMapNotFound" - // ReasonOptionalConfigMapKeyNotFound is the reason used in events emitted when an optional config map key is not found. + // ReasonOptionalConfigMapKeyNotFound is the reason used in events emitted when an optional configmap key is not found. ReasonOptionalConfigMapKeyNotFound = "OptionalConfigMapKeyNotFound" - // ReasonFailedToReadOptionalConfigMap is the reason used in events emitted when an optional config map could not be read. + // ReasonFailedToReadOptionalConfigMap is the reason used in events emitted when an optional configmap could not be read. ReasonFailedToReadOptionalConfigMap = "FailedToReadOptionalConfigMap" // ReasonOptionalSecretNotFound is the reason used in events emitted when an optional secret is not found. @@ -26,11 +30,11 @@ const ( // ReasonFailedToReadOptionalSecret is the reason used in events emitted when an optional secret could not be read. ReasonFailedToReadOptionalSecret = "FailedToReadOptionalSecret" - // ReasonMandatoryConfigMapNotFound is the reason used in events emitted when an mandatory config map is not found. + // ReasonMandatoryConfigMapNotFound is the reason used in events emitted when an mandatory configmap is not found. ReasonMandatoryConfigMapNotFound = "MandatoryConfigMapNotFound" - // ReasonMandatoryConfigMapKeyNotFound is the reason used in events emitted when an mandatory config map key is not found. + // ReasonMandatoryConfigMapKeyNotFound is the reason used in events emitted when an mandatory configmap key is not found. ReasonMandatoryConfigMapKeyNotFound = "MandatoryConfigMapKeyNotFound" - // ReasonFailedToReadMandatoryConfigMap is the reason used in events emitted when an mandatory config map could not be read. + // ReasonFailedToReadMandatoryConfigMap is the reason used in events emitted when an mandatory configmap could not be read. ReasonFailedToReadMandatoryConfigMap = "FailedToReadMandatoryConfigMap" // ReasonMandatorySecretNotFound is the reason used in events emitted when an mandatory secret is not found. @@ -39,131 +43,314 @@ const ( ReasonMandatorySecretKeyNotFound = "MandatorySecretKeyNotFound" // ReasonFailedToReadMandatorySecret is the reason used in events emitted when an mandatory secret could not be read. ReasonFailedToReadMandatorySecret = "FailedToReadMandatorySecret" + + // ReasonInvalidEnvironmentVariableNames is the reason used in events emitted when a configmap/secret referenced in a ".spec.containers[*].envFrom" field contains invalid environment variable names. + ReasonInvalidEnvironmentVariableNames = "InvalidEnvironmentVariableNames" ) -// populateEnvironmentVariables populates Secrets and ConfigMap into environment variables -func (s *Server) populateEnvironmentVariables(ctx context.Context, pod *corev1.Pod, recorder record.EventRecorder) error { - for _, c := range pod.Spec.Containers { - for i, e := range c.Env { - if e.ValueFrom != nil { - // Populate ConfigMaps to Env - if e.ValueFrom.ConfigMapKeyRef != nil { - vf := e.ValueFrom.ConfigMapKeyRef - // Check whether the key reference is optional. - // This will control whether we fail when unable to read the requested key. - optional := vf != nil && *vf.Optional - // Try to grab the referenced config map. - cm, err := s.resourceManager.GetConfigMap(vf.Name, pod.Namespace) - if err != nil { - // We couldn't fetch the config map. - // However, if the key reference is optional we should not fail. - if optional { - if errors.IsNotFound(err) { - recorder.Eventf(pod, corev1.EventTypeWarning, ReasonOptionalConfigMapNotFound, "skipping optional envvar %q: configmap %q not found", e.Name, vf.Name) - } else { - log.G(ctx).Warnf("failed to read configmap %q: %v", vf.Name, err) - recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadOptionalConfigMap, "skipping optional envvar %q: failed to read configmap %q", e.Name, vf.Name) - } - // Continue on to the next reference. - continue - } - // At this point we know the key reference is mandatory. - // Hence, we should return a meaningful error. - if errors.IsNotFound(err) { - recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatoryConfigMapNotFound, "configmap %q not found", vf.Name) - return fmt.Errorf("required configmap %q not found", vf.Name) - } - recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadMandatoryConfigMap, "failed to read configmap %q", vf.Name) - return fmt.Errorf("failed to read required configmap %q: %v", vf.Name, err) - } - // At this point we have successfully fetched the target config map. - // We must now try to grab the requested key. - var ( - keyExists bool - keyValue string - ) - if keyValue, keyExists = cm.Data[vf.Key]; !keyExists { - // The requested key does not exist. - // However, we should not fail if the key reference is optional. - if optional { - // Continue on to the next reference. - recorder.Eventf(pod, corev1.EventTypeWarning, ReasonOptionalConfigMapKeyNotFound, "skipping optional envvar %q: key %q does not exist in configmap %q", e.Name, vf.Key, vf.Name) - continue - } - // At this point we know the key reference is mandatory. - // Hence, we should fail. - recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatoryConfigMapKeyNotFound, "key %q does not exist in configmap %q", vf.Key, vf.Name) - return fmt.Errorf("configmap %q doesn't contain the %q key required by pod %s", vf.Name, vf.Key, pod.Name) - } - // Populate the environment variable and continue on to the next reference. - c.Env[i].Value = keyValue - continue - } - // Populate Secrets to Env - if e.ValueFrom.SecretKeyRef != nil { - vf := e.ValueFrom.SecretKeyRef - // Check whether the key reference is optional. - // This will control whether we fail when unable to read the requested key. - optional := vf != nil && *vf.Optional - // Try to grab the referenced secret. - cm, err := s.resourceManager.GetSecret(vf.Name, pod.Namespace) - if err != nil { - // We couldn't fetch the secret. - // However, if the key reference is optional we should not fail. - if optional { - if errors.IsNotFound(err) { - recorder.Eventf(pod, corev1.EventTypeWarning, ReasonOptionalSecretNotFound, "skipping optional envvar %q: secret %q not found", e.Name, vf.Name) - } else { - log.G(ctx).Warnf("failed to read secret %q: %v", vf.Name, err) - recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadOptionalSecret, "skipping optional envvar %q: failed to read secret %q", e.Name, vf.Name) - } - // Continue on to the next reference. - continue - } - // At this point we know the key reference is mandatory. - // Hence, we should return a meaningful error. - if errors.IsNotFound(err) { - recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatorySecretNotFound, "secret %q not found", vf.Name) - return fmt.Errorf("required secret %q not found", vf.Name) - } - recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadMandatorySecret, "failed to read secret %q", vf.Name) - return fmt.Errorf("failed to read secret %q: %v", vf.Name, err) - } - // At this point we have successfully fetched the target secret. - // We must now try to grab the requested key. - var ( - keyExists bool - keyValue []byte - ) - if keyValue, keyExists = cm.Data[vf.Key]; !keyExists { - // The requested key does not exist. - // However, we should not fail if the key reference is optional. - if optional { - // Continue on to the next reference. - recorder.Eventf(pod, corev1.EventTypeWarning, ReasonOptionalSecretKeyNotFound, "skipping optional envvar %q: key %q does not exist in secret %q", e.Name, vf.Key, vf.Name) - continue - } - // At this point we know the key reference is mandatory. - // Hence, we should fail. - recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatorySecretKeyNotFound, "key %q does not exist in secret %q", vf.Key, vf.Name) - return fmt.Errorf("secret %q doesn't contain the %q key required by pod %s", vf.Name, vf.Key, pod.Name) - } - // Populate the environment variable and continue on to the next reference. - c.Env[i].Value = string(keyValue) - continue - } - - // TODO: Populate Downward API to Env - if e.ValueFrom.FieldRef != nil { - continue - } - - // TODO: Populate resource requests - if e.ValueFrom.ResourceFieldRef != nil { - continue - } - } +// populateEnvironmentVariables populates the environment of each container (and init container) in the specified pod. +// NOTE: No other function in this file should be called from the outside. +// TODO: Make this the single exported function of a "pkg/environment" package in the future. +func populateEnvironmentVariables(ctx context.Context, pod *corev1.Pod, rm *manager.ResourceManager, recorder record.EventRecorder) error { + // Populate each init container's environment. + for idx := range pod.Spec.InitContainers { + if err := populateContainerEnvironment(ctx, pod, &pod.Spec.InitContainers[idx], rm, recorder); err != nil { + return err + } + } + // Populate each container's environment. + for idx := range pod.Spec.Containers { + if err := populateContainerEnvironment(ctx, pod, &pod.Spec.Containers[idx], rm, recorder); err != nil { + return err } } return nil } + +// populateContainerEnvironment populates the environment of a single container in the specified pod. +func populateContainerEnvironment(ctx context.Context, pod *corev1.Pod, container *corev1.Container, rm *manager.ResourceManager, recorder record.EventRecorder) error { + // Create an "environment map" based on the value of the specified container's ".envFrom" field. + envOne, err := makeEnvironmentMapFromEnvFrom(ctx, pod, container, rm, recorder) + if err != nil { + return err + } + // Create an "environment map" based on the value of the specified container's ".env" field. + envTwo, err := makeEnvironmentMapFromEnv(ctx, pod, container, rm, recorder) + if err != nil { + return err + } + // Empty the container's ".envFrom" field and replace its ".env" field with the final, merged environment. + // Values in "envTwo" (sourced from ".env") will override any values with the same key defined in "envOne" (sourced from ".envFrom"). + // This is in accordance with what the Kubelet itself does. + // https://github.com/kubernetes/kubernetes/blob/v1.13.0/pkg/kubelet/kubelet_pods.go#L557-L558 + container.EnvFrom = []corev1.EnvFromSource{} + container.Env = makeEnvironment(envOne, envTwo) + return nil +} + +// makeEnvironmentMapFromEnvFrom returns a map representing the resolved environment of the specified container after being populated from the entries in the ".envFrom" field. +func makeEnvironmentMapFromEnvFrom(ctx context.Context, pod *corev1.Pod, container *corev1.Container, rm *manager.ResourceManager, recorder record.EventRecorder) (map[string]string, error) { + // Create a map to hold the resulting environment. + res := make(map[string]string, 0) + // Iterate over "envFrom" references in order to populate the environment. + for _, envFrom := range container.EnvFrom { + switch { + // Handle population from a configmap. + case envFrom.ConfigMapRef != nil: + ef := envFrom.ConfigMapRef + // Check whether the configmap reference is optional. + // This will control whether we fail when unable to read the configmap. + optional := ef.Optional != nil && *ef.Optional + // Try to grab the referenced configmap. + m, err := rm.GetConfigMap(ef.Name, pod.Namespace) + if err != nil { + // We couldn't fetch the configmap. + // However, if the configmap reference is optional we should not fail. + if optional { + if errors.IsNotFound(err) { + recorder.Eventf(pod, corev1.EventTypeWarning, ReasonOptionalConfigMapNotFound, "configmap %q not found", ef.Name) + } else { + log.G(ctx).Warnf("failed to read configmap %q: %v", ef.Name, err) + recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadOptionalConfigMap, "failed to read configmap %q", ef.Name) + } + // Continue on to the next reference. + continue + } + // At this point we know the configmap reference is mandatory. + // Hence, we should return a meaningful error. + if errors.IsNotFound(err) { + recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatoryConfigMapNotFound, "configmap %q not found", ef.Name) + return nil, fmt.Errorf("configmap %q not found", ef.Name) + } + recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadMandatoryConfigMap, "failed to read configmap %q", ef.Name) + return nil, fmt.Errorf("failed to fetch configmap %q: %v", ef.Name, err) + } + // At this point we have successfully fetched the target configmap. + // Iterate over the keys defined in the configmap and populate the environment accordingly. + invalidKeys := make([]string, 0) + for key, val := range m.Data { + // If a prefix has been defined, prepend it to the environment variable's name. + if len(envFrom.Prefix) > 0 { + key = envFrom.Prefix + key + } + // Make sure that the resulting key is a valid environment variable name. + // If it isn't, it should be appended to the list of invalid keys and skipped. + if errMsgs := apivalidation.IsEnvVarName(key); len(errMsgs) != 0 { + invalidKeys = append(invalidKeys, key) + continue + } + // Add the key and its value to the environment. + res[key] = val + } + // Report any invalid keys. + if len(invalidKeys) > 0 { + sort.Strings(invalidKeys) + recorder.Eventf(pod, corev1.EventTypeWarning, ReasonInvalidEnvironmentVariableNames, "keys [%s] from configmap %s/%s were skipped since they are invalid as environment variable names", strings.Join(invalidKeys, ", "), m.Namespace, m.Name) + } + // Handle population from a secret. + case envFrom.SecretRef != nil: + ef := envFrom.SecretRef + // Check whether the secret reference is optional. + // This will control whether we fail when unable to read the secret. + optional := ef.Optional != nil && *ef.Optional + // Try to grab the referenced secret. + s, err := rm.GetSecret(ef.Name, pod.Namespace) + if err != nil { + // We couldn't fetch the secret. + // However, if the secret reference is optional we should not fail. + if optional { + if errors.IsNotFound(err) { + recorder.Eventf(pod, corev1.EventTypeWarning, ReasonOptionalSecretNotFound, "secret %q not found", ef.Name) + } else { + log.G(ctx).Warnf("failed to read configmap %q: %v", ef.Name, err) + recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadOptionalSecret, "failed to read secret %q", ef.Name) + } + // Continue on to the next reference. + continue + } + // At this point we know the secret reference is mandatory. + // Hence, we should return a meaningful error. + if errors.IsNotFound(err) { + recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatorySecretNotFound, "secret %q not found", ef.Name) + return nil, fmt.Errorf("configmap %q not found", ef.Name) + } + recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadMandatorySecret, "failed to read secret %q", ef.Name) + return nil, fmt.Errorf("failed to fetch configmap %q: %v", ef.Name, err) + } + // At this point we have successfully fetched the target secret. + // Iterate over the keys defined in the secret and populate the environment accordingly. + invalidKeys := make([]string, 0) + for key, val := range s.Data { + // If a prefix has been defined, prepend it to the environment variable's name. + if len(envFrom.Prefix) > 0 { + key = envFrom.Prefix + key + } + // Make sure that the resulting key is a valid environment variable name. + // If it isn't, it should be appended to the list of invalid keys and skipped. + if errMsgs := apivalidation.IsEnvVarName(key); len(errMsgs) != 0 { + invalidKeys = append(invalidKeys, key) + continue + } + // Add the key and its value to the environment. + res[key] = string(val) + } + // Report any invalid keys. + if len(invalidKeys) > 0 { + sort.Strings(invalidKeys) + recorder.Eventf(pod, corev1.EventTypeWarning, ReasonInvalidEnvironmentVariableNames, "keys [%s] from secret %s/%s were skipped since they are invalid as environment variable names", strings.Join(invalidKeys, ", "), s.Namespace, s.Name) + } + } + } + // Return the populated environment. + return res, nil +} + +// makeEnvironmentMapFromEnv returns a map representing the resolved environment of the specified container after being populated from the entries in the ".env" field. +func makeEnvironmentMapFromEnv(ctx context.Context, pod *corev1.Pod, container *corev1.Container, rm *manager.ResourceManager, recorder record.EventRecorder) (map[string]string, error) { + // Create a map to hold the resolved environment variables. + res := make(map[string]string, len(container.Env)) + // Iterate over environment variables in order to populate the map. + for _, env := range container.Env { + switch { + // Handle values that have been directly provided. + case env.Value != "": + res[env.Name] = env.Value + continue + // Handle population from a configmap key. + case env.ValueFrom != nil && env.ValueFrom.ConfigMapKeyRef != nil: + // The environment variable must be set from a configmap. + vf := env.ValueFrom.ConfigMapKeyRef + // Check whether the key reference is optional. + // This will control whether we fail when unable to read the requested key. + optional := vf != nil && *vf.Optional + // Try to grab the referenced secret. + m, err := rm.GetConfigMap(vf.Name, pod.Namespace) + if err != nil { + // We couldn't fetch the secret. + // However, if the key reference is optional we should not fail. + if optional { + if errors.IsNotFound(err) { + recorder.Eventf(pod, corev1.EventTypeWarning, ReasonOptionalConfigMapNotFound, "skipping optional envvar %q: configmap %q not found", env.Name, vf.Name) + } else { + log.G(ctx).Warnf("failed to read configmap %q: %v", vf.Name, err) + recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadOptionalConfigMap, "skipping optional envvar %q: failed to read configmap %q", env.Name, vf.Name) + } + // Continue on to the next reference. + continue + } + // At this point we know the key reference is mandatory. + // Hence, we should return a meaningful error. + if errors.IsNotFound(err) { + recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatoryConfigMapNotFound, "configmap %q not found", vf.Name) + return nil, fmt.Errorf("configmap %q not found", vf.Name) + } + recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadMandatoryConfigMap, "failed to read configmap %q", vf.Name) + return nil, fmt.Errorf("failed to read configmap %q: %v", vf.Name, err) + } + // At this point we have successfully fetched the target configmap. + // We must now try to grab the requested key. + var ( + keyExists bool + keyValue string + ) + if keyValue, keyExists = m.Data[vf.Key]; !keyExists { + // The requested key does not exist. + // However, we should not fail if the key reference is optional. + if optional { + // Continue on to the next reference. + recorder.Eventf(pod, corev1.EventTypeWarning, ReasonOptionalConfigMapKeyNotFound, "skipping optional envvar %q: key %q does not exist in configmap %q", env.Name, vf.Key, vf.Name) + continue + } + // At this point we know the key reference is mandatory. + // Hence, we should fail. + recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatoryConfigMapKeyNotFound, "key %q does not exist in configmap %q", vf.Key, vf.Name) + return nil, fmt.Errorf("configmap %q doesn't contain the %q key required by pod %s", vf.Name, vf.Key, pod.Name) + } + // Populate the environment variable and continue on to the next reference. + res[env.Name] = keyValue + continue + // Handle population from a secret key. + case env.ValueFrom != nil && env.ValueFrom.SecretKeyRef != nil: + vf := env.ValueFrom.SecretKeyRef + // Check whether the key reference is optional. + // This will control whether we fail when unable to read the requested key. + optional := vf != nil && *vf.Optional + // Try to grab the referenced secret. + s, err := rm.GetSecret(vf.Name, pod.Namespace) + if err != nil { + // We couldn't fetch the secret. + // However, if the key reference is optional we should not fail. + if optional { + if errors.IsNotFound(err) { + recorder.Eventf(pod, corev1.EventTypeWarning, ReasonOptionalSecretNotFound, "skipping optional envvar %q: secret %q not found", env.Name, vf.Name) + } else { + log.G(ctx).Warnf("failed to read secret %q: %v", vf.Name, err) + recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadOptionalSecret, "skipping optional envvar %q: failed to read secret %q", env.Name, vf.Name) + } + // Continue on to the next reference. + continue + } + // At this point we know the key reference is mandatory. + // Hence, we should return a meaningful error. + if errors.IsNotFound(err) { + recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatorySecretNotFound, "secret %q not found", vf.Name) + return nil, fmt.Errorf("secret %q not found", vf.Name) + } + recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadMandatorySecret, "failed to read secret %q", vf.Name) + return nil, fmt.Errorf("failed to read secret %q: %v", vf.Name, err) + } + // At this point we have successfully fetched the target secret. + // We must now try to grab the requested key. + var ( + keyExists bool + keyValue []byte + ) + if keyValue, keyExists = s.Data[vf.Key]; !keyExists { + // The requested key does not exist. + // However, we should not fail if the key reference is optional. + if optional { + // Continue on to the next reference. + recorder.Eventf(pod, corev1.EventTypeWarning, ReasonOptionalSecretKeyNotFound, "skipping optional envvar %q: key %q does not exist in secret %q", env.Name, vf.Key, vf.Name) + continue + } + // At this point we know the key reference is mandatory. + // Hence, we should fail. + recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatorySecretKeyNotFound, "key %q does not exist in secret %q", vf.Key, vf.Name) + return nil, fmt.Errorf("secret %q doesn't contain the %q key required by pod %s", vf.Name, vf.Key, pod.Name) + } + // Populate the environment variable and continue on to the next reference. + res[env.Name] = string(keyValue) + continue + // Handle population from a field (downward API). + case env.ValueFrom != nil && env.ValueFrom.FieldRef != nil: + // TODO Implement the downward API. + continue + // Handle population from a resource request/limit. + case env.ValueFrom != nil && env.ValueFrom.ResourceFieldRef != nil: + // TODO Implement populating resource requests. + continue + } + } + // Return the populated environment. + return res, nil +} + +// makeEnvironment creates the final environment for a container by merging the two specified "environment maps". +// Values in "envTwo" override any values with the same key defined in "envOne". +func makeEnvironment(envOne map[string]string, envTwo map[string]string) []corev1.EnvVar { + tmp := make(map[string]string, 0) + res := make([]corev1.EnvVar, 0) + for key, val := range envOne { + tmp[key] = val + } + for key, val := range envTwo { + tmp[key] = val + } + for key, val := range tmp { + res = append(res, corev1.EnvVar{ + Name: key, + Value: val, + }) + } + return res +} diff --git a/vkubelet/pod.go b/vkubelet/pod.go index 788103c80..9dff71d3c 100644 --- a/vkubelet/pod.go +++ b/vkubelet/pod.go @@ -42,7 +42,7 @@ func (s *Server) createOrUpdatePod(ctx context.Context, pod *corev1.Pod, recorde defer span.End() addPodAttributes(span, pod) - if err := s.populateEnvironmentVariables(ctx, pod, recorder); err != nil { + if err := populateEnvironmentVariables(ctx, pod, s.resourceManager, recorder); err != nil { span.SetStatus(trace.Status{Code: trace.StatusCodeInvalidArgument, Message: err.Error()}) return err }