Files
virtual-kubelet/cmd/virtual-kubelet/internal/provider/mock/mock.go
2025-05-29 19:17:47 +01:00

698 lines
20 KiB
Go

package mock
import (
"context"
"encoding/json"
"fmt"
"io"
"math/rand"
"os"
"strings"
"time"
dto "github.com/prometheus/client_model/go"
"github.com/virtual-kubelet/virtual-kubelet/errdefs"
"github.com/virtual-kubelet/virtual-kubelet/log"
"github.com/virtual-kubelet/virtual-kubelet/node/api"
"github.com/virtual-kubelet/virtual-kubelet/trace"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
)
const (
// Provider configuration defaults.
defaultCPUCapacity = "20"
defaultMemoryCapacity = "100Gi"
defaultPodCapacity = "20"
// Values used in tracing as attribute keys.
namespaceKey = "namespace"
nameKey = "name"
containerNameKey = "containerName"
)
// See: https://github.com/virtual-kubelet/virtual-kubelet/issues/632
/*
var (
_ providers.Provider = (*MockV0Provider)(nil)
_ providers.PodMetricsProvider = (*MockV0Provider)(nil)
_ node.PodNotifier = (*MockProvider)(nil)
)
*/
// MockProvider implements the virtual-kubelet provider interface and stores pods in memory.
type MockProvider struct {
nodeName string
operatingSystem string
internalIP string
daemonEndpointPort int32
pods map[string]*v1.Pod
config MockConfig
startTime time.Time
notifier func(*v1.Pod)
}
// MockConfig contains a mock virtual-kubelet's configurable parameters.
type MockConfig struct {
CPU string `json:"cpu,omitempty"`
Memory string `json:"memory,omitempty"`
Pods string `json:"pods,omitempty"`
Others map[string]string `json:"others,omitempty"`
ProviderID string `json:"providerID,omitempty"`
}
// NewMockProviderMockConfig creates a new MockV0Provider. Mock legacy provider does not implement the new asynchronous podnotifier interface
func NewMockProviderMockConfig(config MockConfig, nodeName, operatingSystem string, internalIP string, daemonEndpointPort int32) (*MockProvider, error) {
// set defaults
if config.CPU == "" {
config.CPU = defaultCPUCapacity
}
if config.Memory == "" {
config.Memory = defaultMemoryCapacity
}
if config.Pods == "" {
config.Pods = defaultPodCapacity
}
provider := MockProvider{
nodeName: nodeName,
operatingSystem: operatingSystem,
internalIP: internalIP,
daemonEndpointPort: daemonEndpointPort,
pods: make(map[string]*v1.Pod),
config: config,
startTime: time.Now(),
}
return &provider, nil
}
// NewMockProvider creates a new MockProvider, which implements the PodNotifier interface
func NewMockProvider(providerConfig, nodeName, operatingSystem string, internalIP string, daemonEndpointPort int32) (*MockProvider, error) {
config, err := loadConfig(providerConfig, nodeName)
if err != nil {
return nil, err
}
return NewMockProviderMockConfig(config, nodeName, operatingSystem, internalIP, daemonEndpointPort)
}
// loadConfig loads the given json configuration files.
func loadConfig(providerConfig, nodeName string) (config MockConfig, err error) {
data, err := os.ReadFile(providerConfig)
if err != nil {
return config, err
}
configMap := map[string]MockConfig{}
err = json.Unmarshal(data, &configMap)
if err != nil {
return config, err
}
if _, exist := configMap[nodeName]; exist {
config = configMap[nodeName]
if config.CPU == "" {
config.CPU = defaultCPUCapacity
}
if config.Memory == "" {
config.Memory = defaultMemoryCapacity
}
if config.Pods == "" {
config.Pods = defaultPodCapacity
}
}
if _, err = resource.ParseQuantity(config.CPU); err != nil {
return config, fmt.Errorf("invalid CPU value %v", config.CPU)
}
if _, err = resource.ParseQuantity(config.Memory); err != nil {
return config, fmt.Errorf("invalid memory value %v", config.Memory)
}
if _, err = resource.ParseQuantity(config.Pods); err != nil {
return config, fmt.Errorf("invalid pods value %v", config.Pods)
}
for _, v := range config.Others {
if _, err = resource.ParseQuantity(v); err != nil {
return config, fmt.Errorf("invalid other value %v", v)
}
}
return config, nil
}
// CreatePod accepts a Pod definition and stores it in memory.
func (p *MockProvider) CreatePod(ctx context.Context, pod *v1.Pod) error {
ctx, span := trace.StartSpan(ctx, "CreatePod")
defer span.End()
// Add the pod's coordinates to the current span.
ctx = addAttributes(ctx, span, namespaceKey, pod.Namespace, nameKey, pod.Name)
log.G(ctx).Infof("receive CreatePod %q", pod.Name)
key, err := buildKey(pod)
if err != nil {
return err
}
now := metav1.NewTime(time.Now())
pod.Status = v1.PodStatus{
Phase: v1.PodRunning,
HostIP: "1.2.3.4",
PodIP: "5.6.7.8",
StartTime: &now,
Conditions: []v1.PodCondition{
{
Type: v1.PodInitialized,
Status: v1.ConditionTrue,
},
{
Type: v1.PodReady,
Status: v1.ConditionTrue,
},
{
Type: v1.PodScheduled,
Status: v1.ConditionTrue,
},
},
}
for _, container := range pod.Spec.Containers {
pod.Status.ContainerStatuses = append(pod.Status.ContainerStatuses, v1.ContainerStatus{
Name: container.Name,
Image: container.Image,
Ready: true,
RestartCount: 0,
State: v1.ContainerState{
Running: &v1.ContainerStateRunning{
StartedAt: now,
},
},
})
}
p.pods[key] = pod
p.notifier(pod)
return nil
}
// UpdatePod accepts a Pod definition and updates its reference.
func (p *MockProvider) UpdatePod(ctx context.Context, pod *v1.Pod) error {
ctx, span := trace.StartSpan(ctx, "UpdatePod")
defer span.End()
// Add the pod's coordinates to the current span.
ctx = addAttributes(ctx, span, namespaceKey, pod.Namespace, nameKey, pod.Name)
log.G(ctx).Infof("receive UpdatePod %q", pod.Name)
key, err := buildKey(pod)
if err != nil {
return err
}
p.pods[key] = pod
p.notifier(pod)
return nil
}
// DeletePod deletes the specified pod out of memory.
func (p *MockProvider) DeletePod(ctx context.Context, pod *v1.Pod) (err error) {
ctx, span := trace.StartSpan(ctx, "DeletePod")
defer span.End()
// Add the pod's coordinates to the current span.
ctx = addAttributes(ctx, span, namespaceKey, pod.Namespace, nameKey, pod.Name)
log.G(ctx).Infof("receive DeletePod %q", pod.Name)
key, err := buildKey(pod)
if err != nil {
return err
}
if _, exists := p.pods[key]; !exists {
return errdefs.NotFound("pod not found")
}
now := metav1.Now()
delete(p.pods, key)
pod.Status.Phase = v1.PodSucceeded
pod.Status.Reason = "MockProviderPodDeleted"
for idx := range pod.Status.ContainerStatuses {
pod.Status.ContainerStatuses[idx].Ready = false
pod.Status.ContainerStatuses[idx].State = v1.ContainerState{
Terminated: &v1.ContainerStateTerminated{
Message: "Mock provider terminated container upon deletion",
FinishedAt: now,
Reason: "MockProviderPodContainerDeleted",
StartedAt: pod.Status.ContainerStatuses[idx].State.Running.StartedAt,
},
}
}
p.notifier(pod)
return nil
}
// GetPod returns a pod by name that is stored in memory.
func (p *MockProvider) GetPod(ctx context.Context, namespace, name string) (pod *v1.Pod, err error) {
ctx, span := trace.StartSpan(ctx, "GetPod")
defer func() {
span.SetStatus(err)
span.End()
}()
// Add the pod's coordinates to the current span.
ctx = addAttributes(ctx, span, namespaceKey, namespace, nameKey, name)
log.G(ctx).Infof("receive GetPod %q", name)
key, err := buildKeyFromNames(namespace, name)
if err != nil {
return nil, err
}
if pod, ok := p.pods[key]; ok {
return pod, nil
}
return nil, errdefs.NotFoundf("pod \"%s/%s\" is not known to the provider", namespace, name)
}
// GetContainerLogs retrieves the logs of a container by name from the provider.
func (p *MockProvider) GetContainerLogs(ctx context.Context, namespace, podName, containerName string, opts api.ContainerLogOpts) (io.ReadCloser, error) {
ctx, span := trace.StartSpan(ctx, "GetContainerLogs")
defer span.End()
// Add pod and container attributes to the current span.
ctx = addAttributes(ctx, span, namespaceKey, namespace, nameKey, podName, containerNameKey, containerName)
log.G(ctx).Infof("receive GetContainerLogs %q", podName)
return io.NopCloser(strings.NewReader("")), nil
}
// RunInContainer executes a command in a container in the pod, copying data
// between in/out/err and the container's stdin/stdout/stderr.
func (p *MockProvider) RunInContainer(ctx context.Context, namespace, name, container string, cmd []string, attach api.AttachIO) error {
log.G(context.TODO()).Infof("receive ExecInContainer %q", container)
return nil
}
// AttachToContainer attaches to the executing process of a container in the pod, copying data
// between in/out/err and the container's stdin/stdout/stderr.
func (p *MockProvider) AttachToContainer(ctx context.Context, namespace, name, container string, attach api.AttachIO) error {
log.G(ctx).Infof("receive AttachToContainer %q", container)
return nil
}
// PortForward forwards a local port to a port on the pod
func (p *MockProvider) PortForward(ctx context.Context, namespace, pod string, port int32, stream io.ReadWriteCloser) error {
log.G(ctx).Infof("receive PortForward %q", pod)
return nil
}
// GetPodStatus returns the status of a pod by name that is "running".
// returns nil if a pod by that name is not found.
func (p *MockProvider) GetPodStatus(ctx context.Context, namespace, name string) (*v1.PodStatus, error) {
ctx, span := trace.StartSpan(ctx, "GetPodStatus")
defer span.End()
// Add namespace and name as attributes to the current span.
ctx = addAttributes(ctx, span, namespaceKey, namespace, nameKey, name)
log.G(ctx).Infof("receive GetPodStatus %q", name)
pod, err := p.GetPod(ctx, namespace, name)
if err != nil {
return nil, err
}
return &pod.Status, nil
}
// GetPods returns a list of all pods known to be "running".
func (p *MockProvider) GetPods(ctx context.Context) ([]*v1.Pod, error) {
ctx, span := trace.StartSpan(ctx, "GetPods")
defer span.End()
log.G(ctx).Info("receive GetPods")
var pods []*v1.Pod
for _, pod := range p.pods {
pods = append(pods, pod)
}
return pods, nil
}
func (p *MockProvider) ConfigureNode(ctx context.Context, n *v1.Node) {
ctx, span := trace.StartSpan(ctx, "mock.ConfigureNode") //nolint:staticcheck,ineffassign
defer span.End()
if p.config.ProviderID != "" {
n.Spec.ProviderID = p.config.ProviderID
}
n.Status.Capacity = p.capacity()
n.Status.Allocatable = p.capacity()
n.Status.Conditions = p.nodeConditions()
n.Status.Addresses = p.nodeAddresses()
n.Status.DaemonEndpoints = p.nodeDaemonEndpoints()
os := p.operatingSystem
if os == "" {
os = "linux"
}
n.Status.NodeInfo.OperatingSystem = os
n.Status.NodeInfo.Architecture = "amd64"
n.Labels["alpha.service-controller.kubernetes.io/exclude-balancer"] = "true"
n.Labels["node.kubernetes.io/exclude-from-external-load-balancers"] = "true"
}
// Capacity returns a resource list containing the capacity limits.
func (p *MockProvider) capacity() v1.ResourceList {
rl := v1.ResourceList{
"cpu": resource.MustParse(p.config.CPU),
"memory": resource.MustParse(p.config.Memory),
"pods": resource.MustParse(p.config.Pods),
}
for k, v := range p.config.Others {
rl[v1.ResourceName(k)] = resource.MustParse(v)
}
return rl
}
// NodeConditions returns a list of conditions (Ready, OutOfDisk, etc), for updates to the node status
// within Kubernetes.
func (p *MockProvider) nodeConditions() []v1.NodeCondition {
// TODO: Make this configurable
return []v1.NodeCondition{
{
Type: "Ready",
Status: v1.ConditionFalse,
LastHeartbeatTime: metav1.Now(),
LastTransitionTime: metav1.Now(),
Reason: "KubeletPending",
Message: "kubelet is pending.",
},
{
Type: "OutOfDisk",
Status: v1.ConditionFalse,
LastHeartbeatTime: metav1.Now(),
LastTransitionTime: metav1.Now(),
Reason: "KubeletHasSufficientDisk",
Message: "kubelet has sufficient disk space available",
},
{
Type: "MemoryPressure",
Status: v1.ConditionFalse,
LastHeartbeatTime: metav1.Now(),
LastTransitionTime: metav1.Now(),
Reason: "KubeletHasSufficientMemory",
Message: "kubelet has sufficient memory available",
},
{
Type: "DiskPressure",
Status: v1.ConditionFalse,
LastHeartbeatTime: metav1.Now(),
LastTransitionTime: metav1.Now(),
Reason: "KubeletHasNoDiskPressure",
Message: "kubelet has no disk pressure",
},
{
Type: "NetworkUnavailable",
Status: v1.ConditionFalse,
LastHeartbeatTime: metav1.Now(),
LastTransitionTime: metav1.Now(),
Reason: "RouteCreated",
Message: "RouteController created a route",
},
}
}
// NodeAddresses returns a list of addresses for the node status
// within Kubernetes.
func (p *MockProvider) nodeAddresses() []v1.NodeAddress {
return []v1.NodeAddress{
{
Type: "InternalIP",
Address: p.internalIP,
},
}
}
// NodeDaemonEndpoints returns NodeDaemonEndpoints for the node status
// within Kubernetes.
func (p *MockProvider) nodeDaemonEndpoints() v1.NodeDaemonEndpoints {
return v1.NodeDaemonEndpoints{
KubeletEndpoint: v1.DaemonEndpoint{
Port: p.daemonEndpointPort,
},
}
}
// GetStatsSummary returns dummy stats for all pods known by this provider.
func (p *MockProvider) GetStatsSummary(ctx context.Context) (*stats.Summary, error) {
var span trace.Span
ctx, span = trace.StartSpan(ctx, "GetStatsSummary") //nolint: ineffassign,staticcheck
defer span.End()
// Grab the current timestamp so we can report it as the time the stats were generated.
time := metav1.NewTime(time.Now())
// Create the Summary object that will later be populated with node and pod stats.
res := &stats.Summary{}
// Populate the Summary object with basic node stats.
res.Node = stats.NodeStats{
NodeName: p.nodeName,
StartTime: metav1.NewTime(p.startTime),
}
// Populate the Summary object with dummy stats for each pod known by this provider.
for _, pod := range p.pods {
var (
// totalUsageNanoCores will be populated with the sum of the values of UsageNanoCores computes across all containers in the pod.
totalUsageNanoCores uint64
// totalUsageBytes will be populated with the sum of the values of UsageBytes computed across all containers in the pod.
totalUsageBytes uint64
)
// Create a PodStats object to populate with pod stats.
pss := stats.PodStats{
PodRef: stats.PodReference{
Name: pod.Name,
Namespace: pod.Namespace,
UID: string(pod.UID),
},
StartTime: pod.CreationTimestamp,
}
// Iterate over all containers in the current pod to compute dummy stats.
for _, container := range pod.Spec.Containers {
// Grab a dummy value to be used as the total CPU usage.
// The value should fit a uint32 in order to avoid overflows later on when computing pod stats.
/* #nosec */
dummyUsageNanoCores := uint64(rand.Uint32())
totalUsageNanoCores += dummyUsageNanoCores
// Create a dummy value to be used as the total RAM usage.
// The value should fit a uint32 in order to avoid overflows later on when computing pod stats.
/* #nosec */
dummyUsageBytes := uint64(rand.Uint32())
totalUsageBytes += dummyUsageBytes
// Append a ContainerStats object containing the dummy stats to the PodStats object.
pss.Containers = append(pss.Containers, stats.ContainerStats{
Name: container.Name,
StartTime: pod.CreationTimestamp,
CPU: &stats.CPUStats{
Time: time,
UsageNanoCores: &dummyUsageNanoCores,
},
Memory: &stats.MemoryStats{
Time: time,
UsageBytes: &dummyUsageBytes,
},
})
}
// Populate the CPU and RAM stats for the pod and append the PodsStats object to the Summary object to be returned.
pss.CPU = &stats.CPUStats{
Time: time,
UsageNanoCores: &totalUsageNanoCores,
}
pss.Memory = &stats.MemoryStats{
Time: time,
UsageBytes: &totalUsageBytes,
}
res.Pods = append(res.Pods, pss)
}
// Return the dummy stats.
return res, nil
}
func (p *MockProvider) generateMockMetrics(metricsMap map[string][]*dto.Metric, resourceType string, label []*dto.LabelPair) map[string][]*dto.Metric {
var (
cpuMetricSuffix = "_cpu_usage_seconds_total"
memoryMetricSuffix = "_memory_working_set_bytes"
dummyValue = float64(100)
)
if metricsMap == nil {
metricsMap = map[string][]*dto.Metric{}
}
finalCpuMetricName := resourceType + cpuMetricSuffix
finalMemoryMetricName := resourceType + memoryMetricSuffix
newCPUMetric := dto.Metric{
Label: label,
Counter: &dto.Counter{
Value: &dummyValue,
},
}
newMemoryMetric := dto.Metric{
Label: label,
Gauge: &dto.Gauge{
Value: &dummyValue,
},
}
// if metric family exists add to metric array
if cpuMetrics, ok := metricsMap[finalCpuMetricName]; ok {
metricsMap[finalCpuMetricName] = append(cpuMetrics, &newCPUMetric)
} else {
metricsMap[finalCpuMetricName] = []*dto.Metric{&newCPUMetric}
}
if memoryMetrics, ok := metricsMap[finalMemoryMetricName]; ok {
metricsMap[finalMemoryMetricName] = append(memoryMetrics, &newMemoryMetric)
} else {
metricsMap[finalMemoryMetricName] = []*dto.Metric{&newMemoryMetric}
}
return metricsMap
}
func (p *MockProvider) getMetricType(metricName string) *dto.MetricType {
var (
dtoCounterMetricType = dto.MetricType_COUNTER
dtoGaugeMetricType = dto.MetricType_GAUGE
cpuMetricSuffix = "_cpu_usage_seconds_total"
memoryMetricSuffix = "_memory_working_set_bytes"
)
if strings.HasSuffix(metricName, cpuMetricSuffix) {
return &dtoCounterMetricType
}
if strings.HasSuffix(metricName, memoryMetricSuffix) {
return &dtoGaugeMetricType
}
return nil
}
func (p *MockProvider) GetMetricsResource(ctx context.Context) ([]*dto.MetricFamily, error) {
var span trace.Span
ctx, span = trace.StartSpan(ctx, "GetMetricsResource") //nolint: ineffassign,staticcheck
defer span.End()
var (
nodeNameStr = "NodeName"
podNameStr = "PodName"
containerNameStr = "containerName"
)
nodeLabels := []*dto.LabelPair{
{
Name: &nodeNameStr,
Value: &p.nodeName,
},
}
metricsMap := p.generateMockMetrics(nil, "node", nodeLabels)
for _, pod := range p.pods {
podLabels := []*dto.LabelPair{
{
Name: &nodeNameStr,
Value: &p.nodeName,
},
{
Name: &podNameStr,
Value: &pod.Name,
},
}
metricsMap = p.generateMockMetrics(metricsMap, "pod", podLabels)
for _, container := range pod.Spec.Containers {
containerLabels := []*dto.LabelPair{
{
Name: &nodeNameStr,
Value: &p.nodeName,
},
{
Name: &podNameStr,
Value: &pod.Name,
},
{
Name: &containerNameStr,
Value: &container.Name,
},
}
metricsMap = p.generateMockMetrics(metricsMap, "container", containerLabels)
}
}
res := []*dto.MetricFamily{}
for metricName := range metricsMap {
tempName := metricName
tempMetrics := metricsMap[tempName]
metricFamily := dto.MetricFamily{
Name: &tempName,
Type: p.getMetricType(tempName),
Metric: tempMetrics,
}
res = append(res, &metricFamily)
}
return res, nil
}
// NotifyPods is called to set a pod notifier callback function. This should be called before any operations are done
// within the provider.
func (p *MockProvider) NotifyPods(ctx context.Context, notifier func(*v1.Pod)) {
p.notifier = notifier
}
func buildKeyFromNames(namespace string, name string) (string, error) {
return fmt.Sprintf("%s-%s", namespace, name), nil
}
// buildKey is a helper for building the "key" for the providers pod store.
func buildKey(pod *v1.Pod) (string, error) {
if pod.Namespace == "" {
return "", fmt.Errorf("pod namespace not found")
}
if pod.Name == "" {
return "", fmt.Errorf("pod name not found")
}
return buildKeyFromNames(pod.Namespace, pod.Name)
}
// addAttributes adds the specified attributes to the provided span.
// attrs must be an even-sized list of string arguments.
// Otherwise, the span won't be modified.
// TODO: Refactor and move to a "tracing utilities" package.
func addAttributes(ctx context.Context, span trace.Span, attrs ...string) context.Context {
if len(attrs)%2 == 1 {
return ctx
}
for i := 0; i < len(attrs); i += 2 {
ctx = span.WithField(ctx, attrs[i], attrs[i+1])
}
return ctx
}