HPA可以根据一定的性能指标自动监测以及控制集群的数量，使其满足一定的预定条件。默认使用cpu利用率来控制集群数量动态变化，同时也可以使用custom metrics来自定义指标。HPA支持的资源类型有replication controller, deployment or replica set
Controller Manager的参数可以设定周期性调整集群数量的时间--horizontal-pod-autoscaler-sync-period，默认为30秒。
研究一下HPA源码，HPA通过ControllerManager来控制监测集群性能指标，启动函数位置：
kubernetes/cmd/kube-controller-manager/app/controllermanager.go
func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunccontrollers["horizontalpodautoscaling"] = startHPAController启动HPA Controller函数
func startHPAController(ctx ControllerContext) (http.Handler, bool, error) {
if !ctx.AvailableResources[schema.GroupVersionResource{Group: "autoscaling", Version: "v1", Resource: "horizontalpodautoscalers"}] {
return nil, false, nil
}
if ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerUseRESTClients {
// use the new-style clients if support for custom metrics is enabled
return startHPAControllerWithRESTClient(ctx)
}
return startHPAControllerWithLegacyClient(ctx)
}研究一下ComponentConfig.HPAController对象
// HPAControllerConfiguration holds configuration for HPAController related features.
HPAController HPAControllerConfiguration// HPAControllerConfiguration contains elements describing HPAController.
type HPAControllerConfiguration struct {
// horizontalPodAutoscalerSyncPeriod is the period for syncing the number of
// pods in horizontal pod autoscaler.
HorizontalPodAutoscalerSyncPeriod metav1.Duration
// horizontalPodAutoscalerUpscaleForbiddenWindow is a period after which next upscale allowed.
HorizontalPodAutoscalerUpscaleForbiddenWindow metav1.Duration
// horizontalPodAutoscalerDownscaleForbiddenWindow is a period after which next downscale allowed.
HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration
// HorizontalPodAutoscalerDowncaleStabilizationWindow is a period for which autoscaler will look
// backwards and not scale down below any recommendation it made during that period.
HorizontalPodAutoscalerDownscaleStabilizationWindow metav1.Duration
// horizontalPodAutoscalerTolerance is the tolerance for when
// resource usage suggests upscaling/downscaling
HorizontalPodAutoscalerTolerance float64
// HorizontalPodAutoscalerUseRESTClients causes the HPA controller to use REST clients
// through the kube-aggregator when enabled, instead of using the legacy metrics client
// through the API server proxy.
HorizontalPodAutoscalerUseRESTClients bool
// HorizontalPodAutoscalerCPUInitializationPeriod is the period after pod start when CPU samples
// might be skipped.
HorizontalPodAutoscalerCPUInitializationPeriod metav1.Duration
// HorizontalPodAutoscalerInitialReadinessDelay is period after pod start during which readiness
// changes are treated as readiness being set for the first time. The only effect of this is that
// HPA will disregard CPU samples from unready pods that had last readiness change during that
// period.
HorizontalPodAutoscalerInitialReadinessDelay metav1.Duration
}HorizontalPodAutoscalerUseRESTClients bool参数决定使用apiserver api还是kube-aggregator api监测数据来控制集群数量
LegacyClient方式： 数据从Heapster中采集获取
func startHPAControllerWithLegacyClient(ctx ControllerContext) (http.Handler, bool, error) {
hpaClient := ctx.ClientBuilder.ClientOrDie("horizontal-pod-autoscaler")
metricsClient := metrics.NewHeapsterMetricsClient(
hpaClient,
metrics.DefaultHeapsterNamespace,
metrics.DefaultHeapsterScheme,
metrics.DefaultHeapsterService,
metrics.DefaultHeapsterPort,
)
return startHPAControllerWithMetricsClient(ctx, metricsClient)
}使用HeapsterMetricsClient方法，Heapster方式采集集群数据从1.11版本之后已经过时了，目前主要使用metrics-server方式
type HeapsterMetricsClient struct {
services v1core.ServiceInterface
podsGetter v1core.PodsGetter
heapsterScheme string
heapsterService string
heapsterPort string
}
func NewHeapsterMetricsClient(client clientset.Interface, namespace, scheme, service, port string) MetricsClient {
return &HeapsterMetricsClient{
services: client.CoreV1().Services(namespace),
podsGetter: client.CoreV1(),
heapsterScheme: scheme,
heapsterService: service,
heapsterPort: port,
}
}RESTClient方式：
func startHPAControllerWithRESTClient(ctx ControllerContext) (http.Handler, bool, error) {
clientConfig := ctx.ClientBuilder.ConfigOrDie("horizontal-pod-autoscaler")
hpaClient := ctx.ClientBuilder.ClientOrDie("horizontal-pod-autoscaler")
apiVersionsGetter := custom_metrics.NewAvailableAPIsGetter(hpaClient.Discovery())
// invalidate the discovery information roughly once per resync interval our API
// information is *at most* two resync intervals old.
go custom_metrics.PeriodicallyInvalidate(
apiVersionsGetter,
ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerSyncPeriod.Duration,
ctx.Stop)
metricsClient := metrics.NewRESTMetricsClient(
resourceclient.NewForConfigOrDie(clientConfig),
custom_metrics.NewForConfig(clientConfig, ctx.RESTMapper, apiVersionsGetter),
external_metrics.NewForConfigOrDie(clientConfig),
)
return startHPAControllerWithMetricsClient(ctx, metricsClient)
}func NewRESTMetricsClient(resourceClient resourceclient.PodMetricsesGetter, customClient customclient.CustomMetricsClient, externalClient externalclient.ExternalMetricsClient) MetricsClient {
return &restMetricsClient{
&resourceMetricsClient{resourceClient},
&customMetricsClient{customClient},
&externalMetricsClient{externalClient},
}
}三种MetricsClient客户端类型
resourceMetricsClient 数据来自于api server
// resourceMetricsClient implements the resource-metrics-related parts of MetricsClient,
// using data from the resource metrics API.
type resourceMetricsClient struct {
client resourceclient.PodMetricsesGetter
}PodMetricsInterface接口
// PodMetricsesGetter has a method to return a PodMetricsInterface.
// A group's client should implement this interface.
type PodMetricsesGetter interface {
PodMetricses(namespace string) PodMetricsInterface
}
// PodMetricsInterface has methods to work with PodMetrics resources.
type PodMetricsInterface interface {
Get(name string, options v1.GetOptions) (*v1beta1.PodMetrics, error)
List(opts v1.ListOptions) (*v1beta1.PodMetricsList, error)
Watch(opts v1.ListOptions) (watch.Interface, error)
PodMetricsExpansion
}customMetricsClient 数据来自于自定义
// customMetricsClient implements the custom-metrics-related parts of MetricsClient,
// using data from the custom metrics API.
type customMetricsClient struct {
client customclient.CustomMetricsClient
}// CustomMetricsClient is a client for fetching metrics
// describing both root-scoped and namespaced resources.
type CustomMetricsClient interface {
RootScopedMetricsGetter
NamespacedMetricsGetter
}
// RootScopedMetricsGetter provides access to an interface for fetching
// metrics describing root-scoped objects. Note that metrics describing
// a namespace are simply considered a special case of root-scoped metrics.
type RootScopedMetricsGetter interface {
RootScopedMetrics() MetricsInterface
}
// NamespacedMetricsGetter provides access to an interface for fetching
// metrics describing resources in a particular namespace.
type NamespacedMetricsGetter interface {
NamespacedMetrics(namespace string) MetricsInterface
}
// MetricsInterface provides access to metrics describing Kubernetes objects.
type MetricsInterface interface {
// GetForObject fetchs the given metric describing the given object.
GetForObject(groupKind schema.GroupKind, name string, metricName string, metricSelector labels.Selector) (*v1beta2.MetricValue, error)
// GetForObjects fetches the given metric describing all objects of the given
// type matching the given label selector (or simply all objects of the given type
// if the selector is nil).
GetForObjects(groupKind schema.GroupKind, selector labels.Selector, metricName string, metricSelector labels.Selector) (*v1beta2.MetricValueList, error)
}
externalMetricsClient 数据来自于外部api
// externalMetricsClient implenets the external metrics related parts of MetricsClient,
// using data from the external metrics API.
type externalMetricsClient struct {
client externalclient.ExternalMetricsClient
}// ExternalMetricsClient is a client for fetching external metrics.
type ExternalMetricsClient interface {
NamespacedMetricsGetter
}
// NamespacedMetricsGetter provides access to an interface for fetching
// metrics in a particular namespace.
type NamespacedMetricsGetter interface {
NamespacedMetrics(namespace string) MetricsInterface
}
// MetricsInterface provides access to external metrics.
type MetricsInterface interface {
// List fetches the metric for the given namespace that maches the given
// metricSelector.
List(metricName string, metricSelector labels.Selector) (*v1beta1.ExternalMetricValueList, error)
}MetricsClient定义了监控数据的来源，主要有LegacyClient以及RESTClient，接下来启动HPAControllerManager
func startHPAControllerWithMetricsClient(ctx ControllerContext, metricsClient metrics.MetricsClient) (http.Handler, bool, error) {
hpaClient := ctx.ClientBuilder.ClientOrDie("horizontal-pod-autoscaler")
hpaClientConfig := ctx.ClientBuilder.ConfigOrDie("horizontal-pod-autoscaler")
// we don't use cached discovery because DiscoveryScaleKindResolver does its own caching,
// so we want to re-fetch every time when we actually ask for it
scaleKindResolver := scale.NewDiscoveryScaleKindResolver(hpaClient.Discovery())
scaleClient, err := scale.NewForConfig(hpaClientConfig, ctx.RESTMapper, dynamic.LegacyAPIPathResolverFunc, scaleKindResolver)
if err != nil {
return nil, false, err
}
go podautoscaler.NewHorizontalController(
hpaClient.CoreV1(),
scaleClient,
hpaClient.AutoscalingV1(),
ctx.RESTMapper,
metricsClient,
ctx.InformerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
ctx.InformerFactory.Core().V1().Pods(),
ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerSyncPeriod.Duration,
ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerDownscaleStabilizationWindow.Duration,
ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerTolerance,
ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerCPUInitializationPeriod.Duration,
ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerInitialReadinessDelay.Duration,
).Run(ctx.Stop)
return nil, true, nil
}创建HorizontalController，构造HorizontalController结构体
// HorizontalController is responsible for the synchronizing HPA objects stored
// in the system with the actual deployments/replication controllers they
// control.
type HorizontalController struct {
scaleNamespacer scaleclient.ScalesGetter
hpaNamespacer autoscalingclient.HorizontalPodAutoscalersGetter
mapper apimeta.RESTMapper
replicaCalc *ReplicaCalculator
eventRecorder record.EventRecorder
downscaleStabilisationWindow time.Duration
// hpaLister is able to list/get HPAs from the shared cache from the informer passed in to
// NewHorizontalController.
hpaLister autoscalinglisters.HorizontalPodAutoscalerLister
hpaListerSynced cache.InformerSynced
// podLister is able to list/get Pods from the shared cache from the informer passed in to
// NewHorizontalController.
podLister corelisters.PodLister
podListerSynced cache.InformerSynced
// Controllers that need to be synced
queue workqueue.RateLimitingInterface
// Latest unstabilized recommendations for each autoscaler.
recommendations map[string][]timestampedRecommendation
}ScalesGetter
// ScalesGetter can produce a ScaleInterface
// for a particular namespace.
type ScalesGetter interface {
Scales(namespace string) ScaleInterface
}
// ScaleInterface can fetch and update scales for
// resources in a particular namespace which implement
// the scale subresource.
type ScaleInterface interface {
// Get fetches the scale of the given scalable resource.
Get(resource schema.GroupResource, name string) (*autoscalingapi.Scale, error)
// Update updates the scale of the given scalable resource.
Update(resource schema.GroupResource, scale *autoscalingapi.Scale) (*autoscalingapi.Scale, error)
}HorizontalPodAutoscalersGetter返回HorizontalPodAutoscalerInterface，其定义了对于HorizontalPodAutoscaler的状态以及操作方法
// HorizontalPodAutoscalersGetter has a method to return a HorizontalPodAutoscalerInterface.
// A group's client should implement this interface.
type HorizontalPodAutoscalersGetter interface {
HorizontalPodAutoscalers(namespace string) HorizontalPodAutoscalerInterface
}
// HorizontalPodAutoscalerInterface has methods to work with HorizontalPodAutoscaler resources.
type HorizontalPodAutoscalerInterface interface {
Create(*v1.HorizontalPodAutoscaler) (*v1.HorizontalPodAutoscaler, error)
Update(*v1.HorizontalPodAutoscaler) (*v1.HorizontalPodAutoscaler, error)
UpdateStatus(*v1.HorizontalPodAutoscaler) (*v1.HorizontalPodAutoscaler, error)
Delete(name string, options *metav1.DeleteOptions) error
DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error
Get(name string, options metav1.GetOptions) (*v1.HorizontalPodAutoscaler, error)
List(opts metav1.ListOptions) (*v1.HorizontalPodAutoscalerList, error)
Watch(opts metav1.ListOptions) (watch.Interface, error)
Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.HorizontalPodAutoscaler, err error)
HorizontalPodAutoscalerExpansion
}具体实现client-go/kubernetes/typed/autoscaling/v1/horizontalpodautoscaler.go，主要利用restful接口改变相应资源的状态以及增删改查等操作。创建方法
// Create takes the representation of a horizontalPodAutoscaler and creates it. Returns the server's representation of the horizontalPodAutoscaler, and an error, if there is any.
func (c *horizontalPodAutoscalers) Create(horizontalPodAutoscaler *v1.HorizontalPodAutoscaler) (result *v1.HorizontalPodAutoscaler, err error) {
result = &v1.HorizontalPodAutoscaler{}
err = c.client.Post().
Namespace(c.ns).
Resource("horizontalpodautoscalers").
Body(horizontalPodAutoscaler).
Do().
Into(result)
return
}HorizontalPodAutoscaler结构体
// configuration of a horizontal pod autoscaler.
type HorizontalPodAutoscaler struct {
metav1.TypeMeta `json:",inline"`
// Standard object metadata. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata
// +optional
metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
// behaviour of autoscaler. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status.
// +optional
Spec HorizontalPodAutoscalerSpec `json:"spec,omitempty" protobuf:"bytes,2,opt,name=spec"`
// current information about the autoscaler.
// +optional
Status HorizontalPodAutoscalerStatus `json:"status,omitempty" protobuf:"bytes,3,opt,name=status"`
}HorizontalPodAutoscalerSpec// specification of a horizontal pod autoscaler.
type HorizontalPodAutoscalerSpec struct {
// reference to scaled resource; horizontal pod autoscaler will learn the current resource consumption
// and will set the desired number of pods by using its Scale subresource.
ScaleTargetRef CrossVersionObjectReference `json:"scaleTargetRef" protobuf:"bytes,1,opt,name=scaleTargetRef"`
// lower limit for the number of pods that can be set by the autoscaler, default 1.
// +optional
MinReplicas *int32 `json:"minReplicas,omitempty" protobuf:"varint,2,opt,name=minReplicas"`
// upper limit for the number of pods that can be set by the autoscaler; cannot be smaller than MinReplicas.
MaxReplicas int32 `json:"maxReplicas" protobuf:"varint,3,opt,name=maxReplicas"`
// target average CPU utilization (represented as a percentage of requested CPU) over all the pods;
// if not specified the default autoscaling policy will be used.
// +optional
TargetCPUUtilizationPercentage *int32 `json:"targetCPUUtilizationPercentage,omitempty" protobuf:"varint,4,opt,name=targetCPUUtilizationPercentage"`
}HorizontalPodAutoscalerStatus// current status of a horizontal pod autoscaler
type HorizontalPodAutoscalerStatus struct {
// most recent generation observed by this autoscaler.
// +optional
ObservedGeneration *int64 `json:"observedGeneration,omitempty" protobuf:"varint,1,opt,name=observedGeneration"`
// last time the HorizontalPodAutoscaler scaled the number of pods;
// used by the autoscaler to control how often the number of pods is changed.
// +optional
LastScaleTime *metav1.Time `json:"lastScaleTime,omitempty" protobuf:"bytes,2,opt,name=lastScaleTime"`
// current number of replicas of pods managed by this autoscaler.
CurrentReplicas int32 `json:"currentReplicas" protobuf:"varint,3,opt,name=currentReplicas"`
// desired number of replicas of pods managed by this autoscaler.
DesiredReplicas int32 `json:"desiredReplicas" protobuf:"varint,4,opt,name=desiredReplicas"`
// current average CPU utilization over all pods, represented as a percentage of requested CPU,
// e.g. 70 means that an average pod is using now 70% of its requested CPU.
// +optional
CurrentCPUUtilizationPercentage *int32 `json:"currentCPUUtilizationPercentage,omitempty" protobuf:"varint,5,opt,name=currentCPUUtilizationPercentage"`
}HorizontalPodAutoscaler对应了k8s资源类型
kind: HorizontalPodAutoscalerapiVersion: autoscaling/v2beta2
kind: HorizontalPodAutoscaler
metadata:
name: php-apache
namespace: default
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: php-apache
minReplicas: 1
maxReplicas: 10
metrics:
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: 50
status:
observedGeneration: 1
lastScaleTime: <some-time>
currentReplicas: 1
desiredReplicas: 1
currentMetrics:
- type: Resource
resource:
name: cpu
current:
averageUtilization: 0
averageValue: 0ReplicaCalculator 计算副本数量结构体
type ReplicaCalculator struct {
metricsClient metricsclient.MetricsClient
podLister corelisters.PodLister
tolerance float64
cpuInitializationPeriod time.Duration
delayOfInitialReadinessStatus time.Duration
}HorizontalController最终构建过程
broadcaster := record.NewBroadcaster()
broadcaster.StartLogging(klog.Infof)
broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: evtNamespacer.Events("")})
recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "horizontal-pod-autoscaler"})
hpaController := &HorizontalController{
eventRecorder: recorder,
scaleNamespacer: scaleNamespacer,
hpaNamespacer: hpaNamespacer,
downscaleStabilisationWindow: downscaleStabilisationWindow,
queue: workqueue.NewNamedRateLimitingQueue(NewDefaultHPARateLimiter(resyncPeriod), "horizontalpodautoscaler"),
mapper: mapper,
recommendations: map[string][]timestampedRecommendation{},
}
hpaInformer.Informer().AddEventHandlerWithResyncPeriod(
cache.ResourceEventHandlerFuncs{
AddFunc: hpaController.enqueueHPA,
UpdateFunc: hpaController.updateHPA,
DeleteFunc: hpaController.deleteHPA,
},
resyncPeriod,
)
hpaController.hpaLister = hpaInformer.Lister()
hpaController.hpaListerSynced = hpaInformer.Informer().HasSynced
hpaController.podLister = podInformer.Lister()
hpaController.podListerSynced = podInformer.Informer().HasSynced
replicaCalc := NewReplicaCalculator(
metricsClient,
hpaController.podLister,
tolerance,
cpuInitializationPeriod,
delayOfInitialReadinessStatus,
)
hpaController.replicaCalc = replicaCalc总结
HPA功能有HPAController实现 HPAController由ControllerContext以及MetricsClient构建产生
首先产生MetricsClient,定义监控数据的来源，主要有LegacyClient以及RESTClient方式，RESTClient由三种方式组成用来扩展监控数据指标：resourceMetricsClient customMetricsClient externalMetricsClient
然后构建NewHorizontalController，启动Run方法。