HPA由HPAController控制实现，启动代码 kubernetes/cmd/kube-controller-manager/app/controllermanager.go
func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunc
controllers["horizontalpodautoscaling"] = startHPAControllerfunc startHPAController(ctx ControllerContext) (http.Handler, bool, error)构建MetricsClient
func startHPAControllerWithRESTClient(ctx ControllerContext) (http.Handler, bool, error)func startHPAControllerWithLegacyClient(ctx ControllerContext) (http.Handler, bool, error)由MetricsClient启动HPAController
func startHPAControllerWithMetricsClient(ctx ControllerContext, metricsClient metrics.MetricsClient) (http.Handler, bool, error)构建HorizontalController
go podautoscaler.NewHorizontalController(
hpaClient.CoreV1(),
scaleClient,
hpaClient.AutoscalingV1(),
ctx.RESTMapper,
metricsClient,
ctx.InformerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
ctx.InformerFactory.Core().V1().Pods(),
ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerSyncPeriod.Duration,
ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerDownscaleStabilizationWindow.Duration,
ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerTolerance,
ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerCPUInitializationPeriod.Duration,
ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerInitialReadinessDelay.Duration,
).Run(ctx.Stop)研究一下具体的Run控制过程
// Run begins watching and syncing.
func (a *HorizontalController) Run(stopCh <-chan struct{}) {
defer utilruntime.HandleCrash()
defer a.queue.ShutDown()
klog.Infof("Starting HPA controller")
defer klog.Infof("Shutting down HPA controller")
if !controller.WaitForCacheSync("HPA", stopCh, a.hpaListerSynced, a.podListerSynced) {
return
}
// start a single worker (we may wish to start more in the future)
go wait.Until(a.worker, time.Second, stopCh)
<-stopCh
}defer是golang语言中的关键字，用于资源的释放，会在函数返回之前进行调用。
如果系统发生panic，则执行相应处理程序并且释放资源。有关linux kernel panic概念
https://blog.csdn.net/dake_160413/article/details/64443274
defer utilruntime.HandleCrash()RateLimitingInterface接口
defer a.queue.ShutDown()queue workqueue.RateLimitingInterface等待缓存同步
if !controller.WaitForCacheSync("HPA", stopCh, a.hpaListerSynced, a.podListerSynced) {
return
}监听同步对象有两种 HPA Pod
// hpaLister is able to list/get HPAs from the shared cache from the informer passed in to
// NewHorizontalController.
hpaLister autoscalinglisters.HorizontalPodAutoscalerLister
hpaListerSynced cache.InformerSynced
// podLister is able to list/get Pods from the shared cache from the informer passed in to
// NewHorizontalController.
podLister corelisters.PodLister
podListerSynced cache.InformerSyncedfunc WaitForCacheSync(controllerName string, stopCh <-chan struct{}, cacheSyncs ...cache.InformerSynced) bool// WaitForCacheSync waits for caches to populate. It returns true if it was successful, false
// if the controller should shutdown
func WaitForCacheSync(stopCh <-chan struct{}, cacheSyncs ...InformerSynced) bool {
err := wait.PollUntil(syncedPollPeriod,
func() (bool, error) {
for _, syncFunc := range cacheSyncs {
if !syncFunc() {
return false, nil
}
}
return true, nil
},
stopCh)
if err != nil {
klog.V(2).Infof("stop requested")
return false
}
klog.V(4).Infof("caches populated")
return true
}// InformerSynced is a function that can be used to determine if an informer has synced. This is useful for determining if caches have synced.
type InformerSynced func() boolInformer功能应该是观察者模式，可以添加多个时间，等到缓存更新时则通知各个时间
// SharedInformer has a shared data cache and is capable of distributing notifications for changes
// to the cache to multiple listeners who registered via AddEventHandler.这里等待HPA以及Pod缓存更新事件
最后周期性执行worker函数
// start a single worker (we may wish to start more in the future)
go wait.Until(a.worker, time.Second, stopCh)
<-stopChUntil函数 以固定的周期执行某个函数直到stop channel is closed
// Until loops until stop channel is closed, running f every period.
//
// Until is syntactic sugar on top of JitterUntil with zero jitter factor and
// with sliding = true (which means the timer for period starts after the f
// completes).
func Until(f func(), period time.Duration, stopCh <-chan struct{}) {
JitterUntil(f, period, 0.0, true, stopCh)
}work函数
func (a *HorizontalController) worker() {
for a.processNextWorkItem() {
}
klog.Infof("horizontal pod autoscaler controller worker shutting down")
}processNextWorkItem函数
func (a *HorizontalController) processNextWorkItem() bool {
key, quit := a.queue.Get()
if quit {
return false
}
defer a.queue.Done(key)
err := a.reconcileKey(key.(string))
if err == nil {
// don't "forget" here because we want to only process a given HPA once per resync interval
return true
}
a.queue.AddRateLimited(key)
utilruntime.HandleError(err)
return true
}从queue中取一个元素
// Controllers that need to be synced
queue workqueue.RateLimitingInterface研究一下workqueue中RateLimitingInterface
// RateLimitingInterface is an interface that rate limits items being added to the queue.
type RateLimitingInterface interface {
DelayingInterface
// AddRateLimited adds an item to the workqueue after the rate limiter says it's ok
AddRateLimited(item interface{})
// Forget indicates that an item is finished being retried. Doesn't matter whether it's for perm failing
// or for success, we'll stop the rate limiter from tracking it. This only clears the `rateLimiter`, you
// still have to call `Done` on the queue.
Forget(item interface{})
// NumRequeues returns back how many times the item was requeued
NumRequeues(item interface{}) int
}其中复合了DelayingInterface 可以延时一段时间添加某个元素
// DelayingInterface is an Interface that can Add an item at a later time. This makes it easier to
// requeue items after failures without ending up in a hot-loop.
type DelayingInterface interface {
Interface
// AddAfter adds an item to the workqueue after the indicated duration has passed
AddAfter(item interface{}, duration time.Duration)
}Interface接口
type Interface interface {
Add(item interface{})
Len() int
Get() (item interface{}, shutdown bool)
Done(item interface{})
ShutDown()
ShuttingDown() bool
}具体实现在
client-go/util/workqueue/queue.go看一下各个方法的说明
// Add marks item as needing processing.
func (q *Type) Add(item interface{}) {// Len returns the current queue length, for informational purposes only. You
// shouldn't e.g. gate a call to Add() or Get() on Len() being a particular
// value, that can't be synchronized properly.
func (q *Type) Len() int {// Get blocks until it can return an item to be processed. If shutdown = true,
// the caller should end their goroutine. You must call Done with item when you
// have finished processing it.
func (q *Type) Get() (item interface{}, shutdown bool) {// Done marks item as done processing, and if it has been marked as dirty again
// while it was being processed, it will be re-added to the queue for
// re-processing.
func (q *Type) Done(item interface{}) {// ShutDown will cause q to ignore all new items added to it. As soon as the
// worker goroutines have drained the existing items in the queue, they will be
// instructed to exit.
func (q *Type) ShutDown() {workqueue定义了一个同步工作队列，可以实现同步控制
回到
key, quit := a.queue.Get()
if quit {
return false
}workqueue中存放的元素是 NewHorizontalController方法中
queue: workqueue.NewNamedRateLimitingQueue(NewDefaultHPARateLimiter(resyncPeriod), "horizontalpodautoscaler")DefaultHPARateLimiter
// NewDefaultHPARateLimiter creates a rate limiter which limits overall (as per the
// default controller rate limiter), as well as per the resync interval
func NewDefaultHPARateLimiter(interval time.Duration) workqueue.RateLimiter {
return NewFixedItemIntervalRateLimiter(interval)
}
func NewFixedItemIntervalRateLimiter(interval time.Duration) workqueue.RateLimiter {
return &FixedItemIntervalRateLimiter{
interval: interval,
}
}// FixedItemIntervalRateLimiter limits items to a fixed-rate interval
type FixedItemIntervalRateLimiter struct {
interval time.Duration
}新建一个queue,监听的元素为 horizontalpodautoscaler
queue: workqueue.NewNamedRateLimitingQueue(NewDefaultHPARateLimiter(resyncPeriod), "horizontalpodautoscaler"),func NewNamedRateLimitingQueue(rateLimiter RateLimiter, name string) RateLimitingInterface {
return &rateLimitingType{
DelayingInterface: NewNamedDelayingQueue(name),
rateLimiter: rateLimiter,
}
}func NewNamedDelayingQueue(name string) DelayingInterface {
return newDelayingQueue(clock.RealClock{}, name)
}
func newDelayingQueue(clock clock.Clock, name string) DelayingInterface {
ret := &delayingType{
Interface: NewNamed(name),
clock: clock,
heartbeat: clock.NewTicker(maxWait),
stopCh: make(chan struct{}),
waitingForAddCh: make(chan *waitFor, 1000),
metrics: newRetryMetrics(name),
}
go ret.waitingLoop()
return ret
}func newRetryMetrics(name string) retryMetrics {
var ret *defaultRetryMetrics
if len(name) == 0 {
return ret
}
return &defaultRetryMetrics{
retries: globalMetricsFactory.metricsProvider.NewRetriesMetric(name),
}
}可知queue为一个RateLimitingInterface 监听的数据元素为horizontalpodautoscaler metrics数据
可以使用kubectl或者yaml文件来声明HPA资源对象HorizontalPodAutoscaler，声明的HPA会放入到一个queue中，然后启动一个线程来不断取出queue中HPA周期性执行
回到processNextWorkItem方法，每个周期每次取出一个队列中首HPA来执行（可以研究一下改成多线程的控制），等待该HPA完成
key, quit := a.queue.Get()
if quit {
return false
}
defer a.queue.Done(key)从队列中取出待执行的HPA策略的metadata
执行reconcileKey方法 取出命名空间以及hpa-name
func (a *HorizontalController) reconcileKey(key string) error {
namespace, name, err := cache.SplitMetaNamespaceKey(key)
if err != nil {
return err
}
hpa, err := a.hpaLister.HorizontalPodAutoscalers(namespace).Get(name)
if errors.IsNotFound(err) {
klog.Infof("Horizontal Pod Autoscaler %s has been deleted in %s", name, namespace)
delete(a.recommendations, key)
return nil
}
return a.reconcileAutoscaler(hpa, key)
}根据hpa name 获取hpa策略，执行具体的算法来控制集群副本数量
hpa, err := a.hpaLister.HorizontalPodAutoscalers(namespace).Get(name)
if errors.IsNotFound(err) {
klog.Infof("Horizontal Pod Autoscaler %s has been deleted in %s", name, namespace)
delete(a.recommendations, key)
return nil
}
return a.reconcileAutoscaler(hpa, key)HorizontalPodAutoscalerLister 监听器模式
// HorizontalPodAutoscalerLister helps list HorizontalPodAutoscalers.
type HorizontalPodAutoscalerLister interface {
// List lists all HorizontalPodAutoscalers in the indexer.
List(selector labels.Selector) (ret []*v1.HorizontalPodAutoscaler, err error)
// HorizontalPodAutoscalers returns an object that can list and get HorizontalPodAutoscalers.
HorizontalPodAutoscalers(namespace string) HorizontalPodAutoscalerNamespaceLister
HorizontalPodAutoscalerListerExpansion
}// HorizontalPodAutoscalerNamespaceLister helps list and get HorizontalPodAutoscalers.
type HorizontalPodAutoscalerNamespaceLister interface {
// List lists all HorizontalPodAutoscalers in the indexer for a given namespace.
List(selector labels.Selector) (ret []*v1.HorizontalPodAutoscaler, err error)
// Get retrieves the HorizontalPodAutoscaler from the indexer for a given namespace and name.
Get(name string) (*v1.HorizontalPodAutoscaler, error)
HorizontalPodAutoscalerNamespaceListerExpansion
}最终返回HorizontalPodAutoscale，其对应定义了yaml文件
最后执行算法来调整集群副本数量 reconcileAutoscaler
func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.HorizontalPodAutoscaler, key string) error回到processNextWorkItem方法
func (a *HorizontalController) processNextWorkItem() bool {
key, quit := a.queue.Get()
if quit {
return false
}
defer a.queue.Done(key)
err := a.reconcileKey(key.(string))
if err == nil {
// don't "forget" here because we want to only process a given HPA once per resync interval
return true
}
a.queue.AddRateLimited(key)
utilruntime.HandleError(err)
return true
}如果该hpa执行成功则直接返回，否则再次进入队列，延时执行
接下来具体研究一下hpa算法细节reconcileAutoscaler
总结：
HorizontalController构建成功，启动run方法
等待hpa,pod的缓存监听同步 hpa对象放入到queue中
周期性执行worker方法来取出queue中hpa对象，执行相应逻辑，执行成功继续执行下一个hpa,执行失败，则该hpa进入queue中延时执行