kube-scheduler源码分析（二）之 registerAlgorithmProvider

以下代码分析基于 kubernetes v1.12.0 版本。

此部分主要介绍调度中使用的各种调度算法，包括调度算法的注册部分。注册部分的代码主要在/pkg/scheduler/algorithmprovider中，具体的预选策略和优选策略的算法实现在/pkg/scheduler/algorithm中。

1. ApplyFeatureGates

注册调度算法的调用入口在SchedulerCommand的Run函数中。

此部分代码位于/cmd/kube-scheduler/app/server.go

  1. // Run runs the Scheduler.
  2. func Run(c schedulerserverconfig.CompletedConfig, stopCh <-chan struct{}) error {
  3.     ...
  4.     // Apply algorithms based on feature gates.
  5.     // TODO: make configurable?
  6.     algorithmprovider.ApplyFeatureGates()
  7.   ...
  8. }

ApplyFeatureGates的具体实现在pkg/scheduler/algorithmprovider的包中。

此部分代码位于/pkg/scheduler/algorithmprovider/plugins.go

  1. // ApplyFeatureGates applies algorithm by feature gates.
  2. func ApplyFeatureGates() {
  3.     defaults.ApplyFeatureGates()
  4. }

ApplyFeatureGates具体实现如下：

此部分代码位于/pkg/scheduler/algorithmprovider/defaults/defaults.go

根据feature移除部分调度策略。

  1. // ApplyFeatureGates applies algorithm by feature gates.
  2. func ApplyFeatureGates() {
  3.     if utilfeature.DefaultFeatureGate.Enabled(features.TaintNodesByCondition) {
  4.         // Remove "CheckNodeCondition", "CheckNodeMemoryPressure", "CheckNodePIDPressurePred"
  5.         // and "CheckNodeDiskPressure" predicates
  6.         factory.RemoveFitPredicate(predicates.CheckNodeConditionPred)
  7.         factory.RemoveFitPredicate(predicates.CheckNodeMemoryPressurePred)
  8.         factory.RemoveFitPredicate(predicates.CheckNodeDiskPressurePred)
  9.         factory.RemoveFitPredicate(predicates.CheckNodePIDPressurePred)
  10.         // Remove key "CheckNodeCondition", "CheckNodeMemoryPressure" and "CheckNodeDiskPressure"
  11.         // from ALL algorithm provider
  12.         // The key will be removed from all providers which in algorithmProviderMap[]
  13.         // if you just want remove specific provider, call func RemovePredicateKeyFromAlgoProvider()
  14.         factory.RemovePredicateKeyFromAlgorithmProviderMap(predicates.CheckNodeConditionPred)
  15.         factory.RemovePredicateKeyFromAlgorithmProviderMap(predicates.CheckNodeMemoryPressurePred)
  16.         factory.RemovePredicateKeyFromAlgorithmProviderMap(predicates.CheckNodeDiskPressurePred)
  17.         factory.RemovePredicateKeyFromAlgorithmProviderMap(predicates.CheckNodePIDPressurePred)
  19.         // Fit is determined based on whether a pod can tolerate all of the node's taints
  20.         factory.RegisterMandatoryFitPredicate(predicates.PodToleratesNodeTaintsPred, predicates.PodToleratesNodeTaints)
  21.         // Fit is determined based on whether a pod can tolerate unschedulable of node
  22.         factory.RegisterMandatoryFitPredicate(predicates.CheckNodeUnschedulablePred, predicates.CheckNodeUnschedulablePredicate)
  23.         // Insert Key "PodToleratesNodeTaints" and "CheckNodeUnschedulable" To All Algorithm Provider
  24.         // The key will insert to all providers which in algorithmProviderMap[]
  25.         // if you just want insert to specific provider, call func InsertPredicateKeyToAlgoProvider()
  26.         factory.InsertPredicateKeyToAlgorithmProviderMap(predicates.PodToleratesNodeTaintsPred)
  27.         factory.InsertPredicateKeyToAlgorithmProviderMap(predicates.CheckNodeUnschedulablePred)
  29.         glog.Warningf("TaintNodesByCondition is enabled, PodToleratesNodeTaints predicate is mandatory")
  30.     }
  32.     // Prioritizes nodes that satisfy pod's resource limits
  33.     if utilfeature.DefaultFeatureGate.Enabled(features.ResourceLimitsPriorityFunction) {
  34.         factory.RegisterPriorityFunction2("ResourceLimitsPriority", priorities.ResourceLimitsPriorityMap, nil, 1)
  35.     }
  37. }

2. init

当函数逻辑调用到algorithmprovider包时，就会自动调用init的初始化函数，此部分主要包括对预选算法和优选算法的注册。

此部分代码位于/pkg/scheduler/algorithmprovider/defaults/defaults.go

  1. func init() {
  2.     // Register functions that extract metadata used by predicates and priorities computations.
  3.     factory.RegisterPredicateMetadataProducerFactory(
  4.         func(args factory.PluginFactoryArgs) algorithm.PredicateMetadataProducer {
  5.             return predicates.NewPredicateMetadataFactory(args.PodLister)
  6.         })
  7.     factory.RegisterPriorityMetadataProducerFactory(
  8.         func(args factory.PluginFactoryArgs) algorithm.PriorityMetadataProducer {
  9.             return priorities.NewPriorityMetadataFactory(args.ServiceLister, args.ControllerLister, args.ReplicaSetLister, args.StatefulSetLister)
  10.         })
  12.     registerAlgorithmProvider(defaultPredicates(), defaultPriorities())
  14.     // IMPORTANT NOTES for predicate developers:
  15.     // We are using cached predicate result for pods belonging to the same equivalence class.
  16.     // So when implementing a new predicate, you are expected to check whether the result
  17.     // of your predicate function can be affected by related API object change (ADD/DELETE/UPDATE).
  18.     // If yes, you are expected to invalidate the cached predicate result for related API object change.
  19.     // For example:
  20.     // https://github.com/kubernetes/kubernetes/blob/36a218e/plugin/pkg/scheduler/factory/factory.go#L422
  22.     // Registers predicates and priorities that are not enabled by default, but user can pick when creating their
  23.     // own set of priorities/predicates.
  25.     // PodFitsPorts has been replaced by PodFitsHostPorts for better user understanding.
  26.     // For backwards compatibility with 1.0, PodFitsPorts is registered as well.
  27.     factory.RegisterFitPredicate("PodFitsPorts", predicates.PodFitsHostPorts)
  28.     // Fit is defined based on the absence of port conflicts.
  29.     // This predicate is actually a default predicate, because it is invoked from
  30.     // predicates.GeneralPredicates()
  31.     factory.RegisterFitPredicate(predicates.PodFitsHostPortsPred, predicates.PodFitsHostPorts)
  32.     // Fit is determined by resource availability.
  33.     // This predicate is actually a default predicate, because it is invoked from
  34.     // predicates.GeneralPredicates()
  35.     factory.RegisterFitPredicate(predicates.PodFitsResourcesPred, predicates.PodFitsResources)
  36.     // Fit is determined by the presence of the Host parameter and a string match
  37.     // This predicate is actually a default predicate, because it is invoked from
  38.     // predicates.GeneralPredicates()
  39.     factory.RegisterFitPredicate(predicates.HostNamePred, predicates.PodFitsHost)
  40.     // Fit is determined by node selector query.
  41.     factory.RegisterFitPredicate(predicates.MatchNodeSelectorPred, predicates.PodMatchNodeSelector)
  43.     // ServiceSpreadingPriority is a priority config factory that spreads pods by minimizing
  44.     // the number of pods (belonging to the same service) on the same node.
  45.     // Register the factory so that it's available, but do not include it as part of the default priorities
  46.     // Largely replaced by "SelectorSpreadPriority", but registered for backward compatibility with 1.0
  47.     factory.RegisterPriorityConfigFactory(
  48.         "ServiceSpreadingPriority",
  49.         factory.PriorityConfigFactory{
  50.             MapReduceFunction: func(args factory.PluginFactoryArgs) (algorithm.PriorityMapFunction, algorithm.PriorityReduceFunction) {
  51.                 return priorities.NewSelectorSpreadPriority(args.ServiceLister, algorithm.EmptyControllerLister{}, algorithm.EmptyReplicaSetLister{}, algorithm.EmptyStatefulSetLister{})
  52.             },
  53.             Weight: 1,
  54.         },
  55.     )
  56.     // EqualPriority is a prioritizer function that gives an equal weight of one to all nodes
  57.     // Register the priority function so that its available
  58.     // but do not include it as part of the default priorities
  59.     factory.RegisterPriorityFunction2("EqualPriority", core.EqualPriorityMap, nil, 1)
  60.     // Optional, cluster-autoscaler friendly priority function - give used nodes higher priority.
  61.     factory.RegisterPriorityFunction2("MostRequestedPriority", priorities.MostRequestedPriorityMap, nil, 1)
  62.     factory.RegisterPriorityFunction2(
  63.         "RequestedToCapacityRatioPriority",
  64.         priorities.RequestedToCapacityRatioResourceAllocationPriorityDefault().PriorityMap,
  65.         nil,
  66.         1)
  67. }

以下对init中的注册进行拆分介绍。

2.1. registerAlgorithmProvider

此部分主要注册默认的预选和优选策略。

  1. // Register functions that extract metadata used by predicates and priorities computations.
  2. factory.RegisterPredicateMetadataProducerFactory(
  3.     func(args factory.PluginFactoryArgs) algorithm.PredicateMetadataProducer {
  4.         return predicates.NewPredicateMetadataFactory(args.PodLister)
  5.     })
  6. factory.RegisterPriorityMetadataProducerFactory(
  7.     func(args factory.PluginFactoryArgs) algorithm.PriorityMetadataProducer {
  8.         return priorities.NewPriorityMetadataFactory(args.ServiceLister, args.ControllerLister, args.ReplicaSetLister, args.StatefulSetLister)
  9.     })
  11. registerAlgorithmProvider(defaultPredicates(), defaultPriorities())

registerAlgorithmProvider

注册AlgorithmProvider，其中包括DefaultProvider和ClusterAutoscalerProvider。

  1. func registerAlgorithmProvider(predSet, priSet sets.String) {
  2.     // Registers algorithm providers. By default we use 'DefaultProvider', but user can specify one to be used
  3.     // by specifying flag.
  4.     factory.RegisterAlgorithmProvider(factory.DefaultProvider, predSet, priSet)
  5.     // Cluster autoscaler friendly scheduling algorithm.
  6.     factory.RegisterAlgorithmProvider(ClusterAutoscalerProvider, predSet,
  7.         copyAndReplace(priSet, "LeastRequestedPriority", "MostRequestedPriority"))
  8. }

2.2. RegisterFitPredicate

在init部分注册预选策略函数。

预选策略如下：

调度策略 函数 描述
PodFitsPorts PodFitsHostPorts PodFitsPorts已经被PodFitsHostPorts代替，此处主要是为了兼容性。
PodFitsHostPortsPred PodFitsHostPorts 判断是否与宿主机的端口冲突。
PodFitsResourcesPred PodFitsResources 判断node资源是否充足。
HostNamePred PodFitsHost 判断pod所指定调度的节点是否是当前的节点。
MatchNodeSelectorPred PodMatchNodeSelector 判断pod指定的node selector是否匹配当前的node。

具体代码如下：

  1. // PodFitsPorts has been replaced by PodFitsHostPorts for better user understanding.
  2. // For backwards compatibility with 1.0, PodFitsPorts is registered as well.
  3. factory.RegisterFitPredicate("PodFitsPorts", predicates.PodFitsHostPorts)
  4. // Fit is defined based on the absence of port conflicts.
  5. // This predicate is actually a default predicate, because it is invoked from
  6. // predicates.GeneralPredicates()
  7. factory.RegisterFitPredicate(predicates.PodFitsHostPortsPred, predicates.PodFitsHostPorts)
  8. // Fit is determined by resource availability.
  9. // This predicate is actually a default predicate, because it is invoked from
  10. // predicates.GeneralPredicates()
  11. factory.RegisterFitPredicate(predicates.PodFitsResourcesPred, predicates.PodFitsResources)
  12. // Fit is determined by the presence of the Host parameter and a string match
  13. // This predicate is actually a default predicate, because it is invoked from
  14. // predicates.GeneralPredicates()
  15. factory.RegisterFitPredicate(predicates.HostNamePred, predicates.PodFitsHost)
  16. // Fit is determined by node selector query.
  17. factory.RegisterFitPredicate(predicates.MatchNodeSelectorPred, predicates.PodMatchNodeSelector)

2.3. RegisterPriorityFunction2

在init部分注册优选策略函数。

  1. // EqualPriority is a prioritizer function that gives an equal weight of one to all nodes
  2. // Register the priority function so that its available
  3. // but do not include it as part of the default priorities
  4. factory.RegisterPriorityFunction2("EqualPriority", core.EqualPriorityMap, nil, 1)
  5. // Optional, cluster-autoscaler friendly priority function - give used nodes higher priority.
  6. factory.RegisterPriorityFunction2("MostRequestedPriority", priorities.MostRequestedPriorityMap, nil, 1)
  7. factory.RegisterPriorityFunction2(
  8.     "RequestedToCapacityRatioPriority",
  9.     priorities.RequestedToCapacityRatioResourceAllocationPriorityDefault().PriorityMap,
  10.     nil,
  11.     1)

3. defaultPredicates

此部分为默认预选策略的注册函数。

默认的预选策略如下：

预选策略 函数 描述
NoVolumeZoneConflictPred NewVolumeZonePredicate 判断pod使用到的volume是否有节点的要求。目前只支持pvc。
MaxEBSVolumeCountPred NewMaxPDVolumeCountPredicate 判断pod使用EBSVolume在该节点上是否已经达到上限了。
MaxGCEPDVolumeCountPred NewMaxPDVolumeCountPredicate 判断pod使用GCEPDVolume在该节点上是否已经达到上限了。
MaxAzureDiskVolumeCountPred NewMaxPDVolumeCountPredicate 判断pod使用AzureDiskVolume在该节点上是否已经达到上限了。
MaxCSIVolumeCountPred NewCSIMaxVolumeLimitPredicate 判断CSIVolume是否达到上限了。
MatchInterPodAffinityPred NewPodAffinityPredicate 匹配pod的亲缘性。
NoDiskConflictPred NoDiskConflict 判断是否有disk volumes的冲突。
GeneralPred GeneralPredicates 通用的预选策略
CheckNodeMemoryPressurePred CheckNodeMemoryPressurePredicate 判断节点内存是否充足。
CheckNodeDiskPressurePred CheckNodeDiskPressurePredicate 判断节点是否有磁盘压力。
CheckNodePIDPressurePred CheckNodePIDPressurePredicate 判断节点上的PID
CheckNodeConditionPred CheckNodeConditionPredicate 判断node是否ready。
PodToleratesNodeTaintsPred PodToleratesNodeTaints 判断pod是否可以容忍节点的taints。
CheckVolumeBindingPred NewVolumeBindingPredicate 判断是否有volume拓扑的要求。

具体代码如下：

  1. func defaultPredicates() sets.String {
  2.     return sets.NewString(
  3.         // Fit is determined by volume zone requirements.
  4.         factory.RegisterFitPredicateFactory(
  5.             predicates.NoVolumeZoneConflictPred,
  6.             func(args factory.PluginFactoryArgs) algorithm.FitPredicate {
  7.                 return predicates.NewVolumeZonePredicate(args.PVInfo, args.PVCInfo, args.StorageClassInfo)
  8.             },
  9.         ),
  10.         // Fit is determined by whether or not there would be too many AWS EBS volumes attached to the node
  11.         factory.RegisterFitPredicateFactory(
  12.             predicates.MaxEBSVolumeCountPred,
  13.             func(args factory.PluginFactoryArgs) algorithm.FitPredicate {
  14.                 return predicates.NewMaxPDVolumeCountPredicate(predicates.EBSVolumeFilterType, args.PVInfo, args.PVCInfo)
  15.             },
  16.         ),
  17.         // Fit is determined by whether or not there would be too many GCE PD volumes attached to the node
  18.         factory.RegisterFitPredicateFactory(
  19.             predicates.MaxGCEPDVolumeCountPred,
  20.             func(args factory.PluginFactoryArgs) algorithm.FitPredicate {
  21.                 return predicates.NewMaxPDVolumeCountPredicate(predicates.GCEPDVolumeFilterType, args.PVInfo, args.PVCInfo)
  22.             },
  23.         ),
  24.         // Fit is determined by whether or not there would be too many Azure Disk volumes attached to the node
  25.         factory.RegisterFitPredicateFactory(
  26.             predicates.MaxAzureDiskVolumeCountPred,
  27.             func(args factory.PluginFactoryArgs) algorithm.FitPredicate {
  28.                 return predicates.NewMaxPDVolumeCountPredicate(predicates.AzureDiskVolumeFilterType, args.PVInfo, args.PVCInfo)
  29.             },
  30.         ),
  31.         factory.RegisterFitPredicateFactory(
  32.             predicates.MaxCSIVolumeCountPred,
  33.             func(args factory.PluginFactoryArgs) algorithm.FitPredicate {
  34.                 return predicates.NewCSIMaxVolumeLimitPredicate(args.PVInfo, args.PVCInfo)
  35.             },
  36.         ),
  37.         // Fit is determined by inter-pod affinity.
  38.         factory.RegisterFitPredicateFactory(
  39.             predicates.MatchInterPodAffinityPred,
  40.             func(args factory.PluginFactoryArgs) algorithm.FitPredicate {
  41.                 return predicates.NewPodAffinityPredicate(args.NodeInfo, args.PodLister)
  42.             },
  43.         ),
  45.         // Fit is determined by non-conflicting disk volumes.
  46.         factory.RegisterFitPredicate(predicates.NoDiskConflictPred, predicates.NoDiskConflict),
  48.         // GeneralPredicates are the predicates that are enforced by all Kubernetes components
  49.         // (e.g. kubelet and all schedulers)
  50.         factory.RegisterFitPredicate(predicates.GeneralPred, predicates.GeneralPredicates),
  52.         // Fit is determined by node memory pressure condition.
  53.         factory.RegisterFitPredicate(predicates.CheckNodeMemoryPressurePred, predicates.CheckNodeMemoryPressurePredicate),
  55.         // Fit is determined by node disk pressure condition.
  56.         factory.RegisterFitPredicate(predicates.CheckNodeDiskPressurePred, predicates.CheckNodeDiskPressurePredicate),
  58.         // Fit is determined by node pid pressure condition.
  59.         factory.RegisterFitPredicate(predicates.CheckNodePIDPressurePred, predicates.CheckNodePIDPressurePredicate),
  61.         // Fit is determined by node conditions: not ready, network unavailable or out of disk.
  62.         factory.RegisterMandatoryFitPredicate(predicates.CheckNodeConditionPred, predicates.CheckNodeConditionPredicate),
  64.         // Fit is determined based on whether a pod can tolerate all of the node's taints
  65.         factory.RegisterFitPredicate(predicates.PodToleratesNodeTaintsPred, predicates.PodToleratesNodeTaints),
  67.         // Fit is determined by volume topology requirements.
  68.         factory.RegisterFitPredicateFactory(
  69.             predicates.CheckVolumeBindingPred,
  70.             func(args factory.PluginFactoryArgs) algorithm.FitPredicate {
  71.                 return predicates.NewVolumeBindingPredicate(args.VolumeBinder)
  72.             },
  73.         ),
  74.     )
  75. }

4. defaultPriorities

此部分主要为默认优选策略的注册函数。

默认优选策略如下：

优选策略 函数 描述
SelectorSpreadPriority NewSelectorSpreadPriority 属于相同service和rs下的pod尽量分布在不同的node上。
InterPodAffinityPriority NewInterPodAffinityPriority 根据pod的亲缘性，将相同拓扑域中的pod放在同一个节点
LeastRequestedPriority LeastRequestedPriorityMap 按最少请求的利用率对节点进行优先级排序。
BalancedResourceAllocation BalancedResourceAllocationMap 实现资源的平衡使用。
NodePreferAvoidPodsPriority CalculateNodePreferAvoidPodsPriorityMap 将此权重设置为足以覆盖所有其他优先级函数。
NodeAffinityPriority CalculateNodeAffinityPriorityMap pod指定label节点调度，来匹配node亲缘性。
TaintTolerationPriority ComputeTaintTolerationPriorityMap pod有设置tolerate属性来容忍node的taint。
ImageLocalityPriority ImageLocalityPriorityMap 根据节点上是否有该pod使用到的镜像打分。

具体代码实现如下：

  1. func defaultPriorities() sets.String {
  2.     return sets.NewString(
  3.         // spreads pods by minimizing the number of pods (belonging to the same service or replication controller) on the same node.
  4.         factory.RegisterPriorityConfigFactory(
  5.             "SelectorSpreadPriority",
  6.             factory.PriorityConfigFactory{
  7.                 MapReduceFunction: func(args factory.PluginFactoryArgs) (algorithm.PriorityMapFunction, algorithm.PriorityReduceFunction) {
  8.                     return priorities.NewSelectorSpreadPriority(args.ServiceLister, args.ControllerLister, args.ReplicaSetLister, args.StatefulSetLister)
  9.                 },
  10.                 Weight: 1,
  11.             },
  12.         ),
  13.         // pods should be placed in the same topological domain (e.g. same node, same rack, same zone, same power domain, etc.)
  14.         // as some other pods, or, conversely, should not be placed in the same topological domain as some other pods.
  15.         factory.RegisterPriorityConfigFactory(
  16.             "InterPodAffinityPriority",
  17.             factory.PriorityConfigFactory{
  18.                 Function: func(args factory.PluginFactoryArgs) algorithm.PriorityFunction {
  19.                     return priorities.NewInterPodAffinityPriority(args.NodeInfo, args.NodeLister, args.PodLister, args.HardPodAffinitySymmetricWeight)
  20.                 },
  21.                 Weight: 1,
  22.             },
  23.         ),
  25.         // Prioritize nodes by least requested utilization.
  26.         factory.RegisterPriorityFunction2("LeastRequestedPriority", priorities.LeastRequestedPriorityMap, nil, 1),
  28.         // Prioritizes nodes to help achieve balanced resource usage
  29.         factory.RegisterPriorityFunction2("BalancedResourceAllocation", priorities.BalancedResourceAllocationMap, nil, 1),
  31.         // Set this weight large enough to override all other priority functions.
  32.         // TODO: Figure out a better way to do this, maybe at same time as fixing #24720.
  33.         factory.RegisterPriorityFunction2("NodePreferAvoidPodsPriority", priorities.CalculateNodePreferAvoidPodsPriorityMap, nil, 10000),
  35.         // Prioritizes nodes that have labels matching NodeAffinity
  36.         factory.RegisterPriorityFunction2("NodeAffinityPriority", priorities.CalculateNodeAffinityPriorityMap, priorities.CalculateNodeAffinityPriorityReduce, 1),
  38.         // Prioritizes nodes that marked with taint which pod can tolerate.
  39.         factory.RegisterPriorityFunction2("TaintTolerationPriority", priorities.ComputeTaintTolerationPriorityMap, priorities.ComputeTaintTolerationPriorityReduce, 1),
  41.         // ImageLocalityPriority prioritizes nodes that have images requested by the pod present.
  42.         factory.RegisterPriorityFunction2("ImageLocalityPriority", priorities.ImageLocalityPriorityMap, nil, 1),
  43.     )
  44. }

参考：