概述

kubelet命令行参数有177个，是kubernetes组件中配置项最多的。

kubelet的选项分为三类，一类是kubletFlags–（启动之后不能随时改变的，只能通过命令行配置，比如ip地址、路径等）；另一类是kubeletConfiguration–（可以在启动之后进行修改，可以通过命令行设置或配置文件或者dynamic Kubelet config–configmap，通过命令行配置是DEPRECATED,一些选项只能通过配置文件设置–比如NodeStatusReportFrequency）；最后是globalFlags–（只能通过命令行配置，比如日志级别、日志路径、版本）。

kubelet是如何启动、初始化并识别这么多选项的？本文就是讲kubelet启动过程中的命令初始化，至于各个组件的运行过程，后续文章再来分析。

本文基于kubernetes 1.18.6版本，请访问源代码阅读仓库。

1 启动过程

kubelet入口文件在cmd\kubelet\kubelet.go，主要流程

1. 初始化随机因子
2. 创建一个cobra Command
3. 初始化日志
4. 执行cobra Command

2 创建cobra Command

这个阶段进行命令行选项的定义、默认值的设置、command的各个字段定义、命令帮助函数和命令使用函数。

主要流程

• 这里新建一个name为kubelet的flagset，为了拥有干净的flagset（不会被CommandLine的flagset污染），而不用cobra的命令行解析。

• 同时设置flagset的NormalizeFunc–将flag名包含下划线"_“转为横杆”-“，比如–iptables-drop-bit和–iptables_drop_bit是一样的。

• 新建一个kubeletFlags里面包含了一些初始选项的默认值：

  具体的field对应的命令行选项见下面kubeletFlags选项，NewKubeletFlags()定义在cmd\kubelet\app\options\options.go

  1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31

  // NewKubeletFlags will create a new KubeletFlags with default values func NewKubeletFlags() *KubeletFlags { remoteRuntimeEndpoint := "" if runtime.GOOS == "linux" { remoteRuntimeEndpoint = "unix:///var/run/dockershim.sock" } else if runtime.GOOS == "windows" { remoteRuntimeEndpoint = "npipe:////./pipe/dockershim" } return &KubeletFlags{ EnableServer: true, ContainerRuntimeOptions: *NewContainerRuntimeOptions(), CertDirectory: "/var/lib/kubelet/pki", RootDirectory: defaultRootDir, // defaultRootDir为/var/lib/kubelet MasterServiceNamespace: metav1.NamespaceDefault, // NamespaceDefault为default MaxContainerCount: -1, MaxPerPodContainerCount: 1, MinimumGCAge: metav1.Duration{Duration: 0}, NonMasqueradeCIDR: "10.0.0.0/8", RegisterSchedulable: true, ExperimentalKernelMemcgNotification: false, RemoteRuntimeEndpoint: remoteRuntimeEndpoint, NodeLabels: make(map[string]string), VolumePluginDir: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/", RegisterNode: true, SeccompProfileRoot: filepath.Join(defaultRootDir, "seccomp"), // prior to the introduction of this flag, there was a hardcoded cap of 50 images NodeStatusMaxImages: 50, EnableCAdvisorJSONEndpoints: false, } }

  其中NewContainerRuntimeOptions()定义在cmd\kubelet\app\options\container_runtime.go

  1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34

  const ( // When these values are updated, also update test/utils/image/manifest.go defaultPodSandboxImageName = "k8s.gcr.io/pause" defaultPodSandboxImageVersion = "3.2" ) var ( defaultPodSandboxImage = defaultPodSandboxImageName + ":" + defaultPodSandboxImageVersion ) // NewContainerRuntimeOptions will create a new ContainerRuntimeOptions with // default values. func NewContainerRuntimeOptions() *config.ContainerRuntimeOptions { dockerEndpoint := "" if runtime.GOOS != "windows" { dockerEndpoint = "unix:///var/run/docker.sock" } return &config.ContainerRuntimeOptions{ ContainerRuntime: kubetypes.DockerContainerRuntime, //DockerContainerRuntime为docker RedirectContainerStreaming: false, DockerEndpoint: dockerEndpoint, DockershimRootDirectory: "/var/lib/dockershim", PodSandboxImage: defaultPodSandboxImage, ImagePullProgressDeadline: metav1.Duration{Duration: 1 * time.Minute}, ExperimentalDockershim: false, //Alpha feature CNIBinDir: "/opt/cni/bin", CNIConfDir: "/etc/cni/net.d", CNICacheDir: "/var/lib/cni/cache", } }

• 创建一个kubeletConfiguration，用于设置kubeletConfiguration的flag选项的默认值，它通过创建v1beta1的KubeletConfiguration 然后通过scheme.Default()生成默认的配置，再通过scheme.Convert转换成内部的KubeletConfiguration。

  具体的field对应的命令行选项见下面kubeletConfigurationFlag选项

  1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63

  KubeletConfiguration{ SyncFrequency: metav1.Duration{Duration: 1 * time.Minute}, FileCheckFrequency: metav1.Duration{Duration: 20 * time.Second}, HTTPCheckFrequency: metav1.Duration{Duration: 20 * time.Second}, Address: "0.0.0.0", Port: 10250, Authentication.Anonymous.Enabled: true, // by applyLegacyDefaults, by default false, Authentication.Webhook.Enabled: false, // by applyLegacyDefaults, by default true, Authentication.Webhook.CacheTTL: metav1.Duration{Duration: 2 * time.Minute}, Authorization.Mode: "AlwaysAllow", // by applyLegacyDefaults, by default "webhook", Authorization.Webhook.CacheAuthorizedTTL: metav1.Duration{Duration: 5 * time.Minute}, Authorization.Webhook.CacheUnauthorizedTTL: metav1.Duration{Duration: 30 * time.Second}, ReadOnlyPort: 10255, // by applyLegacyDefaults, RegistryPullQPS: 5, RegistryBurst: 10, EventRecordQPS: 5, EventBurst: 10, EnableDebuggingHandlers: true, HealthzPort: 10248, HealthzBindAddress: "127.0.0.1", OOMScoreAdj: -999, StreamingConnectionIdleTimeout: metav1.Duration{Duration: 4 * time.Hour}, NodeStatusReportFrequency: metav1.Duration{Duration: 5 * time.Minute}, NodeStatusUpdateFrequency: metav1.Duration{Duration: 10 * time.Second}, NodeLeaseDurationSeconds: 40, ImageMinimumGCAge: metav1.Duration{Duration: 2 * time.Minute}, ImageGCHighThresholdPercent: 85, ImageGCLowThresholdPercent: 80, VolumeStatsAggPeriod: metav1.Duration{Duration: time.Minute}, CgroupsPerQOS: true, CgroupDriver: "cgroupfs", CPUManagerPolicy: "none", CPUManagerReconcilePeriod: metav1.Duration{Duration: 10 * time.Second}, TopologyManagerPolicy: "none", RuntimeRequestTimeout: metav1.Duration{Duration: 2 * time.Minute}, HairpinMode: "promiscuous-bridge", MaxPods: 110, PodPidsLimit: -1, ResolverConfig: "/etc/resolv.conf", CPUCFSQuota: true, CPUCFSQuotaPeriod: metav1.Duration{Duration: 100 * time.Millisecond}, MaxOpenFiles: 1000000, ContentType: "application/vnd.kubernetes.protobuf", KubeAPIQPS: 5, KubeAPIBurst: 10, SerializeImagePulls: true, EvictionHard: map[string]string{ "memory.available": "100Mi", "nodefs.available": "10%", "nodefs.inodesFree": "5%", "imagefs.available": "15%", }, EvictionPressureTransitionPeriod: metav1.Duration{Duration: 5 * time.Minute}, EnableControllerAttachDetach: true, MakeIPTablesUtilChains: true, IPTablesMasqueradeBit: 14, IPTablesDropBit: 15, FailSwapOn: true, ContainerLogMaxSize: "10Mi", ContainerLogMaxFiles: 5, ConfigMapAndSecretChangeDetectionStrategy: "Watch", EnforceNodeAllocatable: []string{"pods"}, }

• 创建cobra.Command，定义了Use、Long、DisableFlagParsing、Run这四个字段。

• kubeletFlags选项添加到flagset

• kubeletConfigurationFlag选项添加到flagset

• globalFlag添加到flagset，globalFlag包含klog、verflag、Cadvisor、logflush相关选项

• 设置help选项

• 自定义command的helpfunc和usageFunc

看到这里会发现kubelet并没有使用cobra的命令解析和定义、helpfunc和usageFunc，这是为什么呢？

是因为globalFlag里面定义的选项flag，都是定义在pflag.Command下面–通过import模块自动注册。而cobra默认的helpfunc和usageFunc和excute会执行mergePersistentFlags()–会引入pflag.Command的flag，由于历史原因有一些不需要的选项也注册在pflag.Command， 它们也会被引用进来，所以才会新建一个flagset并自己解析命令，保持干净。

代码注释中也提到了，只是不是很详细

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21

	// keep cleanFlagSet separate, so Cobra doesn't pollute it with the global flags
	kubeletFlags.AddFlags(cleanFlagSet)
	options.AddKubeletConfigFlags(cleanFlagSet, kubeletConfig)
	options.AddGlobalFlags(cleanFlagSet)
	cleanFlagSet.BoolP("help", "h", false, fmt.Sprintf("help for %s", cmd.Name()))

	// ugly, but necessary, because Cobra's default UsageFunc and HelpFunc pollute the flagset with global flags
	const usageFmt = "Usage:\n  %s\n\nFlags:\n%s"
	cmd.SetUsageFunc(func(cmd *cobra.Command) error {
		fmt.Fprintf(cmd.OutOrStderr(), usageFmt, cmd.UseLine(), cleanFlagSet.FlagUsagesWrapped(2))
		return nil
	})
	cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) {
		fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine(), cleanFlagSet.FlagUsagesWrapped(2))
	})


// AddGlobalFlags explicitly registers flags that libraries (glog, verflag, etc.) register
// against the global flagsets from "flag" and "github.com/spf13/pflag".
// We do this in order to prevent unwanted flags from leaking into the Kubelet's flagset.
func AddGlobalFlags(fs *pflag.FlagSet) {

2.1 kubeletFlags选项

kubeletFlags 包含kubelet运行中不能改变的选项、或者不能安全改变的选项、不能再不同的实例共享的。

KubeletConfiguration定义了不同实例能够共享的选项。

// KubeletFlags contains configuration flags for the Kubelet.

// A configuration field should go in KubeletFlags instead of KubeletConfiguration if any of these are true:

// - its value will never, or cannot safely be changed during the lifetime of a node, or

// - its value cannot be safely shared between nodes at the same time (e.g. a hostname);

// KubeletConfiguration is intended to be shared between nodes.

2.2 kubeletConfigurationFlag选项

kubeletConfigurationFlag分为两类， 一类deprecated command line是可以在命令行（即将被废弃）和配置文件和dynamic kubelet config中配置的，另一类configuration是不能配置在命令行的，只能在配置文件中和dynamic kubelet config中。

2.3 GlobalFlags

globalFlags包含了klog选项、cadvisor选项、CredentialProvider选项、logflush选项。

klog选项定义通过klog.InitFlags

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21

// InitFlags is for explicitly initializing the flags.
func InitFlags(flagset *flag.FlagSet) {
	if flagset == nil {
		flagset = flag.CommandLine
	}

	flagset.StringVar(&logging.logDir, "log_dir", logging.logDir, "If non-empty, write log files in this directory")
	flagset.StringVar(&logging.logFile, "log_file", logging.logFile, "If non-empty, use this log file")
	flagset.Uint64Var(&logging.logFileMaxSizeMB, "log_file_max_size", logging.logFileMaxSizeMB,
		"Defines the maximum size a log file can grow to. Unit is megabytes. "+
			"If the value is 0, the maximum file size is unlimited.")
	flagset.BoolVar(&logging.toStderr, "logtostderr", logging.toStderr, "log to standard error instead of files")
	flagset.BoolVar(&logging.alsoToStderr, "alsologtostderr", logging.alsoToStderr, "log to standard error as well as files")
	flagset.Var(&logging.verbosity, "v", "number for the log level verbosity")
	flagset.BoolVar(&logging.skipHeaders, "add_dir_header", logging.addDirHeader, "If true, adds the file directory to the header") //这里应该是&logging.addDirHeader,在下一个klog版本v2.0.0修复了,所有的1.18版本都会有这个问题
	flagset.BoolVar(&logging.skipHeaders, "skip_headers", logging.skipHeaders, "If true, avoid header prefixes in the log messages")
	flagset.BoolVar(&logging.skipLogHeaders, "skip_log_headers", logging.skipLogHeaders, "If true, avoid headers when opening log files")
	flagset.Var(&logging.stderrThreshold, "stderrthreshold", "logs at or above this threshold go to stderr")
	flagset.Var(&logging.vmodule, "vmodule", "comma-separated list of pattern=N settings for file-filtered logging")
	flagset.Var(&logging.traceLocation, "log_backtrace_at", "when logging hits line file:N, emit a stack trace")
}

cadvisor定义过程

cadvisor flag通过匿名import来导入的，代码保存在vendor/github.com/google/cadvisor下

1
2
3
4
5
6
7
8

// ensure libs have a chance to globally register their flags
	_ "github.com/google/cadvisor/container/common"
	_ "github.com/google/cadvisor/container/containerd"
	_ "github.com/google/cadvisor/container/docker"
	_ "github.com/google/cadvisor/container/raw"
	_ "github.com/google/cadvisor/machine"
	_ "github.com/google/cadvisor/manager"
	_ "github.com/google/cadvisor/storage"

以github.com/google/cadvisor/container/docker为例，它定义了下面这些选项

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

var ArgDockerEndpoint = flag.String("docker", "unix:///var/run/docker.sock", "docker endpoint")
var ArgDockerTLS = flag.Bool("docker-tls", false, "use TLS to connect to docker")
var ArgDockerCert = flag.String("docker-tls-cert", "cert.pem", "path to client certificate")
var ArgDockerKey = flag.String("docker-tls-key", "key.pem", "path to private key")
var ArgDockerCA = flag.String("docker-tls-ca", "ca.pem", "path to trusted CA")

// The namespace under which Docker aliases are unique.
const DockerNamespace = "docker"

// Regexp that identifies docker cgroups, containers started with
// --cgroup-parent have another prefix than 'docker'
var dockerCgroupRegexp = regexp.MustCompile(`([a-z0-9]{64})`)

var dockerEnvWhitelist = flag.String("docker_env_metadata_whitelist", "", "a comma-separated list of environment variable keys that needs to be collected for docker containers")

var (
	// Basepath to all container specific information that libcontainer stores.
	dockerRootDir string

	dockerRootDirFlag = flag.String("docker_root", "/var/lib/docker", "DEPRECATED: docker root is read from docker info (this is a fallback, default: /var/lib/docker)")

	dockerRootDirOnce sync.Once

	// flag that controls globally disabling thin_ls pending future enhancements.
	// in production, it has been found that thin_ls makes excessive use of iops.
	// in an iops restricted environment, usage of thin_ls must be controlled via blkio.
	// pending that enhancement, disable its usage.
	disableThinLs = true
)

cadvisor选项中除了--docker-root和housekeeping-interval不是deprecated，其他选项都是deprecated。其中update_machine_info_interval这个flag未注册到kubelet中，根据commit记录，update_machine_info_interval是在升级cadvisor版本到0.33.0时候新添加进来的，可能因为这些都是deprecated，所以没有注册到kubelet中。

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

// These flags were also implicit from cadvisor, but are actually used by something in the core repo:
	// TODO(mtaufen): This one is stil used by our salt, but for heaven's sake it's even deprecated in cadvisor
	register(global, local, "docker_root")
	// e2e node tests rely on this
	register(global, local, "housekeeping_interval")

	// These flags were implicit from cadvisor, and are mistakes that should be registered deprecated:
	const deprecated = "This is a cadvisor flag that was mistakenly registered with the Kubelet. Due to legacy concerns, it will follow the standard CLI deprecation timeline before being removed."

	registerDeprecated(global, local, "application_metrics_count_limit", deprecated)
	registerDeprecated(global, local, "boot_id_file", deprecated)
	registerDeprecated(global, local, "container_hints", deprecated)
	registerDeprecated(global, local, "containerd", deprecated)
	registerDeprecated(global, local, "docker", deprecated)
	registerDeprecated(global, local, "docker_env_metadata_whitelist", deprecated)
	registerDeprecated(global, local, "docker_only", deprecated)
	registerDeprecated(global, local, "docker-tls", deprecated)
	registerDeprecated(global, local, "docker-tls-ca", deprecated)
	registerDeprecated(global, local, "docker-tls-cert", deprecated)
	registerDeprecated(global, local, "docker-tls-key", deprecated)
	registerDeprecated(global, local, "enable_load_reader", deprecated)
	registerDeprecated(global, local, "event_storage_age_limit", deprecated)
	registerDeprecated(global, local, "event_storage_event_limit", deprecated)
	registerDeprecated(global, local, "global_housekeeping_interval", deprecated)
	registerDeprecated(global, local, "log_cadvisor_usage", deprecated)
	registerDeprecated(global, local, "machine_id_file", deprecated)
	registerDeprecated(global, local, "storage_driver_user", deprecated)
	registerDeprecated(global, local, "storage_driver_password", deprecated)
	registerDeprecated(global, local, "storage_driver_host", deprecated)
	registerDeprecated(global, local, "storage_driver_db", deprecated)
	registerDeprecated(global, local, "storage_driver_table", deprecated)
	registerDeprecated(global, local, "storage_driver_secure", deprecated)
	registerDeprecated(global, local, "storage_driver_buffer_duration", deprecated)

CredentialProviderFlags

在cmd\kubelet\app\options\globalflags.go里匿名import

1
2

	// ensure libs have a chance to globally register their flags
	_ "k8s.io/kubernetes/pkg/credentialprovider/azure"

在pkg\credentialprovider\azure\azure_credentials.go定义

1
2

var flagConfigFile = pflag.String("azure-container-registry-config", "",
	"Path to the file containing Azure container registry configuration information.")

version

--version选项定义在staging\src\k8s.io\component-base\version\verflag\verflag.go，它接受三个值false (默认值)、true(不加参数默认为true)、raw(显示json格式的详细版本信息)。

后面文章专门介绍kubernetes的版本相关的发布规则、version的相关hack技术

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

func VersionVar(p *versionValue, name string, value versionValue, usage string) {
	*p = value
	flag.Var(p, name, usage)
	// "--version" will be treated as "--version=true"
	flag.Lookup(name).NoOptDefVal = "true"
}

func Version(name string, value versionValue, usage string) *versionValue {
	p := new(versionValue)
	VersionVar(p, name, value, usage)
	return p
}

const versionFlagName = "version"

var (
	versionFlag = Version(versionFlagName, VersionFalse, "Print version information and quit")
)

log选项

定义在staging\src\k8s.io\component-base\logs\logs.go

1
2
3

const logFlushFreqFlagName = "log-flush-frequency"

var logFlushFreq = pflag.Duration(logFlushFreqFlagName, 5*time.Second, "Maximum number of seconds between log flushes")

2.4 help选项

特殊的选项，就是显示命令的帮助信息，使用-h或–help。

1

cleanFlagSet.BoolP("help", "h", false, fmt.Sprintf("help for %s", cmd.Name()))