概述

kubelet命令行参数有177个，是kubernetes组件中配置项最多的。

kubelet的选项分为三类，一类是kubletFlags–（启动之后不能随时改变的，只能通过命令行配置，比如ip地址、路径等）；另一类是kubeletConfiguration–（可以在启动之后进行修改，可以通过命令行设置或配置文件或者dynamic Kubelet config–configmap，通过命令行配置是DEPRECATED,一些选项只能通过配置文件设置–比如NodeStatusReportFrequency）；最后是globalFlags–（只能通过命令行配置，比如日志级别、日志路径、版本）。

kubelet是如何启动、初始化并识别这么多选项的？本文就是讲kubelet启动过程中的命令初始化，至于各个组件的运行过程，后续文章再来分析。

本文基于kubernetes 1.18.6版本，请访问源代码阅读仓库。

1 启动过程

kubelet入口文件在cmd\kubelet\kubelet.go，主要流程

1. 初始化随机因子
2. 创建一个cobra Command
3. 初始化日志
4. 执行cobra Command

2 创建cobra Command

这个阶段进行命令行选项的定义、默认值的设置、command的各个字段定义、命令帮助函数和命令使用函数。

主要流程

• 这里新建一个name为kubelet的flagset，为了拥有干净的flagset（不会被CommandLine的flagset污染），而不用cobra的命令行解析。

• 同时设置flagset的NormalizeFunc–将flag名包含下划线"_“转为横杆”-“，比如–iptables-drop-bit和–iptables_drop_bit是一样的。

• 新建一个kubeletFlags里面包含了一些初始选项的默认值：

  具体的field对应的命令行选项见下面kubeletFlags选项，NewKubeletFlags()定义在cmd\kubelet\app\options\options.go

  go

  // NewKubeletFlags will create a new KubeletFlags with default values
  func NewKubeletFlags() *KubeletFlags {
      remoteRuntimeEndpoint := ""
      if runtime.GOOS == "linux" {
          remoteRuntimeEndpoint = "unix:///var/run/dockershim.sock"
      } else if runtime.GOOS == "windows" {
          remoteRuntimeEndpoint = "npipe:////./pipe/dockershim"
      }
  
      return &KubeletFlags{
          EnableServer:                        true,
          ContainerRuntimeOptions:             *NewContainerRuntimeOptions(),
          CertDirectory:                       "/var/lib/kubelet/pki",
          RootDirectory:                       defaultRootDir, // defaultRootDir为/var/lib/kubelet
          MasterServiceNamespace:              metav1.NamespaceDefault, // NamespaceDefault为default
          MaxContainerCount:                   -1,
          MaxPerPodContainerCount:             1,
          MinimumGCAge:                        metav1.Duration{Duration: 0},
          NonMasqueradeCIDR:                   "10.0.0.0/8",
          RegisterSchedulable:                 true,
          ExperimentalKernelMemcgNotification: false,
          RemoteRuntimeEndpoint:               remoteRuntimeEndpoint,
          NodeLabels:                          make(map[string]string),
          VolumePluginDir:                     "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/",
          RegisterNode:                        true,
          SeccompProfileRoot:                  filepath.Join(defaultRootDir, "seccomp"),
          // prior to the introduction of this flag, there was a hardcoded cap of 50 images
          NodeStatusMaxImages:         50,
          EnableCAdvisorJSONEndpoints: false,
      }
  }

  其中NewContainerRuntimeOptions()定义在cmd\kubelet\app\options\container_runtime.go

  go

  const (
      // When these values are updated, also update test/utils/image/manifest.go
      defaultPodSandboxImageName    = "k8s.gcr.io/pause"
      defaultPodSandboxImageVersion = "3.2"
  )
  
  var (
      defaultPodSandboxImage = defaultPodSandboxImageName +
          ":" + defaultPodSandboxImageVersion
  )
  
  // NewContainerRuntimeOptions will create a new ContainerRuntimeOptions with
  // default values.
  func NewContainerRuntimeOptions() *config.ContainerRuntimeOptions {
      dockerEndpoint := ""
      if runtime.GOOS != "windows" {
          dockerEndpoint = "unix:///var/run/docker.sock"
      }
  
      return &config.ContainerRuntimeOptions{
          ContainerRuntime:           kubetypes.DockerContainerRuntime, //DockerContainerRuntime为docker
          RedirectContainerStreaming: false,
          DockerEndpoint:             dockerEndpoint,
          DockershimRootDirectory:    "/var/lib/dockershim",
          PodSandboxImage:            defaultPodSandboxImage,
          ImagePullProgressDeadline:  metav1.Duration{Duration: 1 * time.Minute},
          ExperimentalDockershim:     false,
  
          //Alpha feature
          CNIBinDir:   "/opt/cni/bin",
          CNIConfDir:  "/etc/cni/net.d",
          CNICacheDir: "/var/lib/cni/cache",
      }
  }

• 创建一个kubeletConfiguration，用于设置kubeletConfiguration的flag选项的默认值，它通过创建v1beta1的KubeletConfiguration 然后通过scheme.Default()生成默认的配置，再通过scheme.Convert转换成内部的KubeletConfiguration。

  具体的field对应的命令行选项见下面kubeletConfigurationFlag选项

  go

   KubeletConfiguration{
  		SyncFrequency: metav1.Duration{Duration: 1 * time.Minute},
  		FileCheckFrequency: metav1.Duration{Duration: 20 * time.Second},
  		HTTPCheckFrequency: metav1.Duration{Duration: 20 * time.Second},
  		Address: "0.0.0.0",
  		Port: 10250,
  		Authentication.Anonymous.Enabled: true, // by applyLegacyDefaults, by default false,
  		Authentication.Webhook.Enabled: false, // by applyLegacyDefaults, by default true,
  		Authentication.Webhook.CacheTTL: metav1.Duration{Duration: 2 * time.Minute},
  		Authorization.Mode: "AlwaysAllow", // by applyLegacyDefaults, by default "webhook",
  		Authorization.Webhook.CacheAuthorizedTTL: metav1.Duration{Duration: 5 * time.Minute},
  		Authorization.Webhook.CacheUnauthorizedTTL: metav1.Duration{Duration: 30 * time.Second},
  		ReadOnlyPort: 10255, // by applyLegacyDefaults,
  		RegistryPullQPS: 5,
  		RegistryBurst: 10,
  		EventRecordQPS: 5,
  		EventBurst: 10,
  		EnableDebuggingHandlers: true,
  		HealthzPort: 10248,
  		HealthzBindAddress: "127.0.0.1",
  		OOMScoreAdj: -999,
  		StreamingConnectionIdleTimeout: metav1.Duration{Duration: 4 * time.Hour},
  		NodeStatusReportFrequency: metav1.Duration{Duration: 5 * time.Minute},
  		NodeStatusUpdateFrequency: metav1.Duration{Duration: 10 * time.Second},
  		NodeLeaseDurationSeconds: 40,
  		ImageMinimumGCAge: metav1.Duration{Duration: 2 * time.Minute},
  		ImageGCHighThresholdPercent: 85,
  		ImageGCLowThresholdPercent: 80,
  		VolumeStatsAggPeriod: metav1.Duration{Duration: time.Minute},
  		CgroupsPerQOS: true,
  		CgroupDriver: "cgroupfs",
  		CPUManagerPolicy: "none",
  		CPUManagerReconcilePeriod: metav1.Duration{Duration: 10 * time.Second},
  		TopologyManagerPolicy: "none",
  		RuntimeRequestTimeout: metav1.Duration{Duration: 2 * time.Minute},
  		HairpinMode: "promiscuous-bridge",
  		MaxPods: 110,
  		PodPidsLimit: -1,
  		ResolverConfig: "/etc/resolv.conf",
  		CPUCFSQuota: true,
  		CPUCFSQuotaPeriod: metav1.Duration{Duration: 100 * time.Millisecond},
  		MaxOpenFiles: 1000000,
  		ContentType: "application/vnd.kubernetes.protobuf",
  		KubeAPIQPS: 5,
  		KubeAPIBurst: 10,
  		SerializeImagePulls: true,
  		EvictionHard: map[string]string{
  			"memory.available":  "100Mi",
  			"nodefs.available":  "10%",
  			"nodefs.inodesFree": "5%",
  			"imagefs.available": "15%",
  		},
  		EvictionPressureTransitionPeriod: metav1.Duration{Duration: 5 * time.Minute},
  		EnableControllerAttachDetach: true,
  		MakeIPTablesUtilChains: true,
  		IPTablesMasqueradeBit: 14,
  		IPTablesDropBit: 15,
  		FailSwapOn: true,
  		ContainerLogMaxSize: "10Mi",
  		ContainerLogMaxFiles: 5,
  		ConfigMapAndSecretChangeDetectionStrategy: "Watch",
  		EnforceNodeAllocatable: []string{"pods"},
  	}

• 创建cobra.Command，定义了Use、Long、DisableFlagParsing、Run这四个字段。

• kubeletFlags选项添加到flagset

• kubeletConfigurationFlag选项添加到flagset

• globalFlag添加到flagset，globalFlag包含klog、verflag、Cadvisor、logflush相关选项

• 设置help选项

• 自定义command的helpfunc和usageFunc

看到这里会发现kubelet并没有使用cobra的命令解析和定义、helpfunc和usageFunc，这是为什么呢？

是因为globalFlag里面定义的选项flag，都是定义在pflag.Command下面–通过import模块自动注册。而cobra默认的helpfunc和usageFunc和excute会执行mergePersistentFlags()–会引入pflag.Command的flag，由于历史原因有一些不需要的选项也注册在pflag.Command， 它们也会被引用进来，所以才会新建一个flagset并自己解析命令，保持干净。

代码注释中也提到了，只是不是很详细

	// keep cleanFlagSet separate, so Cobra doesn't pollute it with the global flags
	kubeletFlags.AddFlags(cleanFlagSet)
	options.AddKubeletConfigFlags(cleanFlagSet, kubeletConfig)
	options.AddGlobalFlags(cleanFlagSet)
	cleanFlagSet.BoolP("help", "h", false, fmt.Sprintf("help for %s", cmd.Name()))

	// ugly, but necessary, because Cobra's default UsageFunc and HelpFunc pollute the flagset with global flags
	const usageFmt = "Usage:\n  %s\n\nFlags:\n%s"
	cmd.SetUsageFunc(func(cmd *cobra.Command) error {
		fmt.Fprintf(cmd.OutOrStderr(), usageFmt, cmd.UseLine(), cleanFlagSet.FlagUsagesWrapped(2))
		return nil
	})
	cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) {
		fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine(), cleanFlagSet.FlagUsagesWrapped(2))
	})


// AddGlobalFlags explicitly registers flags that libraries (glog, verflag, etc.) register
// against the global flagsets from "flag" and "github.com/spf13/pflag".
// We do this in order to prevent unwanted flags from leaking into the Kubelet's flagset.
func AddGlobalFlags(fs *pflag.FlagSet) {

2.1 kubeletFlags选项

kubeletFlags 包含kubelet运行中不能改变的选项、或者不能安全改变的选项、不能再不同的实例共享的。

KubeletConfiguration定义了不同实例能够共享的选项。

// KubeletFlags contains configuration flags for the Kubelet.

// A configuration field should go in KubeletFlags instead of KubeletConfiguration if any of these are true:

// - its value will never, or cannot safely be changed during the lifetime of a node, or

// - its value cannot be safely shared between nodes at the same time (e.g. a hostname);

// KubeletConfiguration is intended to be shared between nodes.

2.2 kubeletConfigurationFlag选项

kubeletConfigurationFlag分为两类， 一类deprecated command line是可以在命令行（即将被废弃）和配置文件和dynamic kubelet config中配置的，另一类configuration是不能配置在命令行的，只能在配置文件中和dynamic kubelet config中。

2.3 GlobalFlags

globalFlags包含了klog选项、cadvisor选项、CredentialProvider选项、logflush选项。

klog选项定义通过klog.InitFlags

// InitFlags is for explicitly initializing the flags.
func InitFlags(flagset *flag.FlagSet) {
	if flagset == nil {
		flagset = flag.CommandLine
	}

	flagset.StringVar(&logging.logDir, "log_dir", logging.logDir, "If non-empty, write log files in this directory")
	flagset.StringVar(&logging.logFile, "log_file", logging.logFile, "If non-empty, use this log file")
	flagset.Uint64Var(&logging.logFileMaxSizeMB, "log_file_max_size", logging.logFileMaxSizeMB,
		"Defines the maximum size a log file can grow to. Unit is megabytes. "+
			"If the value is 0, the maximum file size is unlimited.")
	flagset.BoolVar(&logging.toStderr, "logtostderr", logging.toStderr, "log to standard error instead of files")
	flagset.BoolVar(&logging.alsoToStderr, "alsologtostderr", logging.alsoToStderr, "log to standard error as well as files")
	flagset.Var(&logging.verbosity, "v", "number for the log level verbosity")
	flagset.BoolVar(&logging.skipHeaders, "add_dir_header", logging.addDirHeader, "If true, adds the file directory to the header") //这里应该是&logging.addDirHeader,在下一个klog版本v2.0.0修复了,所有的1.18版本都会有这个问题
	flagset.BoolVar(&logging.skipHeaders, "skip_headers", logging.skipHeaders, "If true, avoid header prefixes in the log messages")
	flagset.BoolVar(&logging.skipLogHeaders, "skip_log_headers", logging.skipLogHeaders, "If true, avoid headers when opening log files")
	flagset.Var(&logging.stderrThreshold, "stderrthreshold", "logs at or above this threshold go to stderr")
	flagset.Var(&logging.vmodule, "vmodule", "comma-separated list of pattern=N settings for file-filtered logging")
	flagset.Var(&logging.traceLocation, "log_backtrace_at", "when logging hits line file:N, emit a stack trace")
}

cadvisor定义过程

cadvisor flag通过匿名import来导入的，代码保存在vendor/github.com/google/cadvisor下

// ensure libs have a chance to globally register their flags
	_ "github.com/google/cadvisor/container/common"
	_ "github.com/google/cadvisor/container/containerd"
	_ "github.com/google/cadvisor/container/docker"
	_ "github.com/google/cadvisor/container/raw"
	_ "github.com/google/cadvisor/machine"
	_ "github.com/google/cadvisor/manager"
	_ "github.com/google/cadvisor/storage"

以github.com/google/cadvisor/container/docker为例，它定义了下面这些选项

var ArgDockerEndpoint = flag.String("docker", "unix:///var/run/docker.sock", "docker endpoint")
var ArgDockerTLS = flag.Bool("docker-tls", false, "use TLS to connect to docker")
var ArgDockerCert = flag.String("docker-tls-cert", "cert.pem", "path to client certificate")
var ArgDockerKey = flag.String("docker-tls-key", "key.pem", "path to private key")
var ArgDockerCA = flag.String("docker-tls-ca", "ca.pem", "path to trusted CA")

// The namespace under which Docker aliases are unique.
const DockerNamespace = "docker"

// Regexp that identifies docker cgroups, containers started with
// --cgroup-parent have another prefix than 'docker'
var dockerCgroupRegexp = regexp.MustCompile(`([a-z0-9]{64})`)

var dockerEnvWhitelist = flag.String("docker_env_metadata_whitelist", "", "a comma-separated list of environment variable keys that needs to be collected for docker containers")

var (
	// Basepath to all container specific information that libcontainer stores.
	dockerRootDir string

	dockerRootDirFlag = flag.String("docker_root", "/var/lib/docker", "DEPRECATED: docker root is read from docker info (this is a fallback, default: /var/lib/docker)")

	dockerRootDirOnce sync.Once

	// flag that controls globally disabling thin_ls pending future enhancements.
	// in production, it has been found that thin_ls makes excessive use of iops.
	// in an iops restricted environment, usage of thin_ls must be controlled via blkio.
	// pending that enhancement, disable its usage.
	disableThinLs = true
)

cadvisor选项中除了--docker-root和housekeeping-interval不是deprecated，其他选项都是deprecated。其中update_machine_info_interval这个flag未注册到kubelet中，根据commit记录，update_machine_info_interval是在升级cadvisor版本到0.33.0时候新添加进来的，可能因为这些都是deprecated，所以没有注册到kubelet中。

// These flags were also implicit from cadvisor, but are actually used by something in the core repo:
	// TODO(mtaufen): This one is stil used by our salt, but for heaven's sake it's even deprecated in cadvisor
	register(global, local, "docker_root")
	// e2e node tests rely on this
	register(global, local, "housekeeping_interval")

	// These flags were implicit from cadvisor, and are mistakes that should be registered deprecated:
	const deprecated = "This is a cadvisor flag that was mistakenly registered with the Kubelet. Due to legacy concerns, it will follow the standard CLI deprecation timeline before being removed."

	registerDeprecated(global, local, "application_metrics_count_limit", deprecated)
	registerDeprecated(global, local, "boot_id_file", deprecated)
	registerDeprecated(global, local, "container_hints", deprecated)
	registerDeprecated(global, local, "containerd", deprecated)
	registerDeprecated(global, local, "docker", deprecated)
	registerDeprecated(global, local, "docker_env_metadata_whitelist", deprecated)
	registerDeprecated(global, local, "docker_only", deprecated)
	registerDeprecated(global, local, "docker-tls", deprecated)
	registerDeprecated(global, local, "docker-tls-ca", deprecated)
	registerDeprecated(global, local, "docker-tls-cert", deprecated)
	registerDeprecated(global, local, "docker-tls-key", deprecated)
	registerDeprecated(global, local, "enable_load_reader", deprecated)
	registerDeprecated(global, local, "event_storage_age_limit", deprecated)
	registerDeprecated(global, local, "event_storage_event_limit", deprecated)
	registerDeprecated(global, local, "global_housekeeping_interval", deprecated)
	registerDeprecated(global, local, "log_cadvisor_usage", deprecated)
	registerDeprecated(global, local, "machine_id_file", deprecated)
	registerDeprecated(global, local, "storage_driver_user", deprecated)
	registerDeprecated(global, local, "storage_driver_password", deprecated)
	registerDeprecated(global, local, "storage_driver_host", deprecated)
	registerDeprecated(global, local, "storage_driver_db", deprecated)
	registerDeprecated(global, local, "storage_driver_table", deprecated)
	registerDeprecated(global, local, "storage_driver_secure", deprecated)
	registerDeprecated(global, local, "storage_driver_buffer_duration", deprecated)

CredentialProviderFlags

在cmd\kubelet\app\options\globalflags.go里匿名import

	// ensure libs have a chance to globally register their flags
	_ "k8s.io/kubernetes/pkg/credentialprovider/azure"

在pkg\credentialprovider\azure\azure_credentials.go定义

var flagConfigFile = pflag.String("azure-container-registry-config", "",
	"Path to the file containing Azure container registry configuration information.")

version

--version选项定义在staging\src\k8s.io\component-base\version\verflag\verflag.go，它接受三个值false (默认值)、true(不加参数默认为true)、raw(显示json格式的详细版本信息)。

后面文章专门介绍kubernetes的版本相关的发布规则、version的相关hack技术

func VersionVar(p *versionValue, name string, value versionValue, usage string) {
	*p = value
	flag.Var(p, name, usage)
	// "--version" will be treated as "--version=true"
	flag.Lookup(name).NoOptDefVal = "true"
}

func Version(name string, value versionValue, usage string) *versionValue {
	p := new(versionValue)
	VersionVar(p, name, value, usage)
	return p
}

const versionFlagName = "version"

var (
	versionFlag = Version(versionFlagName, VersionFalse, "Print version information and quit")
)

log选项

定义在staging\src\k8s.io\component-base\logs\logs.go

const logFlushFreqFlagName = "log-flush-frequency"

var logFlushFreq = pflag.Duration(logFlushFreqFlagName, 5*time.Second, "Maximum number of seconds between log flushes")

2.4 help选项

特殊的选项，就是显示命令的帮助信息，使用-h或–help。

cleanFlagSet.BoolP("help", "h", false, fmt.Sprintf("help for %s", cmd.Name()))