一 前言
-
CNI和kubelet关系
kubelet负责创建POD，POD启动和删除（可以类比为虚拟机）需要设置网络环境。kubelet自身不关心网络实现，它调用CNI实现。
CNI实现能否同时存在多个？
答案：可以有多个，但是kubelet只会使用第一个有效的插件。如果不同的POD网络插件不一致，它们将无法通信。
二 kubelet启动
image
2.1 kubelet启动命令
kubelet \
--network-plugin=cni
--cni-conf-dir=/etc/cni/net.d
--cni-bin-dir=/opt/cni/bin
...
通过--network-plugin指定要使用的网络插件类型。
2.2 Runtime设置
cmd/kubelet/app/server.go:createAndInitKubelet
默认k8s的运行时是Docker。
switch containerRuntime {
case kubetypes.DockerContainerRuntime:
streamingConfig := getStreamingConfig(kubeCfg, kubeDeps, crOptions)
ds, err := dockershim.NewDockerService(kubeDeps.DockerClientConfig, crOptions.PodSandboxImage, streamingConfig,
&pluginSettings, runtimeCgroups, kubeCfg.CgroupDriver, crOptions.DockershimRootDirectory, !crOptions.RedirectContainerStreaming)
// 省略...
case kubetypes.RemoteContainerRuntime:
// No-op.
break
default:
return nil, fmt.Errorf("unsupported CRI runtime: %q", containerRuntime)
}
2.3 网络插件加载
pkg/kubelet/dockershim/docker_service.go:NewDockerService
网络插件有两种实现cni、kubenet，一般都使用cni。
func NewDockerService(config *ClientConfig, podSandboxImage string, streamingConfig *streaming.Config, pluginSettings *NetworkPluginSettings,
cgroupsName string, kubeCgroupDriver string, dockershimRootDir string, startLocalStreamingServer bool) (DockerService, error) {
client := NewDockerClientFromConfig(config)
// cni插件
pluginSettings.PluginBinDirs = cni.SplitDirs(pluginSettings.PluginBinDirString)
cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, pluginSettings.PluginCacheDir, pluginSettings.PluginBinDirs)
// kubenet插件
cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDirs, pluginSettings.PluginCacheDir))
netHost := &dockerNetworkHost{
&namespaceGetter{ds},
&portMappingGetter{ds},
}
// 插件初始化
plug, err := network.InitNetworkPlugin(cniPlugins, pluginSettings.PluginName, netHost, pluginSettings.HairpinMode, pluginSettings.NonMasqueradeCIDR, pluginSettings.MTU)
// 省略
return ds, nil
}
三 CNI插件
3.1 插件初始化
func ProbeNetworkPlugins(confDir, cacheDir string, binDirs []string) []network.NetworkPlugin {
//设置可执行程序路径、配置文件路径等
plugin := &cniNetworkPlugin{
defaultNetwork: nil,
loNetwork: getLoNetwork(binDirs),
execer: utilexec.New(),
confDir: confDir,
binDirs: binDirs,
cacheDir: cacheDir,
}
// sync NetworkConfig in best effort during probing.
plugin.syncNetworkConfig()
return []network.NetworkPlugin{plugin}
}
3.2 插件实现加载
按照配置文件名称加载实现，找到第一个有效的实现作为CNI实现。
func getDefaultCNINetwork(confDir string, binDirs []string) (*cniNetwork, error) {
// 从/etc/cni/net.d 查找后缀为conf,conflist,json的配置文件
files, err := libcni.ConfFiles(confDir, []string{".conf", ".conflist", ".json"})
cniConfig := &libcni.CNIConfig{Path: binDirs}
sort.Strings(files)
for _, confFile := range files {
var confList *libcni.NetworkConfigList
if strings.HasSuffix(confFile, ".conflist") {
confList, err = libcni.ConfListFromFile(confFile)
} else {
conf, err := libcni.ConfFromFile(confFile)
confList, err = libcni.ConfListFromConf(conf)
}
// 校验可执行程序是否存在
caps, err := cniConfig.ValidateNetworkList(context.TODO(), confList)
return &cniNetwork{
name: confList.Name,
NetworkConfig: confList,
CNIConfig: cniConfig,
Capabilities: caps,
}, nil
}
return nil, fmt.Errorf("no valid networks found in %s", confDir)
}
3.3 配置
读取配置文件，它的内容如下图所示.
{
"cniVersion": "0.3.1",
"name": "my-cni",
"type": "my-cni",
"podcidr": "10.244.2.0/24"
}
四 网络插件初始化
下面以cni的网络插件分析，cni从配置文件加载用户的网络实现后，开始监控配置是否变更。
func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode kubeletconfig.HairpinMode, nonMasqueradeCIDR string, mtu int) error {
err := plugin.platformInit()
plugin.host = host
plugin.syncNetworkConfig()
// 定时监控配置文件是否发生变更
go wait.Forever(plugin.syncNetworkConfig, defaultSyncConfigPeriod)
return nil
}
五 插件使用
5.1 容器网络设置
在pod创建后启动之前设置网络信息
func (plugin *cniNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations, options map[string]string) error {
if err := plugin.checkInitialized(); err != nil {
return err
}
netnsPath, err := plugin.host.GetNetNS(id.ID)
// 添加pod到网络中
_, err = plugin.addToNetwork(cniTimeoutCtx, plugin.getDefaultNetwork(), name, namespace, id, netnsPath, annotations, options)
return err
}
func (plugin *cniNetworkPlugin) addToNetwork(ctx context.Context, network *cniNetwork, podName string, podNamespace string, podSandboxID kubecontainer.ContainerID, podNetnsPath string, annotations, options map[string]string) (cnitypes.Result, error) {
rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podSandboxID, podNetnsPath, annotations, options)
// 调用CNI接口
res, err := cniNet.AddNetworkList(ctx, netConf, rt)
return res, nil
}
5.2 CNI
cni关联到用户的实现，从代码中可以看出，自己的CNI实现是一个可执行程序，需要包含ADD等命令。
type CNI interface {
AddNetworkList(ctx context.Context, net *NetworkConfigList, rt *RuntimeConf) (types.Result, error)
CheckNetworkList(ctx context.Context, net *NetworkConfigList, rt *RuntimeConf) error
DelNetworkList(ctx context.Context, net *NetworkConfigList, rt *RuntimeConf) error
GetNetworkListCachedResult(net *NetworkConfigList, rt *RuntimeConf) (types.Result, error)
AddNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) (types.Result, error)
CheckNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) error
DelNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) error
GetNetworkCachedResult(net *NetworkConfig, rt *RuntimeConf) (types.Result, error)
ValidateNetworkList(ctx context.Context, net *NetworkConfigList) ([]string, error)
ValidateNetwork(ctx context.Context, net *NetworkConfig) ([]string, error)
}
func (c *CNIConfig) addNetwork(ctx context.Context, name, cniVersion string, net *NetworkConfig, prevResult types.Result, rt *RuntimeConf) (types.Result, error) {
c.ensureExec()
pluginPath, err := c.exec.FindInPath(net.Network.Type, c.Path)
newConf, err := buildOneConfig(name, cniVersion, net, prevResult, rt)
// 调用ADD方法
return invoke.ExecPluginWithResult(ctx, pluginPath, newConf.Bytes, c.args("ADD", rt), c.exec)
}
// AddNetworkList executes a sequence of plugins with the ADD command
func (c *CNIConfig) AddNetworkList(ctx context.Context, list *NetworkConfigList, rt *RuntimeConf) (types.Result, error) {
for _, net := range list.Plugins {
result, err = c.addNetwork(ctx, list.Name, list.CNIVersion, net, result, rt)
}
return result, nil
}
