kubestack 源码分析
简介：KubeStack is an OpenStack network provider for kubernetes.KubeStack is devided into two functions:[*]kubestack running on the same host with kube-controller-manager, which provides network management for openstack
[*]kubestack running on each minion host, which setups container's network interfaces
kubestack.conf的默认配置文件如下所示：
auth-url = http://${IF_IP}:5000/v2.0
username = admin
password = admin
tenant-name = admin
region = RegionOne
ext-net-id = ${EXT_NET_ID}
create-monitor = yes
monitor-delay = 1m
monitor-timeout = 30s
monitor-max-retries = 3
plugin-name = ovs
// kubestack.go
1、func main()
1、调用config, err := os.Open(configFile)打开配置文件，configFile默认为/etc/kubestack.conf
2、调用openstack, err := common.NewOpenStack(config)
3、调用server := kubestack.NewKubeHandler(openstack)
4、最后，调用server.Serve(port)进行监听，端口号默认为4237
OpenStack数据结构如下所示：
// OpenStack is an implementation of network provider Interface for OpenStack
type OpenStack struct {
network *gophercloud.ServiceClient
identity *gophercloud.ServiceClient
provider *gophercloud.ProviderClient
region string
lbOpts LoadBalancerOpts
pluginOpts PluginOpts
ExtNetID string
Plugin plugins.PluginInterface
}
// pkg/common/openstack.go
2、func NewOpenStack(config io.Reader) (*OpenStack, error)
[*]调用err := gcfg.ReadInto(&cfg, config)读取配置信息
[*]调用provider, err := openstack.AuthenticatedClient(cfg.toAuthOptions()) 用于auth openstack
[*]调用identity, err := openstack.NewIdentityV2(provider, gophercloud.EndpointOpts{Availability: gophercloud.AvailabilityAdmin})用于find identity endpoint
[*]调用network, err := openstack.NewNetworkV2(provider, gophercloud.EndpointOpts{Region: cfg.Global.Region})
[*]创建 os := OpenStack{
identity: identity,
network: network,
provider: provider,
region: cfg.Global.Region,
lbOpts: cfg.LoadBalancer,
pluginOpts: cfg.Plugin,
ExtNetID: cfg.Global.ExtNetID,
}
[*]当cfg.Plugin.PluginName 不为nil时：
[*]integrationBriage := "br-int"，若cfg.Plugin.IntegrationBridge不为""，则设置integrationBriage = cfg.Plugin.IntegrationBridge
[*]调用plugin, _ := plugins.GetNetworkPlugin(cfg.Plugin.PluginName)
[*]若plugin不为nil，则调用plugin.Init(integrationBriage)，并且os.Plugin = plugin
[*]最后，返回return &os, nil
// pkg/plugins/plugins.go
// GetNetworkPlugin creates an instance of the named network plugin, or nil if
// the name is unkown. The error return is only used if the named plugin
// was known but failed to initialize
3、func GetNetworkPlugin(name string) (PluginInterface, error)
该函数仅仅调用f, found := plugins获取plugin的Factory函数f()，调用该f()函数生成相应的PluginInterface
（如果相应的Plugin被支持，则相应的包在初始化的时候就已经在init()函数中进行了注册，因此一定能在此时找到）
PluginInterface数据结构如下所示：
type PluginInterface interface {
SetupInterface(podName, podInfraContainerID string, port *ports.Port, ipcidr, gateway string, dnsServers []string, containerRuntime string) error
DestroyInterface(podName, podInfraContainerID string, port *ports.Port, containerRuntime string) error
Init(integrationBridge string) error
}
// pkg/kubestack/kubestack.go
4、func NewKubeHandler(driver *common.OpenStack) *kubeHandler
该函数仅仅只是初始化并注册grpc server，生成h := &KubeHandler{driver: driver, server: grpc.NewServer()}
kubeHandler的结构如下所示：
// KubeHandler forwards requests and responses between
// the docker daemon and the plugin.
type KubeHandler struct {
driver *common.OpenStack
server *grpc.Server
}
-------------到此为止，kubestack 初始化完成，接下来为网络的创建过程-------------
// pkg/kubestack/kubestack.go
1、func (h *KubeHandler) CreateNetwork(c context.Context, req *provider.CreateNetworkRequest) (*provider.CommonResponse, error)
1、首先调用req.Network.TenantID = h.driver.ToTenantID(req.Network.TenantID)，将tenantName转换为tenantID
2、再调用err := h.driver.CreateNetwork(req.Network)创建网络
Network数据结构如下所示：
type Network struct {
Name string
Uid string
TenantID string
SegmentID int32
Subnets []*Subnet
// Status of network
// Valid value: Initializing, Active, Pending, Failed, Terminating
Status string
}
Subnet数据结构如下所示：
type Subnet struct {
Name string
Uid string
Cidr string
Gateway string
Tenantid string
Dnsservers []string
Routes []*Route
}
// pkg/common/openstack.go
2、func (os *OpenStack) CreateNetwork(network *provider.Network) error
1、首先创建opts := networks.CreateOpts{
Name: network.Name,
AdminStateUp: &adminStateUp,
TenantID: network.TenantID,
}
再调用osNet, err := networks.Create(os.network, opts).Extract()创建网络
2、同样调用routerOpts := routers.CreateOpts{
Name: network.Name,
TenantID: network.TenantID,
GatewayInfo: &routers.GatewayInfo{NetworkID: os.ExtNetID},
}
再调用osRouter, err := routers.Create(os.network, routerOpts).Extract()创建router
3、最后，遍历network.Subnets创建子网
[*]首先创建 subnetOpts := subnets.CreateOpts{
NetworkID: networkID, //其实就是osNet.ID
CIDR: sub.Cidr,
Name: sub.Name,
IPVersion: gophercloud.IPv4,
TenantID: network.TenantID,
GatewaryIP: &sub.Gateway,
DNSNameserver: sub.Dnsservers,
}
再调用s, err := subnets.Create(os.network, subnetOpts).Extract()创建子网
// 将子网加入router中
2. 创建opts := routers.AddInterfaceOpts{SubnetID: s.ID}，
再调用routers.AddInterface(os.network, osRouter.ID, opts).Extract()
---------------------------------------创建pod-----------------------------------------
// pkg/kubestack/kubestack.go
1、func (h *KubeHandler) SetupPod(c context.Context, req *provider.SetupPodRequest) (*provider.CommonResponse, error)
该函数仅仅是调用h.driver.SetupPod(req.PodName, req.Namespace, req.PodInfraContainerID, req.Network, req.ContainerRuntime)
SetupPodRequest数据结构如下所示：
type SetupPodRequest struct {
PodName string
Namespace string
ContainerRuntime string
PodInfraContainerID string
Network *Network
}
// pkg/common/openstack.go
2、func (os *OpenStack) SetupPod(podName, namespace, podInfraContainerID string, network *provider.Network, containerRuntime string)
1、调用portName := os.BuildPortName(podName, namespace, network.Uid)，该函数的作用仅仅是将podNamePrefix和上述三个参数连成一个字符串而已
2、获取dns server的ip，调用networkPorts, err := os.ListPorts(network.Uid, "network:dhcp")，再遍历networkPorts，调用dnsServers = append(dnsServers, p.FixedIPs.IPAddress)
3、调用port, err := os.GetPort(portName)，从openstack中获取port，如果port不存在，则调用portWithBinding, err := os.CreatePort(network.Uid, network.TenantID, portName, podHostname)创建一个
4、如果port.DeviceOwner和实际的deviceOwner不符，则更新hostname
5、调用subnet, err := os.getProviderSubnet(port.FixedIPs.SubnetID)获取子网和网关
6、最后，调用os.Plugin.SetupInterface(podName+"_"+namespace, podInfraContainerID, port, fmt.Sprintf("%s/%d", port.FixedIPs.IPAddress, prefixSize), subnet.Gateway, dnsServers, containerRuntime)为pod设置网卡
-----------------------------------OVS plugin实现---------------------------------------
OVSPlugin的数据结构很简单，如下所示：
type OVSPlugin struct {
IntegrationBridage string
}
// pkg/plugins/openvswitch/openvswitch.go
1、func (p *OVSPlugin) Init(integrationBridge string) error
该函数仅仅将p.IntegrationBridage赋值为integrationBridge
// pkg/plugins/openvswitch/openvswitch.go
2、func (p *OVSPlugin) SetupInterface(podName, podInfraContainerID string, port *ports.Port, ipcidr, gatewary string, dnsServers []string, containerRuntime string) error
1、先调用err := p.SetupOVSInterface(podName, podInfraContainerID, port, ipcidr, gatewary, containerRuntime)创建OVS interface
2、再根据containerRuntime的不同，分别调用p.SetupDockerInterface(podName, podInfraContainerID, port, ipcidr, gateway)或者p.SetupHyperInterface(podName, podInfraContainerID, port, ipcidr, gateway, dnsServers)
// pkg/plugins/openvswitch/openvswitch.go
3、func (p *OVSPlugin)SetupOVSInterface(podName, podInfraContainerID string, port *ports.Port, ipcidr, gateway string, containerRuntime string)
1、该函数调用exec.RunCommand函数创建一个veth pair以及一个bridge，并将其中一个veth加入bridge中
2、调用ovs-vsctl，对另一个veth进行操作
// pkg/plugins/openvswitch/openvswitch.go
4、func (p *OVSPlugin) SetupDockerInterface(podName, podInfraContainerID string, port *ports.Port, ipcidr, gateway string) error
1、先创建一个veth对，tapName和vifName，并且将tapName加入之前创建的bridge中
2、将vifName的MAC地址设置为port.MACAddress
3、根据podInfraContainerID容器的pid找到对应的net ns，并把它软链接到/var/run/netns目录，从而能用ip命令对其直接进行操作。
4、将vifName加入net ns中，再删除其中的eth0网卡，再将vifName重命名为eth0
5、调用`ip addr add dev eth0 ipcidr`添加ip，以及`ip route add default via gateway`创建默认路由
// pkg/plugins/openvswitch/openvswitch.go
5、func (p *OVSPlugin) SetupHyperInterface(podName, podInfraContainerID string, port *ports.Port, ipcidr, gateway string, dnsServers []string)
1、首先获取bridge和tapName
2、创建interfaceSpec := mapstring{
"bridge": bridge,
"ifname": tapName,
"mac": port.MACAddress,
"ip": ipcidr,
"gateway": gateway,
} 生成network interface的配置
3、调用podSpec, err := p.getHyperPodSpec(podName)获取原始的hyper配置，并解码至specData中
4、调用specData["interfaces"] = []mapstring{interfaceSpec}将network interface的配置加入其中
5、在specData["dns"]的基础之上添加dnsServers
6、最后调用newPodSpec, err := json.Marshal(specData)和p.saveHyperPodSpec(string(newPodSpec), podName)将新的hyper配置写入文件中
注：Neutron Port代表的是一个逻辑交换机中的一个虚拟交换机端口。虚拟机实例将自己的网卡和这些端口相连，这些逻辑端口同时定义了接入到它们中的接口的MAC地址和IP地址。当一个IP地址和一个端口关联起来的时候，这意味着这个端口和一个子网相连，因为这个IP地址就是从一个特定子网的allocation pool中获取的。
页:
[1]