cni: add ipv6/dual-stack support for bridge, calico cni

Author: kartikjoshi21

pkg/drivers/kic/kic.go

@@ -95,7 +95,6 @@ func (d *Driver) Create() error {
 	if params.Memory != "0" {
 		params.Memory += "mb"
 	}
-
 	networkName := d.NodeConfig.Network
 	if networkName == "" {
 		networkName = d.NodeConfig.ClusterName
@@ -106,21 +105,28 @@ func (d *Driver) Create() error {
 		d.OCIBinary,
 		networkName,
 		d.NodeConfig.Subnet,
-		d.NodeConfig.Subnetv6, // NEW
+		d.NodeConfig.Subnetv6,
 		staticIP,
-		d.NodeConfig.StaticIPv6, // NEW
-		params.IPFamily,         // NEW
+		d.NodeConfig.StaticIPv6,
+		params.IPFamily,
 	)
 	if err != nil {
 		msg := "Unable to create dedicated network, this might result in cluster IP change after restart: {{.error}}"
 		args := out.V{"error": err}
 		if staticIP != "" {
+			// With a user-requested static IP we can’t safely fall back
+			// to the default bridge network.
 			exit.Message(reason.IfDedicatedNetwork, msg, args)
 		}
 		out.WarningT(msg, args)
+		// NOTE: Do NOT set params.Network here – we’ll fall back to Docker’s
+		// default bridge network.
+	} else {
+		// Only attach to the dedicated network when creation succeeded.
+		// For IPv6-only clusters the gateway may be nil, but the network
+		// still exists and can be used.
+		params.Network = networkName
 	}
-	// Always attach to the created user network (even if gateway is nil in IPv6-only)
-	params.Network = networkName
 
 	// Now decide static IPs per family
 	switch params.IPFamily {
@@ -461,7 +467,11 @@ func (d *Driver) Remove() error {
 		return fmt.Errorf("expected no container ID be found for %q after delete. but got %q", d.MachineName, id)
 	}
 
-	if err := oci.RemoveNetwork(d.OCIBinary, d.NodeConfig.ClusterName); err != nil {
+	networkName := d.NodeConfig.ClusterName
+	if d.NodeConfig.Network != "" {
+		networkName = d.NodeConfig.Network
+	}
+	if err := oci.RemoveNetwork(d.OCIBinary, networkName); err != nil {
 		klog.Warningf("failed to remove network (which might be okay) %s: %v", d.NodeConfig.ClusterName, err)
 	}
 	return nil

pkg/drivers/kic/oci/network_create.go

@@ -36,7 +36,12 @@ import (
 // defaultFirstSubnetAddr is a first subnet to be used on first kic cluster
 // it is one octet more than the one used by KVM to avoid possible conflict
 const defaultFirstSubnetAddr = "192.168.49.0"
-const defaultFirstSubnetAddrv6 = "fd00::/64"
+
+// defaultFirstSubnetAddrv6 is the first IPv6 subnet used for kic networks.
+// Avoid fd00::/64 because Docker's IPv6 pools (e.g. fixed-cidr-v6) often sit
+// under fd00::/xx and will trigger "Pool overlaps with other one on this
+// address space". Use a different ULA /64 instead.
+const defaultFirstSubnetAddrv6 = "fd01::/64"
 
 // name of the default bridge network, used to lookup the MTU (see #9528)
 const dockerDefaultBridge = "bridge"
@@ -94,6 +99,7 @@ func CreateNetworkWithIPFamily(ociBin, networkName, subnet, subnetv6, staticIP,
 	if berr != nil {
 		klog.Warningf("failed to get mtu information from the %s's default network %q: %v", ociBin, bridgeName, berr)
 	}
+
 	// ----- IPv6/dual flow -----
 	if ipFamily == "ipv6" || ipFamily == "dual" {
 		// Decide v6 subnet
@@ -112,16 +118,19 @@ func CreateNetworkWithIPFamily(ociBin, networkName, subnet, subnetv6, staticIP,
 		// Build args; enable IPv6 always in this branch
 		args := []string{"network", "create", "--driver=bridge", "--ipv6"}
 
-		// For dual, also choose a free IPv4 subnet similar to the v4-only flow
-		if ipFamily == "dual" {
+		// For dual-stack with an explicit static IPv4, we must also control v4.
+		// In the common case (no static IPv4), let Docker choose a free IPv4
+		// subnet and only pin the IPv6 range. This avoids
+		// "Pool overlaps with other one on this address space" when 192.168.0.0/16
+		// is already taken by other Docker networks.
+		if ipFamily == "dual" && staticIP != "" {
 			tries := 20
 			start := firstSubnetAddr(subnet)
 			if staticIP != "" {
 				tries = 1
 				start = staticIP
 			}
 
-			// Try up to 5 candidate /24s starting at start, stepping by 9 (as before)
 			var lastErr error
 			for attempts, subnetAddr := 0, start; attempts < 5; attempts++ {
 				var p *network.Parameters
@@ -150,7 +159,6 @@ func CreateNetworkWithIPFamily(ociBin, networkName, subnet, subnetv6, staticIP,
 				}
 
 				out := rr.Output()
-				// Respect same retry conditions as v4-only
 				if strings.Contains(out, "Pool overlaps") ||
 					(strings.Contains(out, "failed to allocate gateway") && strings.Contains(out, "Address already in use")) ||
 					strings.Contains(out, "is being used by a network interface") ||
@@ -159,14 +167,16 @@ func CreateNetworkWithIPFamily(ociBin, networkName, subnet, subnetv6, staticIP,
 					subnetAddr = p.IP
 					continue
 				}
-				// Non-retryable
+
 				klog.Errorf("error creating dual-stack network %s: %v", networkName, err)
 				return nil, fmt.Errorf("un-retryable: %w", err)
 			}
 			return nil, fmt.Errorf("failed to create %s network %s (dual): %v", ociBin, networkName, lastErr)
 		}
 
-		// ipv6-only (no IPv4)
+		// ipv6-only / “IPv6 + auto IPv4” branch:
+		//  - ipFamily == "ipv6"
+		//  - OR ipFamily == "dual" && staticIP == "" (Docker picks IPv4 range)
 		args = append(args, "--subnet", subnetv6)
 		if ociBin == Docker && bridgeInfo.mtu > 0 {
 			args = append(args, "-o", fmt.Sprintf("com.docker.network.driver.mtu=%d", bridgeInfo.mtu))
@@ -178,7 +188,7 @@ func CreateNetworkWithIPFamily(ociBin, networkName, subnet, subnetv6, staticIP,
 		)
 
 		if _, err := runCmd(exec.Command(ociBin, args...)); err != nil {
-			klog.Warningf("failed to create %s network %q (ipv6-only): %v", ociBin, networkName, err)
+			klog.Warningf("failed to create %s network %q (ipv6/dual): %v", ociBin, networkName, err)
 			return nil, fmt.Errorf("create %s network %q: %w", ociBin, networkName, err)
 		}
 		ni, _ := containerNetworkInspect(ociBin, networkName)

pkg/minikube/cni/bridge.go

@@ -17,10 +17,10 @@ limitations under the License.
 package cni
 
 import (
-	"bytes"
+	"encoding/json"
 	"fmt"
 	"os/exec"
-	"text/template"
+	"strings"
 
 	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/assets"
@@ -32,38 +32,73 @@ import (
 // ref: https://www.cni.dev/plugins/current/meta/portmap/
 // ref: https://www.cni.dev/plugins/current/meta/firewall/
 
-// note: "cannot set hairpin mode and promiscuous mode at the same time"
-// ref: https://github.com/containernetworking/plugins/blob/7e9ada51e751740541969e1ea5a803cbf45adcf3/plugins/main/bridge/bridge.go#L424
-var bridgeConf = template.Must(template.New("bridge").Parse(`
-{
-  "cniVersion": "0.4.0",
-  "name": "bridge",
-  "plugins": [
-    {
-      "type": "bridge",
-      "bridge": "bridge",
-      "addIf": "true",
-      "isDefaultGateway": true,
-      "forceAddress": false,
-      "ipMasq": true,
-      "hairpinMode": true,
-      "ipam": {
-          "type": "host-local",
-          "subnet": "{{.PodCIDR}}"
-      }
-    },
-    {
-      "type": "portmap",
-      "capabilities": {
-          "portMappings": true
-      }
-    },
-    {
-       "type": "firewall"
-    }
-  ]
+// renderBridgeConflist builds a bridge CNI config that supports IPv4-only, IPv6-only, or dual-stack.
+func renderBridgeConflist(k8s config.KubernetesConfig) ([]byte, error) {
+	// minimal structs for JSON marshal
+	type rng struct {
+		Subnet string `json:"subnet"`
+	}
+	type ipam struct {
+		Type   string  `json:"type"`
+		Subnet string  `json:"subnet,omitempty"` // single-stack (v4 or v6)
+		Ranges [][]rng `json:"ranges,omitempty"` // dual-stack
+	}
+	type bridge struct {
+		Type             string `json:"type"`
+		Bridge           string `json:"bridge"`
+		IsDefaultGateway bool   `json:"isDefaultGateway"`
+		HairpinMode      bool   `json:"hairpinMode"`
+		IPMasq           bool   `json:"ipMasq"`
+		IPAM             ipam   `json:"ipam"`
+	}
+	type plugin struct {
+		Type         string          `json:"type"`
+		Capabilities map[string]bool `json:"capabilities,omitempty"`
+	}
+	type conflist struct {
+		CNIVersion string        `json:"cniVersion"`
+		Name       string        `json:"name"`
+		Plugins    []interface{} `json:"plugins"`
+	}
+
+	v4 := k8s.PodCIDR != ""
+	v6 := k8s.PodCIDRv6 != ""
+
+	cfgIPAM := ipam{Type: "host-local"}
+	switch {
+	case v4 && v6:
+		cfgIPAM.Ranges = [][]rng{{{Subnet: k8s.PodCIDR}}, {{Subnet: k8s.PodCIDRv6}}}
+	case v6:
+		cfgIPAM.Subnet = k8s.PodCIDRv6
+	default:
+		// fall back to previous default if unset upstream
+		cidr := k8s.PodCIDR
+		if cidr == "" {
+			cidr = DefaultPodCIDR
+		}
+		cfgIPAM.Subnet = cidr
+	}
+
+	// NAT generally not desired for IPv6; keep masquerade only for v4
+	ipMasq := v4 && !v6
+
+	br := bridge{
+		Type: "bridge", Bridge: "cni0",
+		IsDefaultGateway: true,
+		HairpinMode:      true,
+		IPMasq:           ipMasq,
+		IPAM:             cfgIPAM,
+	}
+	portmap := plugin{Type: "portmap", Capabilities: map[string]bool{"portMappings": true}}
+	firewall := plugin{Type: "firewall"}
+
+	out := conflist{
+		CNIVersion: "1.0.0",
+		Name:       "k8s-pod-network",
+		Plugins:    []interface{}{br, portmap, firewall},
+	}
+	return json.MarshalIndent(out, "", "  ")
 }
-`))
 
 // Bridge is a simple CNI manager for single-node usage
 type Bridge struct {
@@ -76,14 +111,11 @@ func (c Bridge) String() string {
 }
 
 func (c Bridge) netconf() (assets.CopyableFile, error) {
-	input := &tmplInput{PodCIDR: DefaultPodCIDR}
-
-	b := bytes.Buffer{}
-	if err := bridgeConf.Execute(&b, input); err != nil {
+	cfgBytes, err := renderBridgeConflist(c.cc.KubernetesConfig)
+	if err != nil {
 		return nil, err
 	}
-
-	return assets.NewMemoryAssetTarget(b.Bytes(), "/etc/cni/net.d/1-k8s.conflist", "0644"), nil
+	return assets.NewMemoryAssetTarget(cfgBytes, "/etc/cni/net.d/1-k8s.conflist", "0644"), nil
 }
 
 // Apply enables the CNI
@@ -110,5 +142,15 @@ func (c Bridge) Apply(r Runner) error {
 
 // CIDR returns the default CIDR used by this CNI
 func (c Bridge) CIDR() string {
+
+	// Prefer explicitly-set CIDRs from the cluster config.
+	k := c.cc.KubernetesConfig
+	if k.PodCIDRv6 != "" && (strings.ToLower(k.IPFamily) == "ipv6" || k.PodCIDR == "") {
+		return k.PodCIDRv6
+	}
+
+	if k.PodCIDR != "" {
+		return k.PodCIDR
+	}
 	return DefaultPodCIDR
 }