diff --git a/network-agent/cmd/network-agent/main.go b/network-agent/cmd/network-agent/main.go index c11ac6f2..b57a2cda 100644 --- a/network-agent/cmd/network-agent/main.go +++ b/network-agent/cmd/network-agent/main.go @@ -246,7 +246,7 @@ func validateService(svc service.Service) error { func summarizeConfig(cfg service.Config) string { return fmt.Sprintf( - "eth_name=%q object_dir=%q cidr=%q mvm_inner_ip=%q mvm_mac_addr=%q mvm_gw_dest_ip=%q mvm_gw_mac_addr=%q mvm_mask=%d mvm_mtu=%d tap_init_num=%d state_dir=%q tap_fd_socket_path=%q host_proxy_bind_ip=%q connect_timeout=%s", + "eth_name=%q object_dir=%q cidr=%q mvm_inner_ip=%q mvm_mac_addr=%q mvm_gw_dest_ip=%q mvm_gw_mac_addr=%q mvm_mask=%d mvm_mtu=%d tap_init_num=%d state_dir=%q tap_fd_socket_path=%q host_proxy_bind_ip=%q connect_timeout=%s host_port_min=%d host_port_max=%d", cfg.EthName, cfg.ObjectDir, cfg.CIDR, @@ -261,6 +261,8 @@ func summarizeConfig(cfg service.Config) string { cfg.TapFDSocketPath, cfg.HostProxyBindIP, cfg.ConnectTimeout, + cfg.HostPortMin, + cfg.HostPortMax, ) } diff --git a/network-agent/internal/service/config.go b/network-agent/internal/service/config.go index b760fcf3..73cd2417 100644 --- a/network-agent/internal/service/config.go +++ b/network-agent/internal/service/config.go @@ -49,6 +49,17 @@ type Config struct { // shared-dict op should be sub-millisecond; this is generous on // purpose so a transient kernel hiccup doesn't fail the push. CubeEgressPushTimeout time.Duration + + // HostPortMin / HostPortMax bound the inclusive range from which + // network-agent allocates host-side ports for sandbox port-mapping + // NAT rules. With the defaults (20000-29999, 10000 ports total) and + // 4 default exposed ports per sandbox, a node caps at ~2500 alive + // sandboxes regardless of cpu/mem quotas. Operators can widen the + // range (e.g. 20000-59999) to push the ceiling closer to the + // underlying mem quota wall. When both are zero the allocator falls + // back to 20000-29999 for backwards compatibility. + HostPortMin uint16 + HostPortMax uint16 } func DefaultConfig() Config { @@ -69,6 +80,8 @@ func DefaultConfig() Config { ConnectTimeout: 5 * time.Second, CubeEgressAdminURL: "http://127.0.0.1:9090", CubeEgressPushTimeout: 2 * time.Second, + HostPortMin: defaultPortMin, + HostPortMax: defaultPortMax, } } @@ -89,6 +102,8 @@ type cubeletNetworkConfig struct { MvmMtu int `toml:"mvm_mtu"` CubeEgressAdminURL string `toml:"cube_egress_admin_url"` CubeEgressPushTimeout string `toml:"cube_egress_push_timeout"` + HostPortMin uint16 `toml:"host_port_min"` + HostPortMax uint16 `toml:"host_port_max"` } const cubeletNetworkPluginKey = "io.cubelet.internal.v1.network" @@ -154,5 +169,11 @@ func LoadConfigFromCubeletTOML(base Config, path string) (Config, error) { } base.CubeEgressPushTimeout = d } + if networkCfg.HostPortMin != 0 { + base.HostPortMin = networkCfg.HostPortMin + } + if networkCfg.HostPortMax != 0 { + base.HostPortMax = networkCfg.HostPortMax + } return base, nil } diff --git a/network-agent/internal/service/local_service.go b/network-agent/internal/service/local_service.go index 41f82259..bc7b0d75 100644 --- a/network-agent/internal/service/local_service.go +++ b/network-agent/internal/service/local_service.go @@ -94,7 +94,7 @@ func NewLocalService(cfg Config) (Service, error) { if err != nil { return nil, err } - ports, err := newPortAllocator() + ports, err := newPortAllocator(cfg.HostPortMin, cfg.HostPortMax) if err != nil { return nil, err } diff --git a/network-agent/internal/service/port_allocator.go b/network-agent/internal/service/port_allocator.go index 0a96fd52..dd3169dc 100644 --- a/network-agent/internal/service/port_allocator.go +++ b/network-agent/internal/service/port_allocator.go @@ -13,8 +13,8 @@ import ( ) const ( - portMin uint16 = 20000 - portMax uint16 = 29999 + defaultPortMin uint16 = 20000 + defaultPortMax uint16 = 29999 ) type portAllocator struct { @@ -25,11 +25,20 @@ type portAllocator struct { assigned map[uint16]struct{} } -func newPortAllocator() (*portAllocator, error) { +func newPortAllocator(min, max uint16) (*portAllocator, error) { + if min == 0 && max == 0 { + min, max = defaultPortMin, defaultPortMax + } + if min > max { + return nil, fmt.Errorf("invalid host port range: min=%d > max=%d", min, max) + } + if min < 1024 { + return nil, fmt.Errorf("invalid host port range: min=%d must be >= 1024", min) + } alloc := &portAllocator{ - min: portMin, - max: portMax, - next: portMin, + min: min, + max: max, + next: min, assigned: make(map[uint16]struct{}), } reservedPorts, err := getReservedPorts() @@ -37,7 +46,7 @@ func newPortAllocator() (*portAllocator, error) { return nil, err } for _, port := range reservedPorts { - if port < portMin || port > portMax { + if port < min || port > max { continue } alloc.assigned[port] = struct{}{}