feat!: add toolset versioning

docs/TOOLSET_VERSIONING.md

@@ -0,0 +1,53 @@
+# Toolset Versioning
+
+This document describes how toolsets are versioned, the rules for toolsets changing versions, and how to configure which 
+tools/toolsets should be used through their versions.
+
+## How Toolsets and Tools are Versioned
+
+All tools/prompts and toolsets are versioned as one of "alpha", "beta", or "ga"/"stable". Each toolset has a default version
+for the toolset, however individual tools/prompts may have their own versions. For example, a toolset as a whole may be in beta,
+however a newly added tool in that toolset may only be in alpha.
+
+The general idea for these versions is:
+- "alpha": the toolset is not guaranteed to work well
+- "beta": the toolset is not guaranteed to work well, but we are evaluating how well it works
+- "stable": the toolset works well, and we are evaluating how well it works to avoid regressions
+
+## Rules for Tool/Prompt/Toolset Versioning
+
+Below are the criteria for the versioning of every tool/prompt/toolset.
+
+### Alpha
+
+All tools/prompts/toolsets begin in "alpha". If you are contributing a new tool/prompt/toolset, this is the version to set.
+There are no minimum requirements for something to be considered alpha, apart from the code getting merged.
+
+### Beta
+
+For a tool/prompt/toolset to enter into "beta", we require that there are eval scenarios. For a toolset to enter "beta", there must be scenarios
+excercising all of the tools and prompts in the toolset. For individual tools and prompts to enter "beta", we only require an eval scenario
+for the specific tool or prompt.
+
+**Note**: for beta we do not require that all the eval scenarios are passing - we just require that they exist.
+
+### GA/Stable
+
+For a tool/prompt/toolset to enter into "stable", we require that 95% or more of the eval scenarios are passing. There is the same requirements as "beta" in terms of the number of evaluation scenarios.
+
+## Configuring tools/toolsets on the server by their version
+
+When configuring the MCP server, you can set a default toolset version to use for all tools with the `default_toolset_version` key. 
+Within all the toolsets you enable, only the tools which meet this minimum version will be enabled. For example, if a toolset has 
+both "alpha" and "beta" tools and you enable only "beta" tools on the toolset, you will not see any of the "alpha" tools.
+
+You can also enable specific minimum versions for specific toolsets using the "toolset:version" syntax when enabling the toolset.
+For example, if you want to allow all the "alpha" tools in the "core" toolset, you could set `toolsets = [ "core:alpha" ]`, and this would
+enable all alpha+ tools in the core toolset.
+
+See a full config example below:
+```toml
+default_toolset_version = "beta"
+
+toolsets = [ "core", "config", "helm:alpha" ]
+```

pkg/api/prompts.go

@@ -9,6 +9,7 @@ type ServerPrompt struct {
 	Handler        PromptHandlerFunc
 	ClusterAware   *bool
 	ArgumentSchema map[string]PromptArgument
+	Version        *Version // Optional version - defaults to toolset version if not set
 }
 
 // IsClusterAware indicates whether the prompt can accept a "cluster" or "context" parameter

pkg/api/toolsets.go

@@ -3,16 +3,46 @@ package api
 import (
 	"context"
 	"encoding/json"
+	"fmt"
+	"strings"
 
 	"github.com/containers/kubernetes-mcp-server/pkg/output"
 	"github.com/google/jsonschema-go/jsonschema"
 )
 
+type Version int
+
+const (
+	VersionUnknown Version = iota
+	VersionAlpha
+	VersionBeta
+	VersionGA
+)
+
+func (v *Version) UnmarshalText(text []byte) error {
+	var tmp Version
+	switch strings.ToLower(string(text)) {
+	case "alpha":
+		tmp = VersionAlpha
+	case "beta":
+		tmp = VersionBeta
+	case "ga", "", "stable":
+		tmp = VersionGA
+	default:
+		return fmt.Errorf("unknown version '%s': must be one of 'alpha', 'beta', 'stable', 'ga'", text)
+	}
+
+	*v = tmp
+
+	return nil
+}
+
 type ServerTool struct {
 	Tool               Tool
 	Handler            ToolHandlerFunc
 	ClusterAware       *bool
 	TargetListProvider *bool
+	Version            *Version // Optional version - defaults to toolset version if not set
 }
 
 // IsClusterAware indicates whether the tool can accept a "cluster" or "context" parameter
@@ -46,6 +76,9 @@ type Toolset interface {
 	// GetPrompts returns the prompts provided by this toolset.
 	// Returns nil if the toolset doesn't provide any prompts.
 	GetPrompts() []ServerPrompt
+	// GetVersion returns the version of the toolset.
+	// This version can be overridden by specific tools/prompts (e.g. a toolset may be beta, but have an alpha tool).
+	GetVersion() Version
 }
 
 type ToolCallRequest interface {

pkg/config/config.go

@@ -86,6 +86,9 @@ type StaticConfig struct {
 	// This can be used to provide specific instructions on how the client should use the server
 	ServerInstructions string `toml:"server_instructions,omitempty"`
 
+	// Which toolset version to enable (any tools/toolsets below this will be disabled)
+	DefaultToolsetVersion api.Version `toml:"default_toolset_version"`
+
 	// Internal: parsed provider configs (not exposed to TOML package)
 	parsedClusterProviderConfigs map[string]api.ExtendedConfig
 	// Internal: parsed toolset configs (not exposed to TOML package)

pkg/config/config_default.go

@@ -4,12 +4,14 @@ import (
 	"bytes"
 
 	"github.com/BurntSushi/toml"
+	"github.com/containers/kubernetes-mcp-server/pkg/api"
 )
 
 func Default() *StaticConfig {
 	defaultConfig := StaticConfig{
-		ListOutput: "table",
-		Toolsets:   []string{"core", "config", "helm"},
+		ListOutput:            "table",
+		Toolsets:              []string{"core", "config"},
+		DefaultToolsetVersion: api.VersionBeta, // TODO: once the core toolset moves to GA, switch this to GA
 	}
 	overrides := defaultOverrides()
 	mergedConfig := mergeConfig(defaultConfig, overrides)

pkg/kubernetes-mcp-server/cmd/root.go

@@ -57,43 +57,45 @@ kubernetes-mcp-server --port 8080 --disable-multi-cluster
 )
 
 const (
-	flagVersion              = "version"
-	flagLogLevel             = "log-level"
-	flagConfig               = "config"
-	flagConfigDir            = "config-dir"
-	flagPort                 = "port"
-	flagSSEBaseUrl           = "sse-base-url"
-	flagKubeconfig           = "kubeconfig"
-	flagToolsets             = "toolsets"
-	flagListOutput           = "list-output"
-	flagReadOnly             = "read-only"
-	flagDisableDestructive   = "disable-destructive"
-	flagStateless            = "stateless"
-	flagRequireOAuth         = "require-oauth"
-	flagOAuthAudience        = "oauth-audience"
-	flagAuthorizationURL     = "authorization-url"
-	flagServerUrl            = "server-url"
-	flagCertificateAuthority = "certificate-authority"
-	flagDisableMultiCluster  = "disable-multi-cluster"
+	flagVersion               = "version"
+	flagLogLevel              = "log-level"
+	flagConfig                = "config"
+	flagConfigDir             = "config-dir"
+	flagPort                  = "port"
+	flagSSEBaseUrl            = "sse-base-url"
+	flagKubeconfig            = "kubeconfig"
+	flagToolsets              = "toolsets"
+	flagListOutput            = "list-output"
+	flagReadOnly              = "read-only"
+	flagDisableDestructive    = "disable-destructive"
+	flagStateless             = "stateless"
+	flagRequireOAuth          = "require-oauth"
+	flagOAuthAudience         = "oauth-audience"
+	flagAuthorizationURL      = "authorization-url"
+	flagServerUrl             = "server-url"
+	flagCertificateAuthority  = "certificate-authority"
+	flagDisableMultiCluster   = "disable-multi-cluster"
+	flagDefaultToolsetVersion = "default-toolset-version"
 )
 
 type MCPServerOptions struct {
-	Version              bool
-	LogLevel             int
-	Port                 string
-	SSEBaseUrl           string
-	Kubeconfig           string
-	Toolsets             []string
-	ListOutput           string
-	ReadOnly             bool
-	DisableDestructive   bool
-	Stateless            bool
-	RequireOAuth         bool
-	OAuthAudience        string
-	AuthorizationURL     string
-	CertificateAuthority string
-	ServerURL            string
-	DisableMultiCluster  bool
+	Version               bool
+	LogLevel              int
+	Port                  string
+	SSEBaseUrl            string
+	Kubeconfig            string
+	Toolsets              []string
+	ListOutput            string
+	ReadOnly              bool
+	DisableDestructive    bool
+	Stateless             bool
+	RequireOAuth          bool
+	OAuthAudience         string
+	AuthorizationURL      string
+	CertificateAuthority  string
+	ServerURL             string
+	DisableMultiCluster   bool
+	DefaultToolsetVersion string
 
 	ConfigPath   string
 	ConfigDir    string
@@ -154,6 +156,7 @@ func NewMCPServer(streams genericiooptions.IOStreams) *cobra.Command {
 	cmd.Flags().StringVar(&o.CertificateAuthority, flagCertificateAuthority, o.CertificateAuthority, "Certificate authority path to verify certificates. Optional. Only valid if require-oauth is enabled.")
 	_ = cmd.Flags().MarkHidden(flagCertificateAuthority)
 	cmd.Flags().BoolVar(&o.DisableMultiCluster, flagDisableMultiCluster, o.DisableMultiCluster, "Disable multi cluster tools. Optional. If true, all tools will be run against the default cluster/context.")
+	cmd.Flags().StringVar(&o.DefaultToolsetVersion, flagDefaultToolsetVersion, o.DefaultToolsetVersion, "Default version to enable for tools/toolsets, within the enabled tools and toolsets.")
 
 	return cmd
 }
@@ -225,6 +228,12 @@ func (m *MCPServerOptions) loadFlags(cmd *cobra.Command) {
 	if cmd.Flag(flagDisableMultiCluster).Changed && m.DisableMultiCluster {
 		m.StaticConfig.ClusterProviderStrategy = api.ClusterProviderDisabled
 	}
+	if cmd.Flag(flagDefaultToolsetVersion).Changed {
+		var v api.Version
+		if err := v.UnmarshalText([]byte(m.DefaultToolsetVersion)); err == nil {
+			m.StaticConfig.DefaultToolsetVersion = v
+		}
+	}
 }
 
 func (m *MCPServerOptions) initializeLogging() {

pkg/kubernetes-mcp-server/cmd/root_sighup_test.go

@@ -81,9 +81,9 @@ func (s *SIGHUPSuite) TestSIGHUPReloadsConfigFromFile() {
 		s.False(slices.Contains(s.server.GetEnabledTools(), "helm_list"))
 	})
 
-	// Modify the config file to add helm toolset
+	// Modify the config file to add helm toolset (with alpha version to include alpha-versioned tools)
 	s.Require().NoError(os.WriteFile(configPath, []byte(`
-		toolsets = ["core", "config", "helm"]
+		toolsets = ["core", "config", "helm:alpha"]
 	`), 0644))
 
 	// Send SIGHUP to current process
@@ -97,10 +97,10 @@ func (s *SIGHUPSuite) TestSIGHUPReloadsConfigFromFile() {
 }
 
 func (s *SIGHUPSuite) TestSIGHUPReloadsFromDropInDirectory() {
-	// Create initial config file - with helm enabled
+	// Create initial config file - with helm enabled (with alpha version to include alpha-versioned tools)
 	configPath := filepath.Join(s.tempDir, "config.toml")
 	s.Require().NoError(os.WriteFile(configPath, []byte(`
-		toolsets = ["core", "config", "helm"]
+		toolsets = ["core", "config", "helm:alpha"]
 	`), 0644))
 
 	// Create initial drop-in file that removes helm
@@ -115,9 +115,9 @@ func (s *SIGHUPSuite) TestSIGHUPReloadsFromDropInDirectory() {
 		s.False(slices.Contains(s.server.GetEnabledTools(), "helm_list"))
 	})
 
-	// Update drop-in file to add helm back
+	// Update drop-in file to add helm back (with alpha version to include alpha-versioned tools)
 	s.Require().NoError(os.WriteFile(dropInPath, []byte(`
-		toolsets = ["core", "config", "helm"]
+		toolsets = ["core", "config", "helm:alpha"]
 	`), 0644))
 
 	// Send SIGHUP
@@ -161,9 +161,9 @@ func (s *SIGHUPSuite) TestSIGHUPWithInvalidConfigContinues() {
 		s.False(slices.Contains(s.server.GetEnabledTools(), "helm_list"))
 	})
 
-	// Now fix the config and add helm
+	// Now fix the config and add helm (with alpha version to include alpha-versioned tools)
 	s.Require().NoError(os.WriteFile(configPath, []byte(`
-		toolsets = ["core", "config", "helm"]
+		toolsets = ["core", "config", "helm:alpha"]
 	`), 0644))
 
 	// Send another SIGHUP
@@ -189,9 +189,9 @@ func (s *SIGHUPSuite) TestSIGHUPWithConfigDirOnly() {
 		s.False(slices.Contains(s.server.GetEnabledTools(), "helm_list"))
 	})
 
-	// Update drop-in file to add helm
+	// Update drop-in file to add helm (with alpha version to include alpha-versioned tools)
 	s.Require().NoError(os.WriteFile(dropInPath, []byte(`
-		toolsets = ["core", "config", "helm"]
+		toolsets = ["core", "config", "helm:alpha"]
 	`), 0644))
 
 	// Send SIGHUP

pkg/kubernetes-mcp-server/cmd/root_test.go

@@ -299,8 +299,8 @@ func TestToolsets(t *testing.T) {
 		ioStreams, out := testStream()
 		rootCmd := NewMCPServer(ioStreams)
 		rootCmd.SetArgs([]string{"--version", "--port=1337", "--log-level=1"})
-		if err := rootCmd.Execute(); !strings.Contains(out.String(), "- Toolsets: core, config, helm") {
-			t.Fatalf("Expected toolsets 'full', got %s %v", out, err)
+		if err := rootCmd.Execute(); !strings.Contains(out.String(), "- Toolsets: core, config") {
+			t.Fatalf("Expected toolsets 'core, config', got %s %v", out, err)
 		}
 	})
 	t.Run("set with --toolsets", func(t *testing.T) {

pkg/mcp/helm_test.go

@@ -31,6 +31,9 @@ type HelmSuite struct {
 
 func (s *HelmSuite) SetupTest() {
 	s.BaseMcpSuite.SetupTest()
+	s.Require().NoError(toml.Unmarshal([]byte(`
+	toolsets = [ "helm:alpha" ]
+	`), s.Cfg), "Expected to parse toolsets config")
 	clearHelmReleases(s.T().Context(), kubernetes.NewForConfigOrDie(envTestRestConfig))
 
 	// Capture log output to verify denied resource messages

pkg/mcp/kiali_test.go

@@ -25,7 +25,7 @@ func (s *KialiSuite) SetupTest() {
 	s.mockServer.Config().BearerToken = "token-xyz"
 	kubeConfig := s.Cfg.KubeConfig
 	s.Cfg = test.Must(config.ReadToml([]byte(fmt.Sprintf(`
-		toolsets = ["kiali"]
+		toolsets = ["kiali:alpha"]
 		[toolset_configs.kiali]
 		url = "%s"
 	`, s.mockServer.Config().Host))))

pkg/mcp/kubevirt_test.go

@@ -59,7 +59,7 @@ func (s *KubevirtSuite) TearDownSuite() {
 func (s *KubevirtSuite) SetupTest() {
 	s.BaseMcpSuite.SetupTest()
 	s.Require().NoError(toml.Unmarshal([]byte(`
-		toolsets = [ "kubevirt" ]
+	toolsets = [ "kubevirt:alpha" ]
 	`), s.Cfg), "Expected to parse toolsets config")
 	s.InitMcpClient()
 }

pkg/mcp/mcp.go

@@ -30,7 +30,10 @@ type Configuration struct {
 func (c *Configuration) Toolsets() []api.Toolset {
 	if c.toolsets == nil {
 		for _, toolset := range c.StaticConfig.Toolsets {
-			c.toolsets = append(c.toolsets, toolsets.ToolsetFromString(toolset))
+			versioned := toolsets.VersionedToolsetFromString(toolset, c.DefaultToolsetVersion)
+			if versioned != nil {
+				c.toolsets = append(c.toolsets, versioned)
+			}
 		}
 	}
 	return c.toolsets