diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml new file mode 100644 index 0000000..a223544 --- /dev/null +++ b/.github/workflows/smoke.yml @@ -0,0 +1,152 @@ +name: Smoke Tests + +on: + push: + branches: + - main + pull_request: + types: + - opened + - reopened + - synchronize + +jobs: + smoke: + name: Run Smoke Tests + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Download dependencies + run: go mod download + + - name: Create kind cluster + uses: helm/kind-action@v1 + with: + cluster_name: stackrox-mcp-smoke + + - name: Checkout StackRox repository + uses: actions/checkout@v4 + with: + repository: stackrox/stackrox + path: stackrox-repo + + - name: Deploy StackRox Central + env: + MAIN_IMAGE_TAG: latest + SENSOR_HELM_DEPLOY: "true" + ROX_SCANNER_V4: "false" + ADMISSION_CONTROLLER: "false" + SCANNER_REPLICAS: "0" + COLLECTION_METHOD: "no_collection" + run: | + cd stackrox-repo + ./deploy/k8s/deploy-local.sh + + - name: Wait for Central pods ready + run: kubectl wait --for=condition=ready --timeout=180s pod -l app=central -n stackrox + + - name: Remove resource constraints from Sensor + run: | + # Use kubectl set resources to remove all resource constraints + kubectl set resources deployment/sensor -n stackrox \ + --requests=cpu=0,memory=0 \ + --limits=cpu=0,memory=0 + + # Delete sensor pods to force recreation with new (empty) resources + kubectl delete pods -n stackrox -l app=sensor + + # Wait a bit for pods to be deleted and recreated + sleep 10 + + # Wait for new pods to be ready + kubectl wait --for=condition=ready --timeout=300s pod -l app=sensor -n stackrox + + - name: Extract Central password + id: extract-password + run: | + PASSWORD="$(cat stackrox-repo/deploy/k8s/central-deploy/password)" + echo "::add-mask::${PASSWORD}" + echo "password=${PASSWORD}" >> "$GITHUB_OUTPUT" + + - name: Setup port-forward to Central + run: | + # Create logs directory + mkdir -p logs + + # Kill any existing port-forward on port 8000 + pkill -f "port-forward.*8000" || true + sleep 2 + + # Start port-forward in background + kubectl port-forward -n stackrox svc/central 8000:443 > logs/port-forward.log 2>&1 & + sleep 5 + + # Verify port-forward is working + if ! curl -k -s https://localhost:8000/v1/ping > /dev/null 2>&1; then + echo "Port-forward failed to start. Log:" + cat logs/port-forward.log || true + exit 1 + fi + echo "Port-forward established successfully" + + - name: Install go-junit-report + run: go install github.com/jstemmer/go-junit-report/v2@v2.1.0 + + - name: Run smoke tests with JUnit output + env: + ROX_ENDPOINT: localhost:8000 + ROX_PASSWORD: ${{ steps.extract-password.outputs.password }} + run: | + go test -v -tags=smoke -cover -race -coverprofile=coverage-smoke.out -timeout=20m ./smoke 2>&1 | \ + tee /dev/stderr | \ + go-junit-report -set-exit-code -out junit-smoke.xml + + - name: Upload JUnit test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: junit-smoke-results + path: junit-smoke.xml + if-no-files-found: error + + - name: Upload test results to Codecov + if: always() + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: junit-smoke.xml + + - name: Upload coverage to Codecov + if: always() + uses: codecov/codecov-action@v5 + with: + files: ./coverage-smoke.out + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: false + flags: smoke + name: smoke-tests + + - name: Collect logs + if: always() + run: | + mkdir -p logs + kubectl get pods -A > logs/pods.txt || true + kubectl get events -A --sort-by='.lastTimestamp' > logs/events.txt || true + kubectl logs -n stackrox deployment/central > logs/central.log || true + + - name: Upload logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: smoke-test-logs + path: logs/ + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index 0e487cd..cfd074a 100644 --- a/.gitignore +++ b/.gitignore @@ -10,9 +10,14 @@ # Test output /*.out /*junit.xml +/coverage-report.html # Build output /stackrox-mcp +/bin/ + +# Virtual environments +/ENV_DIR/ # Lint output /report.xml @@ -24,6 +29,7 @@ /e2e-tests/.env /e2e-tests/mcp-reports/ /e2e-tests/bin/ +/e2e-tests/**/mcpchecker /e2e-tests/**/*-out.json # WireMock diff --git a/Makefile b/Makefile index 6226b4a..011f318 100644 --- a/Makefile +++ b/Makefile @@ -78,7 +78,7 @@ e2e-test: ## Run E2E tests (uses WireMock) .PHONY: test-coverage-and-junit test-coverage-and-junit: ## Run unit tests with coverage and junit output go install github.com/jstemmer/go-junit-report/v2@v2.1.0 - $(GOTEST) -v -cover -race -coverprofile=$(COVERAGE_OUT) ./... 2>&1 | go-junit-report -set-exit-code -iocopy -out $(JUNIT_OUT) + $(GOTEST) -v -cover -race -coverprofile=$(COVERAGE_OUT) $(shell go list ./... | grep -v '/smoke$$') 2>&1 | go-junit-report -set-exit-code -iocopy -out $(JUNIT_OUT) .PHONY: test-integration-coverage test-integration-coverage: ## Run integration tests with coverage diff --git a/internal/testutil/integration_helpers.go b/internal/testutil/integration_helpers.go index 21b48e7..260f146 100644 --- a/internal/testutil/integration_helpers.go +++ b/internal/testutil/integration_helpers.go @@ -8,10 +8,8 @@ import ( "testing" "time" - "github.com/modelcontextprotocol/go-sdk/mcp" "github.com/stackrox/stackrox-mcp/internal/app" "github.com/stackrox/stackrox-mcp/internal/config" - "github.com/stretchr/testify/require" ) // CreateIntegrationTestConfig creates a test configuration for integration tests. @@ -55,37 +53,3 @@ func CreateIntegrationMCPClient(t *testing.T) (*MCPTestClient, error) { return NewMCPTestClient(t, runFunc) } - -// SetupInitializedClient creates an initialized MCP client for testing with automatic cleanup. -func SetupInitializedClient(t *testing.T, createClient func(*testing.T) (*MCPTestClient, error)) *MCPTestClient { - t.Helper() - - client, err := createClient(t) - require.NoError(t, err, "Failed to create MCP client") - t.Cleanup(func() { client.Close() }) - - return client -} - -// CallToolAndGetResult calls a tool and verifies it succeeds. -func CallToolAndGetResult(t *testing.T, client *MCPTestClient, toolName string, args map[string]any) *mcp.CallToolResult { - t.Helper() - - ctx := context.Background() - result, err := client.CallTool(ctx, toolName, args) - require.NoError(t, err) - RequireNoError(t, result) - - return result -} - -// GetTextContent extracts text from the first content item. -func GetTextContent(t *testing.T, result *mcp.CallToolResult) string { - t.Helper() - require.NotEmpty(t, result.Content, "should have content in response") - - textContent, ok := result.Content[0].(*mcp.TextContent) - require.True(t, ok, "expected TextContent, got %T", result.Content[0]) - - return textContent.Text -} diff --git a/internal/testutil/test_helpers.go b/internal/testutil/test_helpers.go new file mode 100644 index 0000000..5f995f5 --- /dev/null +++ b/internal/testutil/test_helpers.go @@ -0,0 +1,48 @@ +package testutil + +import ( + "context" + "testing" + + "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/stretchr/testify/require" +) + +// SetupInitializedClient creates an initialized MCP client with automatic cleanup. +func SetupInitializedClient(t *testing.T, createClient func(*testing.T) (*MCPTestClient, error)) *MCPTestClient { + t.Helper() + + client, err := createClient(t) + require.NoError(t, err, "Failed to create MCP client") + t.Cleanup(func() { _ = client.Close() }) + + return client +} + +// CallToolAndGetResult calls a tool and verifies it succeeds. +func CallToolAndGetResult( + t *testing.T, + client *MCPTestClient, + toolName string, + args map[string]any, +) *mcp.CallToolResult { + t.Helper() + + ctx := context.Background() + result, err := client.CallTool(ctx, toolName, args) + require.NoError(t, err) + RequireNoError(t, result) + + return result +} + +// GetTextContent extracts text from the first content item. +func GetTextContent(t *testing.T, result *mcp.CallToolResult) string { + t.Helper() + require.NotEmpty(t, result.Content, "should have content in response") + + textContent, ok := result.Content[0].(*mcp.TextContent) + require.True(t, ok, "expected TextContent, got %T", result.Content[0]) + + return textContent.Text +} diff --git a/smoke/central_helper_test.go b/smoke/central_helper_test.go new file mode 100644 index 0000000..0abe328 --- /dev/null +++ b/smoke/central_helper_test.go @@ -0,0 +1,160 @@ +//go:build smoke + +package smoke + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +const ( + tokenGenerationTimeout = 30 * time.Second + pingTimeout = 5 * time.Second +) + +type GenerateTokenRequest struct { + Name string `json:"name"` + Role string `json:"role,omitempty"` +} + +type GenerateTokenResponse struct { + Token string `json:"token"` +} + +type ClusterHealthResponse struct { + Clusters []struct { + HealthStatus struct { + OverallHealthStatus string `json:"overallHealthStatus"` + } `json:"healthStatus"` + } `json:"clusters"` +} + +// GenerateAPIToken generates an API token using basic authentication. +func GenerateAPIToken(t *testing.T, endpoint, password string) string { + t.Helper() + + tokenReq := GenerateTokenRequest{ + Name: "smoke-test-token", + Role: "Admin", + } + + reqBody, err := json.Marshal(tokenReq) + require.NoError(t, err, "Failed to marshal token request") + + url := fmt.Sprintf("https://%s/v1/apitokens/generate", endpoint) + + req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, url, bytes.NewReader(reqBody)) + require.NoError(t, err, "Failed to create request") + + req.SetBasicAuth("admin", password) + req.Header.Set("Content-Type", "application/json") + + client := &http.Client{ + Timeout: tokenGenerationTimeout, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec + }, + } + + resp, err := client.Do(req) + require.NoError(t, err, "Failed to make token generation request") + defer func() { _ = resp.Body.Close() }() + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err, "Failed to read response body") + + require.Equal(t, http.StatusOK, resp.StatusCode, "Token generation failed: %s", string(body)) + + var tokenResp GenerateTokenResponse + require.NoError(t, json.Unmarshal(body, &tokenResp), "Failed to parse token response") + require.NotEmpty(t, tokenResp.Token, "Received empty token in response") + + return tokenResp.Token +} + +// WaitForCentralReady waits for Central API to be ready by polling /v1/ping. +func WaitForCentralReady(t *testing.T, endpoint, password string) { + t.Helper() + + require.Eventually(t, func() bool { + return isCentralReady(endpoint, password) + }, 2*time.Minute, 2*time.Second, "Central API did not become ready") +} + +// isCentralReady checks if Central API responds to /v1/ping. +func isCentralReady(endpoint, password string) bool { + url := fmt.Sprintf("https://%s/v1/ping", endpoint) + + client := &http.Client{ + Timeout: pingTimeout, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec + }, + } + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) + if err != nil { + return false + } + + req.SetBasicAuth("admin", password) + + resp, err := client.Do(req) + if err != nil { + return false + } + defer func() { _ = resp.Body.Close() }() + + return resp.StatusCode == http.StatusOK +} + +// IsClusterHealthy checks if the first cluster registered with Central is in HEALTHY status. +func IsClusterHealthy(endpoint, password string) bool { + url := fmt.Sprintf("https://%s/v1/clusters", endpoint) + + client := &http.Client{ + Timeout: pingTimeout, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec + }, + } + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) + if err != nil { + return false + } + + req.SetBasicAuth("admin", password) + + resp, err := client.Do(req) + if err != nil { + return false + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return false + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return false + } + + var healthResp ClusterHealthResponse + if err := json.Unmarshal(body, &healthResp); err != nil { + return false + } + + return len(healthResp.Clusters) > 0 && + healthResp.Clusters[0].HealthStatus.OverallHealthStatus == "HEALTHY" +} diff --git a/smoke/smoke_test.go b/smoke/smoke_test.go new file mode 100644 index 0000000..a2bbe44 --- /dev/null +++ b/smoke/smoke_test.go @@ -0,0 +1,99 @@ +//go:build smoke + +package smoke + +import ( + "context" + "encoding/json" + "io" + "os" + "testing" + "time" + + "github.com/stackrox/stackrox-mcp/internal/app" + "github.com/stackrox/stackrox-mcp/internal/config" + "github.com/stackrox/stackrox-mcp/internal/testutil" + "github.com/stretchr/testify/require" +) + +func TestSmoke_RealCluster(t *testing.T) { + endpoint := os.Getenv("ROX_ENDPOINT") + apiToken := os.Getenv("ROX_API_TOKEN") + password := os.Getenv("ROX_PASSWORD") + + require.NotEmpty(t, endpoint, "ROX_ENDPOINT environment variable must be set") + + // Generate token if password provided but no token + if apiToken == "" && password != "" { + t.Log("No API token provided, generating one using password...") + + WaitForCentralReady(t, endpoint, password) + t.Log("Central API is ready") + + apiToken = GenerateAPIToken(t, endpoint, password) + t.Log("Successfully generated API token") + } + + require.NotEmpty(t, apiToken, "Either ROX_API_TOKEN or ROX_PASSWORD must be set") + + require.Eventually(t, func() bool { + healthy := IsClusterHealthy(endpoint, password) + if !healthy { + t.Log("Waiting for cluster to be registered and healthy...") + } + return healthy + }, 6*time.Minute, 2*time.Second, "Cluster did not become healthy") + t.Log("Cluster is healthy and ready for testing") + + client := createSmokeTestClient(t, endpoint, apiToken) + + result := testutil.CallToolAndGetResult(t, client, "list_clusters", map[string]any{}) + responseText := testutil.GetTextContent(t, result) + + var data struct { + Clusters []struct { + Name string `json:"name"` + } `json:"clusters"` + } + require.NoError(t, json.Unmarshal([]byte(responseText), &data)) + require.NotEmpty(t, data.Clusters, "should have at least one cluster") + t.Logf("Found %d cluster(s)", len(data.Clusters)) +} + +func createSmokeTestClient(t *testing.T, endpoint, apiToken string) *testutil.MCPTestClient { + t.Helper() + + cfg := &config.Config{ + Central: config.CentralConfig{ + URL: endpoint, + AuthType: "static", + APIToken: apiToken, + InsecureSkipTLSVerify: true, + RequestTimeout: 30 * time.Second, + MaxRetries: 3, + InitialBackoff: time.Second, + MaxBackoff: 10 * time.Second, + }, + Server: config.ServerConfig{ + Type: "stdio", + }, + Tools: config.ToolsConfig{ + Vulnerability: config.ToolsetVulnerabilityConfig{ + Enabled: true, + }, + ConfigManager: config.ToolConfigManagerConfig{ + Enabled: true, + }, + }, + } + + runFunc := func(ctx context.Context, stdin io.ReadCloser, stdout io.WriteCloser) error { + return app.Run(ctx, cfg, stdin, stdout) + } + + client, err := testutil.NewMCPTestClient(t, runFunc) + require.NoError(t, err, "Failed to create MCP client") + t.Cleanup(func() { client.Close() }) + + return client +}