From fd6209365663ef24dda943ac3c18634723522055 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Wed, 7 Oct 2026 09:39:06 -0500 Subject: [PATCH 1/6] fix(build): compile and run unit tests on darwin Cast statfs Bsize to int64 so pkg/monitors/system builds on macOS, where the field is uint32. Make the unit tests portable: short unix socket paths under the 104-byte limit, squat the wildcard address in the Prometheus bind-failure test, discover the loopback interface name, and skip tests that need 127.0.0.0/8 aliases or inotify rename semantics when the host lacks them. Remove the unused command-call tracking and testKernelPanicLog from the logpattern mocks, and revive the memory estimation assertions that were commented out. Closes #20 Co-Authored-By: Claude Fable 5.1 --- .goreleaser.yml | 2 - cmd/node-doctor/bind_ordering_test.go | 3 +- cmd/node-doctor/healthcheck_test.go | 14 ++++++- pkg/exporters/prometheus/exporter_test.go | 4 +- pkg/health/liveness_readiness_test.go | 3 +- pkg/health/server_test.go | 15 +++++++- pkg/monitors/custom/logpattern_mocks.go | 31 --------------- .../custom/logpattern_security_test.go | 38 +++++++++---------- pkg/monitors/network/clusterdnspod_test.go | 13 +++++++ pkg/monitors/network/cni_health_test.go | 31 +++++++++++---- pkg/monitors/system/disk.go | 2 +- pkg/reload/watcher_test.go | 4 ++ 12 files changed, 90 insertions(+), 70 deletions(-) diff --git a/.goreleaser.yml b/.goreleaser.yml index 282ff70..7b676b6 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -30,8 +30,6 @@ builds: flags: - -trimpath mod_timestamp: '{{.CommitTimestamp}}' - # Note: Darwin builds temporarily disabled due to cross-compilation type issues - # TODO: Fix pkg/monitors/system/disk.go:58 uint32->int64 conversion for darwin # Archive configuration archives: diff --git a/cmd/node-doctor/bind_ordering_test.go b/cmd/node-doctor/bind_ordering_test.go index c9f8d40..4e4aae1 100644 --- a/cmd/node-doctor/bind_ordering_test.go +++ b/cmd/node-doctor/bind_ordering_test.go @@ -5,7 +5,6 @@ import ( "go/ast" "go/parser" "go/token" - "path/filepath" "strings" "testing" "time" @@ -29,7 +28,7 @@ import ( // anyone reorders createExporters so networked init runs first, this test hangs // on an unservable socket and fails. func TestHealthEndpointServesBeforeNetworkedExporters(t *testing.T) { - socket := filepath.Join(t.TempDir(), "health.sock") + socket := testSocketPath(t) // Phase 2 blocks until we release it — standing in for a wedged exporter // Start() on a degraded node. diff --git a/cmd/node-doctor/healthcheck_test.go b/cmd/node-doctor/healthcheck_test.go index 6f91b64..2c10da8 100644 --- a/cmd/node-doctor/healthcheck_test.go +++ b/cmd/node-doctor/healthcheck_test.go @@ -2,17 +2,29 @@ package main import ( "context" + "os" "path/filepath" "testing" "github.com/supporttools/node-doctor/pkg/health" ) +// t.TempDir() paths can exceed the unix socket path limit (104 bytes on macOS). +func testSocketPath(t *testing.T) string { + t.Helper() + dir, err := os.MkdirTemp("", "nd") + if err != nil { + t.Fatalf("mkdir temp: %v", err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + return filepath.Join(dir, "health.sock") +} + // TestRunHealthCheck exercises the exec-probe health check against a real health // server unix socket. This is the code path the Kubernetes exec probes invoke, and // the reason a1pinode01-class hostPort-8080 conflicts no longer crashloop the pod. func TestRunHealthCheck(t *testing.T) { - sockPath := filepath.Join(t.TempDir(), "health.sock") + sockPath := testSocketPath(t) srv, err := health.NewServer(&health.Config{ Enabled: true, Port: 0, // ephemeral TCP, no conflict diff --git a/pkg/exporters/prometheus/exporter_test.go b/pkg/exporters/prometheus/exporter_test.go index 169a1da..d34faff 100644 --- a/pkg/exporters/prometheus/exporter_test.go +++ b/pkg/exporters/prometheus/exporter_test.go @@ -1118,8 +1118,8 @@ func TestExportProblemBeforeStart(t *testing.T) { } func TestPrometheusExporter_StartBindFailure(t *testing.T) { - // Occupy a port so the exporter cannot bind to it. - ln, err := net.Listen("tcp", "127.0.0.1:0") + // Squat on the wildcard: BSD lets a wildcard bind coexist with a 127.0.0.1 bind. + ln, err := net.Listen("tcp", ":0") if err != nil { t.Fatalf("failed to grab a free port: %v", err) } diff --git a/pkg/health/liveness_readiness_test.go b/pkg/health/liveness_readiness_test.go index 1384310..465ea85 100644 --- a/pkg/health/liveness_readiness_test.go +++ b/pkg/health/liveness_readiness_test.go @@ -6,7 +6,6 @@ import ( "errors" "net" "net/http" - "path/filepath" "testing" "time" @@ -20,7 +19,7 @@ import ( func newSocketServer(t *testing.T) (*Server, *http.Client) { t.Helper() - socket := filepath.Join(t.TempDir(), "health.sock") + socket := testSocketPath(t) srv, err := NewServer(&Config{ Enabled: true, BindAddress: "127.0.0.1", diff --git a/pkg/health/server_test.go b/pkg/health/server_test.go index f4729c0..cf97ae7 100644 --- a/pkg/health/server_test.go +++ b/pkg/health/server_test.go @@ -13,6 +13,17 @@ import ( "time" ) +// t.TempDir() paths can exceed the unix socket path limit (104 bytes on macOS). +func testSocketPath(t *testing.T) string { + t.Helper() + dir, err := os.MkdirTemp("", "nd") + if err != nil { + t.Fatalf("mkdir temp: %v", err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + return filepath.Join(dir, "health.sock") +} + // getOverUnixSocket issues an HTTP GET for path against a health server listening // on a unix domain socket, returning the status code. func getOverUnixSocket(t *testing.T, socketPath, path string) int { @@ -37,7 +48,7 @@ func getOverUnixSocket(t *testing.T, socketPath, path string) int { // TestServer_UnixSocket verifies the health endpoints are served over the per-pod // unix socket and that the socket file is cleaned up on Stop. func TestServer_UnixSocket(t *testing.T) { - sockPath := filepath.Join(t.TempDir(), "health.sock") + sockPath := testSocketPath(t) srv, err := NewServer(&Config{ Enabled: true, Port: 0, // ephemeral TCP — no conflict @@ -81,7 +92,7 @@ func TestServer_TCPConflictNonFatal(t *testing.T) { defer func() { _ = squatter.Close() }() occupiedPort := squatter.Addr().(*net.TCPAddr).Port - sockPath := filepath.Join(t.TempDir(), "health.sock") + sockPath := testSocketPath(t) srv, err := NewServer(&Config{ Enabled: true, BindAddress: "127.0.0.1", // non-dual-stack: bind fails outright on the taken port diff --git a/pkg/monitors/custom/logpattern_mocks.go b/pkg/monitors/custom/logpattern_mocks.go index 5432e4c..9b5e6f4 100644 --- a/pkg/monitors/custom/logpattern_mocks.go +++ b/pkg/monitors/custom/logpattern_mocks.go @@ -145,13 +145,6 @@ func (m *mockKmsgReader) ReadKmsg(ctx context.Context, path string) ([]byte, err type mockCommandExecutor struct { outputs map[string]string // command key -> output errors map[string]error // command key -> error - calls []commandCall // Track all calls for verification -} - -// commandCall records a command execution -type commandCall struct { - command string - args []string } // newMockCommandExecutor creates a new mock command executor @@ -159,7 +152,6 @@ func newMockCommandExecutor() *mockCommandExecutor { return &mockCommandExecutor{ outputs: make(map[string]string), errors: make(map[string]error), - calls: make([]commandCall, 0), } } @@ -177,12 +169,6 @@ func (m *mockCommandExecutor) setError(command string, args []string, err error) // ExecuteCommand implements CommandExecutor.ExecuteCommand func (m *mockCommandExecutor) ExecuteCommand(ctx context.Context, command string, args []string) ([]byte, error) { - // Record the call - m.calls = append(m.calls, commandCall{ - command: command, - args: args, - }) - key := m.makeKey(command, args) // Check for error first @@ -199,22 +185,6 @@ func (m *mockCommandExecutor) ExecuteCommand(ctx context.Context, command string return []byte{}, nil } -// getCalls returns all recorded command calls -func (m *mockCommandExecutor) getCalls() []commandCall { - return m.calls -} - -// wasCalled checks if a command was called with specific args -func (m *mockCommandExecutor) wasCalled(command string, args []string) bool { - key := m.makeKey(command, args) - for _, call := range m.calls { - if m.makeKey(call.command, call.args) == key { - return true - } - } - return false -} - // makeKey creates a unique key for a command and its args func (m *mockCommandExecutor) makeKey(command string, args []string) string { return command + " " + strings.Join(args, " ") @@ -235,7 +205,6 @@ const ( testOOMKillLog = "<4>[12345.678901] Out of memory: Kill process 1234 (test-process) score 1000" testDiskErrorLog = "<3>[12346.789012] EXT4-fs (sda1): error count: 5" testNetworkErrorLog = "<4>[12347.890123] NETDEV WATCHDOG: eth0 (e1000e): transmit queue 0 timed out" - testKernelPanicLog = "<0>[12348.901234] Kernel panic - not syncing: VFS: Unable to mount root fs" testNormalLog = "<6>[12349.012345] systemd[1]: Started User Manager for UID 1000." testKubeletErrorLog = "kubelet: Failed to initialize CSI driver" testContainerdLog = "containerd: error creating container" diff --git a/pkg/monitors/custom/logpattern_security_test.go b/pkg/monitors/custom/logpattern_security_test.go index 111c2a5..1fc3a8d 100644 --- a/pkg/monitors/custom/logpattern_security_test.go +++ b/pkg/monitors/custom/logpattern_security_test.go @@ -781,32 +781,30 @@ func TestSecurity_MemoryEstimation_Accuracy(t *testing.T) { units[i] = "test-" + string(rune(i)) + ".service" } - _ = &LogPatternMonitorConfig{ + config := &LogPatternMonitorConfig{ Patterns: patterns, JournalUnits: units, MaxEventsPerPattern: cfg.events, DedupWindow: 5 * time.Minute, } - // estimate := estimateMemoryUsage(config) - // - // // Estimates should scale with config size - // expectedMin := cfg.patterns * 1024 // At least 1KB per pattern - // if estimate < expectedMin { - // t.Errorf("Estimate too low: got %d bytes, expected at least %d", - // estimate, expectedMin) - // } - // - // // Estimates should be conservative (reasonable upper bound) - // expectedMax := 50 * 1024 * 1024 // 50MB max - // if estimate > expectedMax { - // t.Errorf("Estimate too high: got %d bytes, expected at most %d", - // estimate, expectedMax) - // } - - // t.Logf("%s config: %d patterns, %d events, %d units = %.2f MB estimated", - // cfg.name, cfg.patterns, cfg.events, cfg.units, - // float64(estimate)/(1024*1024)) + estimate := config.estimateMemoryUsage() + + expectedMin := cfg.patterns * 1024 + if estimate < expectedMin { + t.Errorf("Estimate too low: got %d bytes, expected at least %d", + estimate, expectedMin) + } + + expectedMax := 50 * 1024 * 1024 + if estimate > expectedMax { + t.Errorf("Estimate too high: got %d bytes, expected at most %d", + estimate, expectedMax) + } + + t.Logf("%s config: %d patterns, %d events, %d units = %.2f MB estimated", + cfg.name, cfg.patterns, cfg.events, cfg.units, + float64(estimate)/(1024*1024)) }) } } diff --git a/pkg/monitors/network/clusterdnspod_test.go b/pkg/monitors/network/clusterdnspod_test.go index 2218635..1274405 100644 --- a/pkg/monitors/network/clusterdnspod_test.go +++ b/pkg/monitors/network/clusterdnspod_test.go @@ -22,6 +22,7 @@ import ( // with no registered result respond 404, simulating an unreachable/errored pod. func startFakeOverlayServers(t *testing.T, hostToResult map[string]clusterdns.ProbeResult) (port int, cleanup func()) { t.Helper() + requireLoopbackAliases(t, hostToResult) mux := http.NewServeMux() mux.HandleFunc("/clusterdns", func(w http.ResponseWriter, r *http.Request) { @@ -56,6 +57,18 @@ func startFakeOverlayServers(t *testing.T, hostToResult map[string]clusterdns.Pr return tcpAddr.Port, ts.Close } +// Linux routes all of 127.0.0.0/8 to lo; macOS only configures 127.0.0.1. +func requireLoopbackAliases(t *testing.T, hostToResult map[string]clusterdns.ProbeResult) { + t.Helper() + for host := range hostToResult { + ln, err := net.Listen("tcp", net.JoinHostPort(host, "0")) + if err != nil { + t.Skipf("loopback alias %s not available: %v", host, err) + } + _ = ln.Close() + } +} + func newTestClusterDNSPodMonitor(port int, peers []Peer, opts func(*ClusterDNSPodConfig)) *ClusterDNSPodMonitor { cfg := &ClusterDNSPodConfig{ ProbePort: port, diff --git a/pkg/monitors/network/cni_health_test.go b/pkg/monitors/network/cni_health_test.go index 4d6543b..f793995 100644 --- a/pkg/monitors/network/cni_health_test.go +++ b/pkg/monitors/network/cni_health_test.go @@ -1,6 +1,7 @@ package network import ( + "net" "os" "path/filepath" "testing" @@ -793,9 +794,25 @@ func TestCheckHealth(t *testing.T) { } } +func loopbackInterfaceName(t *testing.T) string { + t.Helper() + ifaces, err := net.Interfaces() + if err != nil { + t.Fatalf("net.Interfaces: %v", err) + } + for _, iface := range ifaces { + if iface.Flags&net.FlagLoopback != 0 { + return iface.Name + } + } + t.Fatal("no loopback interface found") + return "" +} + func TestCheckInterfaces(t *testing.T) { // This test relies on actual system interfaces, so we test behavior // rather than specific interface names + lo := loopbackInterfaceName(t) t.Run("no expected interfaces", func(t *testing.T) { checker := NewCNIHealthChecker(CNIHealthConfig{ @@ -834,20 +851,20 @@ func TestCheckInterfaces(t *testing.T) { } }) - t.Run("lo interface exists on all systems", func(t *testing.T) { + t.Run("loopback interface exists on all systems", func(t *testing.T) { checker := NewCNIHealthChecker(CNIHealthConfig{ ConfigPath: t.TempDir(), CheckInterfaces: true, - ExpectedInterfaces: []string{"lo"}, + ExpectedInterfaces: []string{lo}, }) result := checker.CheckInterfaces() if !result.Healthy { - t.Errorf("Should be healthy when 'lo' interface exists. Errors: %v", result.Errors) + t.Errorf("Should be healthy when %q interface exists. Errors: %v", lo, result.Errors) } - if len(result.FoundInterfaces) != 1 || result.FoundInterfaces[0] != "lo" { - t.Errorf("Expected 'lo' in FoundInterfaces, got %v", result.FoundInterfaces) + if len(result.FoundInterfaces) != 1 || result.FoundInterfaces[0] != lo { + t.Errorf("Expected %q in FoundInterfaces, got %v", lo, result.FoundInterfaces) } }) @@ -855,7 +872,7 @@ func TestCheckInterfaces(t *testing.T) { checker := NewCNIHealthChecker(CNIHealthConfig{ ConfigPath: t.TempDir(), CheckInterfaces: true, - ExpectedInterfaces: []string{"lo", "nonexistent-xyz123"}, + ExpectedInterfaces: []string{lo, "nonexistent-xyz123"}, }) result := checker.CheckInterfaces() @@ -864,7 +881,7 @@ func TestCheckInterfaces(t *testing.T) { t.Error("Should be unhealthy when some expected interfaces are missing") } if len(result.FoundInterfaces) != 1 { - t.Errorf("FoundInterfaces should have 1 entry (lo), got %v", result.FoundInterfaces) + t.Errorf("FoundInterfaces should have 1 entry (%s), got %v", lo, result.FoundInterfaces) } if len(result.MissingInterfaces) != 1 { t.Errorf("MissingInterfaces should have 1 entry, got %v", result.MissingInterfaces) diff --git a/pkg/monitors/system/disk.go b/pkg/monitors/system/disk.go index 27c7c69..b42ae3b 100644 --- a/pkg/monitors/system/disk.go +++ b/pkg/monitors/system/disk.go @@ -84,7 +84,7 @@ func (p *defaultStatfsProvider) Statfs(path string) (*FilesystemStats, error) { BlocksAvail: stat.Bavail, Files: stat.Files, FilesFree: stat.Ffree, - BlockSize: stat.Bsize, + BlockSize: int64(stat.Bsize), //nolint:unconvert // Bsize is uint32 on darwin Flags: uint64(stat.Flags), }, nil } diff --git a/pkg/reload/watcher_test.go b/pkg/reload/watcher_test.go index e5dd2f7..8bee39e 100644 --- a/pkg/reload/watcher_test.go +++ b/pkg/reload/watcher_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "runtime" "testing" "time" ) @@ -215,6 +216,9 @@ func TestConfigWatcherDebouncing(t *testing.T) { // TestConfigWatcherKubernetesSymlink tests handling of Kubernetes ConfigMap atomic updates func TestConfigWatcherKubernetesSymlink(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("rename-over of an existing symlink only surfaces as a Create event with inotify") + } // This test simulates how Kubernetes updates ConfigMaps: // 1. Creates new directory with timestamp // 2. Writes files to new directory From 259de4cf62c093b588c77fa57fbbd9e0a7996e09 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Wed, 7 Oct 2026 09:39:06 -0500 Subject: [PATCH 2/6] docs: remove template leftovers and rebuild the docs index Delete process docs copied from another project's template (docs/development/*, docs/deployment.md, .githooks/README.md) and replace docs/README.md with an index of the docs that exist, grouped for operators and developers. Repoint the links that targeted the deleted files and add scripts/check-doc-links.sh to catch broken relative links. Co-Authored-By: Claude Fable 5.1 --- .githooks/README.md | 220 ---- CONTRIBUTING.md | 3 +- README.md | 2 +- docs/README.md | 252 +--- docs/deployment.md | 432 ------- .../local-cicd-validation-guide.md | 734 ------------ docs/development/master-task-management.md | 275 ----- docs/development/task-execution-workflow.md | 1018 ----------------- .../development/validation-quick-reference.md | 445 ------- docs/release-process.md | 2 +- scripts/check-doc-links.sh | 21 + 11 files changed, 60 insertions(+), 3344 deletions(-) delete mode 100755 .githooks/README.md delete mode 100644 docs/deployment.md delete mode 100644 docs/development/local-cicd-validation-guide.md delete mode 100644 docs/development/master-task-management.md delete mode 100644 docs/development/task-execution-workflow.md delete mode 100644 docs/development/validation-quick-reference.md create mode 100755 scripts/check-doc-links.sh diff --git a/.githooks/README.md b/.githooks/README.md deleted file mode 100755 index a732dff..0000000 --- a/.githooks/README.md +++ /dev/null @@ -1,220 +0,0 @@ -# Git Hooks for Nexmonyx - -This directory contains Git hooks that help maintain code quality by running validations before certain Git operations. - -## Available Hooks - -### pre-push - -Runs comprehensive validation before pushing to remote: -- Code formatting (gofmt) -- Static analysis (go vet) -- Advanced linting (staticcheck) -- Security scanning (gosec) -- Unit tests with race detection - -This prevents CI/CD failures by catching issues locally before they reach the pipeline. - -## Setup - -### Quick Setup (Recommended) - -Run the setup script: - -```bash -./.githooks/setup.sh -``` - -This will: -1. Configure Git to use `.githooks` directory -2. Enable all available hooks -3. Show current configuration - -### Manual Setup - -Configure Git to use this hooks directory: - -```bash -git config core.hooksPath .githooks -``` - -Verify configuration: - -```bash -git config --get core.hooksPath -# Should output: .githooks -``` - -## Usage - -Once enabled, hooks run automatically: - -```bash -# Pre-push hook runs automatically -git push origin main -``` - -### Bypassing Hooks - -**Warning:** Only bypass hooks when absolutely necessary (e.g., emergency hotfix). - -```bash -# Bypass pre-push validation -git push --no-verify - -# Or bypass all hooks for one commit -git commit --no-verify -m "Emergency fix" -``` - -## Disabling Hooks - -Temporarily disable: - -```bash -git config --unset core.hooksPath -``` - -Re-enable: - -```bash -git config core.hooksPath .githooks -``` - -## Hook Details - -### pre-push Hook - -**What it validates:** -- All 6 components: api, org-management-controller, probe-controller, ingestor, customer-probe-agent, pkg -- 5 checks per component: format, vet, staticcheck, gosec, tests - -**When it runs:** -- Before every `git push` -- After `git push` command, before sending to remote - -**What happens on failure:** -- Push is blocked -- Detailed error output shown -- Instructions provided to fix or bypass - -**Performance:** -- Quick mode available: Only runs format, vet, staticcheck (skip tests) -- Full validation: ~2-5 minutes depending on changes -- Parallel execution: All components validated simultaneously - -## Customization - -### Run Quick Validation Only - -Edit `.githooks/pre-push` and change: - -```bash -./scripts/validate-pipeline-local.sh -``` - -To: - -```bash -./scripts/validate-pipeline-local.sh --quick -``` - -### Validate Specific Components - -Edit `.githooks/pre-push` to add component filtering: - -```bash -# Only validate changed components -CHANGED_COMPONENTS=$(git diff --name-only HEAD @{u} | grep -E '^(api|pkg)/' | cut -d'/' -f1 | sort -u) -for component in $CHANGED_COMPONENTS; do - ./scripts/validate-pipeline-local.sh --component=$component -done -``` - -## Troubleshooting - -### Hook Not Running - -Check configuration: - -```bash -git config --get core.hooksPath -``` - -Verify hook is executable: - -```bash -ls -la .githooks/pre-push -# Should show: -rwxr-xr-x -``` - -Make executable if needed: - -```bash -chmod +x .githooks/pre-push -``` - -### Hook Failing Unexpectedly - -Test hook manually: - -```bash -./.githooks/pre-push -``` - -Run validation script directly: - -```bash -./scripts/validate-pipeline-local.sh --quick -``` - -### Performance Issues - -Use quick mode or component-specific validation: - -```bash -# Quick validation (no tests) -./scripts/validate-pipeline-local.sh --quick - -# Specific component -./scripts/validate-pipeline-local.sh --component=api -``` - -## Best Practices - -1. **Enable hooks for all developers** - - Prevents CI/CD failures - - Faster feedback loop - - Consistent code quality - -2. **Run quick validation frequently** - ```bash - make validate-quick - ``` - -3. **Run full validation before push** - ```bash - make validate-pre-push - ``` - -4. **Only bypass when necessary** - - Emergency hotfixes - - Infrastructure-only changes - - Non-code changes - -5. **Keep hooks updated** - - Pull latest `.githooks` directory - - Re-run setup if hooks change - -## Related Documentation - -- [Local CI/CD Validation Guide](../docs/development/local-cicd-validation-guide.md) -- [Validation Quick Reference](../docs/development/validation-quick-reference.md) *(coming soon)* -- [Task Execution Workflow](../docs/development/task-execution-workflow.md) - -## Support - -For issues or questions: -- Check troubleshooting section above -- Review validation guide documentation -- Test validation script manually -- Check CI/CD pipeline for comparison diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c8d713b..635c5d3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -677,8 +677,7 @@ When contributing to Node Doctor: - **[Architecture Guide](docs/architecture.md)** - System design and component overview - **[Monitors Guide](docs/monitors.md)** - Monitor implementation details - **[Remediation Guide](docs/remediation.md)** - Remediator patterns and safety mechanisms -- **[Task Execution Workflow](docs/development/task-execution-workflow.md)** - Development process -- **[Validation Quick Reference](docs/development/validation-quick-reference.md)** - Testing and validation +- **[Testing Guide](docs/testing-guide.md)** - Testing and validation ### Build Targets diff --git a/README.md b/README.md index 29f5369..321bd3c 100644 --- a/README.md +++ b/README.md @@ -513,7 +513,7 @@ node-doctor/ - [Monitors](docs/monitors.md) - Health check monitor implementations - [Remediation](docs/remediation.md) - Auto-remediation system - [Configuration](docs/configuration.md) - Configuration reference -- [Deployment](docs/deployment.md) - Deployment guide for production clusters +- [Deployment](deployment/README.md) - Deployment guide for production clusters - [Release Process](docs/release-process.md) - Release management and versioning - [Troubleshooting](docs/troubleshooting.md) - Common issues and debugging guide diff --git a/docs/README.md b/docs/README.md index 01f98f8..9f5e647 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,216 +1,36 @@ -# Documentation Hub - -Central navigation for all project documentation. - -## 🚀 Quick Start - -New to the project? Start here: - -1. **[Quick Start Guide](quick-start.md)** - Deploy Node Doctor, import dashboards, configure monitors -2. **[Task Execution Workflow](development/task-execution-workflow.md)** - Mandatory development process -3. **[Template README](../README.md)** - Template usage guide - -## 📚 Core Documentation - -### Development Standards -**Location**: `docs/development/` - -Essential reading for all developers: - -- **[Task Execution Workflow](development/task-execution-workflow.md)** ⭐ MANDATORY - 8-step process -- **[Master Task Management](development/master-task-management.md)** ⭐ Priority system (P1-P5) -- **[TaskForge Usage Guide](development/taskforge-usage-guide.md)** - Project management system -- **[API Handler Architecture](development/api-handler-architecture-standards.md)** - Handler patterns -- **[Helm Chart Standards](development/helm-chart-standards.md)** - Kubernetes deployment -- **[Local CI/CD Validation](development/local-cicd-validation-guide.md)** - Pre-push validation -- **[Database Patterns Guide](development/database-patterns-guide.md)** - Multi-tenant vs single-tenant -- **[Error Response Standard](development/error-response-standard.md)** - API error handling -- **[Web UI-API Synchronization](development/web-ui-api-synchronization.md)** - Frontend sync -- **[Go SDK Synchronization](development/go-sdk-synchronization-guide.md)** - SDK updates - -### Workflows -**Location**: `docs/workflows/` - -Standard process workflows: - -- **[Feature Development](workflows/feature-development.md)** ⭐ Complete feature lifecycle -- **[Project Initialization](workflows/project-initialization.md)** - Project setup -- **[PRD Creation](workflows/prd-creation.md)** - Requirements documents -- **[Sprint Management](workflows/sprint-management.md)** - Agile processes -- **[Deployment](workflows/deployment.md)** - Release procedures - -## đŸŽ¯ By Role - -### For Developers -1. [Task Execution Workflow](development/task-execution-workflow.md) -2. [API Handler Architecture](development/api-handler-architecture-standards.md) -3. [Local CI/CD Validation](development/local-cicd-validation-guide.md) -4. [Feature Development Workflow](workflows/feature-development.md) - -### For DevOps/SRE -1. [Helm Chart Standards](development/helm-chart-standards.md) -2. [Deployment Workflow](workflows/deployment.md) -3. [Local CI/CD Validation](development/local-cicd-validation-guide.md) - -### For Architects -1. [Database Patterns Guide](development/database-patterns-guide.md) -2. [Architecture Review Workflow](workflows/architecture-review.md) -3. [Design Documentation](design/) - -### For Product/QA -1. [PRD Creation Workflow](workflows/prd-creation.md) -2. [Testing Strategy](workflows/testing-strategy.md) -3. [TaskForge Usage Guide](development/taskforge-usage-guide.md) - -### For Project Managers -1. [Master Task Management](development/master-task-management.md) -2. [TaskForge Usage Guide](development/taskforge-usage-guide.md) -3. [Sprint Management Workflow](workflows/sprint-management.md) - -## 📂 Directory Structure - -``` -docs/ -├── README.md # This file - navigation hub -├── development/ # Core standards and guides (30 files) -│ ├── task-execution-workflow.md -│ ├── master-task-management.md -│ ├── taskforge-usage-guide.md -│ ├── api-handler-architecture-standards.md -│ ├── helm-chart-standards.md -│ └── ... (see development/ for full list) -├── workflows/ # Process workflows -│ ├── feature-development.md -│ ├── project-initialization.md -│ ├── prd-creation.md -│ └── ... -├── design/ # Architecture and design specs -├── operations/ # Deployment and operations guides -├── testing/ # Testing strategies and guides -├── security/ # Security policies and procedures -├── examples/ # Configuration examples -└── archive/ # Historical documentation -``` - -## 🔍 Finding Documentation - -### By Topic - -- **Task Management**: `development/master-task-management.md`, `development/taskforge-usage-guide.md` -- **Quality Assurance**: `development/task-execution-workflow.md` (quality gates and QA process) -- **API Development**: `development/api-handler-architecture-standards.md`, `development/error-response-standard.md` -- **Database**: `development/database-patterns-guide.md` -- **Deployment**: `development/helm-chart-standards.md`, `workflows/deployment.md` -- **Testing**: `development/local-cicd-validation-guide.md`, `workflows/testing-strategy.md` -- **Frontend**: `development/web-ui-api-synchronization.md` -- **Operations**: `troubleshooting.md` - Common issues, debugging, performance tuning - -### By Question - -- **"How do I deploy Node Doctor?"** → [Quick Start Guide](quick-start.md) -- **"How do I import Grafana dashboards?"** → [Quick Start Guide](quick-start.md#import-grafana-dashboards) -- **"How do I start a new feature?"** → [Feature Development Workflow](workflows/feature-development.md) -- **"What's the development process?"** → [Task Execution Workflow](development/task-execution-workflow.md) -- **"How do I validate before pushing?"** → [Local CI/CD Validation](development/local-cicd-validation-guide.md) -- **"How do I deploy?"** → [Deployment Workflow](workflows/deployment.md) -- **"What are the API standards?"** → [API Handler Architecture](development/api-handler-architecture-standards.md) -- **"How do I write Helm charts?"** → [Helm Chart Standards](development/helm-chart-standards.md) -- **"How do tasks work?"** → [Master Task Management](development/master-task-management.md) -- **"Why isn't my monitor running?"** → [Troubleshooting Guide](troubleshooting.md) -- **"How do I debug issues?"** → [Troubleshooting Guide](troubleshooting.md) - -## 🎓 Learning Path - -### Week 1: Foundation -1. Read Task Execution Workflow -2. Read Master Task Management -3. Try a small task following the workflow - -### Week 2: Standards -1. Read API Handler Architecture Standards -2. Read Helm Chart Standards -3. Read Local CI/CD Validation Guide -4. Practice validation locally - -### Week 3: Workflows -1. Read Feature Development Workflow -2. Read Deployment Workflow -3. Complete a feature following workflows -4. Get QA and Devils Advocate review - -### Week 4: Mastery -1. Review all skeptic agents -2. Understand quality gates deeply -3. Mentor others on the process -4. Contribute documentation improvements - -## 📝 Documentation Standards - -### Creating Documentation - -1. **Use Markdown**: GitHub-flavored markdown -2. **Clear Structure**: Headings, lists, code blocks -3. **Cross-Reference**: Link to related docs -4. **Examples**: Include practical examples -5. **Update Date**: Add "Last Updated" footer - -### Organizing Documentation - -- **development/**: Standards, guides, and patterns -- **workflows/**: Step-by-step process documentation -- **design/**: Architecture and design decisions -- **operations/**: Deployment and operational guides -- **examples/**: Configuration examples -- **archive/**: Historical documentation (not current) - -### Maintaining Documentation - -- Review quarterly for accuracy -- Update when processes change -- Archive obsolete documentation -- Keep examples current -- Test documented procedures - -## 🔄 Documentation Updates - -This documentation is living and should evolve with the project: - -- **Add**: New standards, workflows, or guides -- **Update**: When processes change -- **Archive**: When practices become obsolete -- **Link**: Cross-reference related documentation -- **Example**: Add real-world examples - -## 💡 Contributing - -When adding documentation: - -1. Determine correct directory -2. Use clear, descriptive filename -3. Follow markdown standards -4. Add cross-references -5. Update this README if needed -6. Include examples -7. Test instructions work - -## ❓ Getting Help - -- **Can't find documentation?** Check this README or search repository -- **Documentation unclear?** Open an issue or ask the team -- **Want to contribute?** Follow contribution guidelines - -## đŸŽ¯ Remember - -**Documentation is code**. It should be: -- Version controlled -- Peer reviewed -- Tested for accuracy -- Updated regularly -- Easy to find and navigate - -Good documentation saves hours of confusion and prevents mistakes. - ---- - -**Last Updated**: 2025-10-29 -**Maintained By**: Development Team +# Node Doctor Documentation + +Index of the docs in this repository. Pick the section for your role. + +## Operators + +Deploying, configuring, and running Node Doctor. + +- [Quick Start](quick-start.md) - Install Node Doctor and import the dashboards. +- [Helm Chart](../helm/node-doctor/README.md) - Chart values and install options. +- [DaemonSet Deployment](../deployment/README.md) - Deploy with plain manifests. +- [Configuration Reference](configuration.md) - Every config option and its default. +- [Monitors](monitors.md) - The health check monitors and their settings. +- [DNS Monitor: Advanced Features](monitors/dns-advanced.md) - Extra DNS monitor options. +- [DNS Monitor: Troubleshooting](monitors/dns-troubleshooting.md) - Diagnose DNS monitor problems. +- [Remediation](remediation.md) - Auto-remediation and its safety controls. +- [Controller Deployment](controller-deployment.md) - Deploy the Node Doctor controller. +- [Troubleshooting](troubleshooting.md) - Common problems and how to debug them. +- [Grafana Dashboards](../dashboards/README.md) - Dashboard files and how to import them. +- [Example Configurations](../config/examples/README.md) - Sample config files. + +## Developers + +Building, testing, and releasing Node Doctor. + +- [Architecture](architecture.md) - Components and how they fit together. +- [Developer Testing Guide](testing-guide.md) - Test patterns and how to run each suite. +- [Testing](testing.md) - Test organization and commands. +- [Release Process](release-process.md) - Versioning, tagging, and publishing a release. +- [Contributing](../CONTRIBUTING.md) - Workflow, coding standards, and PR process. +- [Security Policy](../SECURITY.md) - How to report a vulnerability. +- [E2E Tests](../test/e2e/README.md) - Run the end-to-end suite against a cluster. +- [Integration Tests](../test/integration/README.md) - Run the integration suite. +- [GitHub Actions Workflows](../.github/workflows/README.md) - What each CI workflow does. + +Run `scripts/check-doc-links.sh` to check relative links in every markdown file. diff --git a/docs/deployment.md b/docs/deployment.md deleted file mode 100644 index 06977ec..0000000 --- a/docs/deployment.md +++ /dev/null @@ -1,432 +0,0 @@ -# Node Doctor - Real Cluster Deployment Guide - -This guide walks through deploying Node Doctor to a real Kubernetes cluster. - -## Prerequisites - -- Kubernetes cluster (1.19+) with kubectl access -- Docker installed for building images (for manual deployment) -- Access to Docker Hub registry (for manual deployment) -- Cluster admin privileges (for RBAC and DaemonSet deployment) - -## Deployment Options - -| Method | Recommended For | Description | -|--------|----------------|-------------| -| **Helm Chart** | Production | Easiest installation with configurable values | -| **Automated RC** | Development/Testing | One-command build and deploy for RC releases | -| **Manual** | Custom/Advanced | Full control over each deployment step | - ---- - -## Option 1: Helm Chart Deployment (Recommended) - -The Helm chart is the recommended method for deploying Node Doctor to production clusters. - -### Add the Helm Repository - -```bash -helm repo add supporttools https://charts.support.tools -helm repo update -``` - -### Basic Installation - -```bash -helm install node-doctor supporttools/node-doctor \ - --namespace node-doctor \ - --create-namespace -``` - -### Installation with Custom Values - -```bash -# Create custom values file -cat > custom-values.yaml << 'EOF' -settings: - logLevel: info - logFormat: json - updateInterval: 30s - enableRemediation: true - dryRunMode: false - -resources: - requests: - cpu: 50m - memory: 128Mi - limits: - cpu: 500m - memory: 256Mi - -serviceMonitor: - enabled: true - interval: 30s -EOF - -# Install with custom values -helm install node-doctor supporttools/node-doctor \ - --namespace node-doctor \ - --create-namespace \ - -f custom-values.yaml -``` - -### Verify Helm Deployment - -```bash -# Check DaemonSet status -kubectl get daemonset -n node-doctor node-doctor - -# View pods -kubectl get pods -n node-doctor -l app.kubernetes.io/name=node-doctor - -# Check logs -kubectl logs -n node-doctor -l app.kubernetes.io/name=node-doctor --tail=50 - -# View node conditions -kubectl get nodes -o custom-columns='NAME:.metadata.name,HEALTHY:.status.conditions[?(@.type=="NodeDoctorHealthy")].status' -``` - -### Helm Upgrade and Rollback - -```bash -# Upgrade to latest version -helm repo update -helm upgrade node-doctor supporttools/node-doctor -n node-doctor - -# Upgrade with new values -helm upgrade node-doctor supporttools/node-doctor \ - -n node-doctor \ - --set settings.logLevel=debug - -# Rollback to previous release -helm rollback node-doctor -n node-doctor -``` - -### Uninstall via Helm - -```bash -helm uninstall node-doctor -n node-doctor -kubectl delete namespace node-doctor -``` - -For complete Helm chart configuration options, see [../helm/node-doctor/README.md](../helm/node-doctor/README.md). - ---- - -## Option 2: Automated RC Deployment (Development/Testing) - -For rapid deployment of release candidates to the `a1-ops-prd` cluster: - -```bash -# One command to build, push, and deploy -make bump-rc -``` - -This single command will: -1. ✅ Validate the pipeline (run all tests) -2. ✅ Increment the RC version (e.g., v0.1.0-rc.1 → v0.1.0-rc.2) -3. ✅ Build Docker image with RC tag -4. ✅ Push to Docker Hub registry: `supporttools/node-doctor` -5. ✅ Deploy to `a1-ops-prd` cluster in `node-doctor` namespace -6. ✅ Commit and tag the release - -**Configuration:** -- Cluster: `a1-ops-prd` (from `~/.kube/config`) -- Namespace: `node-doctor` (auto-created if needed) -- Registry: `docker.io/supporttools/node-doctor` -- Version tracking: `.version-rc` file - -**Verify Deployment:** -```bash -# Check pod status -kubectl --context=a1-ops-prd -n node-doctor get pods -l app=node-doctor - -# View logs -kubectl --context=a1-ops-prd -n node-doctor logs -l app=node-doctor - -# Monitor health -kubectl --context=a1-ops-prd -n node-doctor get pods -l app=node-doctor -w -``` - ---- - -## Option 3: Manual Deployment (Advanced) - -If you prefer manual control over each step, follow the detailed instructions below. - -### Step 1: Build and Push Container Image - -```bash -# Build the Docker image with version tag -docker build -t supporttools/node-doctor:v0.1.0 \ - --build-arg VERSION=v0.1.0 \ - --build-arg GIT_COMMIT=$(git rev-parse --short HEAD) \ - --build-arg BUILD_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ) \ - . - -# Tag as latest -docker tag supporttools/node-doctor:v0.1.0 \ - supporttools/node-doctor:latest - -# Login to Docker Hub (if not already logged in) -docker login - -# Push both tags -docker push supporttools/node-doctor:v0.1.0 -docker push supporttools/node-doctor:latest -``` - -**Image Details:** -- Size: ~85 MB (optimized multi-stage Alpine build) -- Base: Alpine Linux 3.19 -- Go: 1.24.4 -- Binary location: `/usr/local/bin/node-doctor` - -## Step 2: Deploy to Kubernetes Cluster - -### 2.1 Deploy RBAC Resources (MUST BE FIRST) - -```bash -# Apply RBAC: ServiceAccount, ClusterRole, ClusterRoleBinding -kubectl apply -f deployment/rbac.yaml - -# Verify RBAC resources created -kubectl get serviceaccount -n kube-system node-doctor -kubectl get clusterrole node-doctor -kubectl get clusterrolebinding node-doctor -``` - -### 2.2 Deploy Node Doctor DaemonSet - -```bash -# Apply DaemonSet (includes ConfigMap and Service) -kubectl apply -f deployment/daemonset.yaml - -# Watch rollout -kubectl rollout status daemonset/node-doctor -n kube-system - -# Verify pods running on all nodes -kubectl get pods -n kube-system -l app=node-doctor -o wide -``` - -### 2.3 Quick Deployment (Both Files) - -```bash -# Apply both in correct order -kubectl apply -f deployment/rbac.yaml -f deployment/daemonset.yaml -``` - -## Step 3: Verify Deployment - -### 3.1 Check Pod Status - -```bash -# View all Node Doctor pods -kubectl get pods -n kube-system -l app=node-doctor - -# Check pod logs -POD_NAME=$(kubectl get pods -n kube-system -l app=node-doctor -o jsonpath='{.items[0].metadata.name}') -kubectl logs -n kube-system $POD_NAME --tail=50 -f -``` - -### 3.2 Test Health Endpoints - -```bash -# Test health endpoint -kubectl exec -n kube-system $POD_NAME -- curl -s localhost:8080/healthz - -# Test readiness -kubectl exec -n kube-system $POD_NAME -- curl -s localhost:8080/ready - -# Test detailed status -kubectl exec -n kube-system $POD_NAME -- curl -s localhost:8080/status | jq . -``` - -### 3.3 Check Metrics - -```bash -# View Prometheus metrics -kubectl exec -n kube-system $POD_NAME -- curl -s localhost:9100/metrics | head -20 - -# Check for Node Doctor specific metrics -kubectl exec -n kube-system $POD_NAME -- curl -s localhost:9100/metrics | grep node_doctor -``` - -### 3.4 Verify Node Conditions - -```bash -# Check if NodeDoctorHealthy condition is set -kubectl get nodes -o json | jq '.items[].status.conditions[] | select(.type == "NodeDoctorHealthy")' - -# View full node conditions -kubectl describe nodes | grep -A 10 "Conditions:" -``` - -### 3.5 Check Kubernetes Events - -```bash -# View Node Doctor events -kubectl get events -n kube-system --field-selector involvedObject.name=node-doctor --sort-by='.lastTimestamp' -``` - -## Step 4: Run Validation Script - -```bash -# Run comprehensive validation -cd /home/mmattox/go/src/github.com/supporttools/node-doctor -bash deployment/validate.sh -``` - -**Expected Output:** -- ✅ kubectl available -- ✅ Cluster connection OK -- ✅ RBAC manifest exists -- ✅ DaemonSet manifest exists -- ✅ DaemonSet deployed -- ✅ All pods running -- ✅ ServiceAccount exists -- ✅ ClusterRole exists -- ✅ ClusterRoleBinding exists -- ✅ ConfigMap exists -- ✅ Service exists -- ✅ Health endpoints responding -- ✅ Metrics endpoint working - -## What's Monitored - -### CPU Monitor (system-cpu) -- Load average thresholds: 80% warning, 95% critical -- Thermal throttling detection -- CPU usage monitoring - -### Memory Monitor (system-memory) -- Memory usage: 85% warning, 95% critical -- Swap usage: 50% warning, 80% critical -- OOM kill detection via /dev/kmsg - -### Disk Monitor (system-disk) -- Monitored paths: /, /var/lib/kubelet, /var/lib/docker, /var/lib/containerd -- Space thresholds: 85% warning, 95% critical -- Inode thresholds: 85% warning, 95% critical -- Readonly filesystem detection -- I/O health monitoring - -## Monitoring Configuration - -Configuration is embedded in the DaemonSet ConfigMap (lines 498-569 of daemonset.yaml). - -To update configuration: -```bash -# Edit ConfigMap -kubectl edit configmap -n kube-system node-doctor-config - -# Or apply updated manifest -kubectl apply -f deployment/daemonset.yaml - -# Restart pods to pick up changes -kubectl rollout restart daemonset/node-doctor -n kube-system -``` - -## Troubleshooting - -### Pods Not Starting - -```bash -# Check pod events -kubectl describe pod -n kube-system $POD_NAME - -# Common issues: -# - ImagePullBackOff: Check Harbor registry access -# - CrashLoopBackOff: Check logs for errors -# - Pending: Check node selectors and tolerations -``` - -### Permission Errors - -```bash -# Verify RBAC is applied -kubectl get serviceaccount -n kube-system node-doctor -kubectl get clusterrole node-doctor -kubectl get clusterrolebinding node-doctor - -# Check pod service account -kubectl get pods -n kube-system -l app=node-doctor -o jsonpath='{.items[0].spec.serviceAccountName}' -``` - -### Monitors Not Working - -```bash -# Check pod has host access -kubectl exec -n kube-system $POD_NAME -- ls -la /host -kubectl exec -n kube-system $POD_NAME -- ls -la /dev/kmsg -kubectl exec -n kube-system $POD_NAME -- cat /proc/meminfo | head -5 - -# Verify security context -kubectl get pods -n kube-system -l app=node-doctor -o jsonpath='{.items[0].spec.containers[0].securityContext}' -``` - -### No Metrics Appearing - -```bash -# Check Prometheus endpoint is accessible -kubectl exec -n kube-system $POD_NAME -- curl -s localhost:9100/metrics - -# Verify Service is created -kubectl get service -n kube-system node-doctor-metrics - -# Check if Prometheus can scrape -kubectl get pods -n kube-system -l app=node-doctor -o jsonpath='{.items[0].metadata.annotations}' -``` - -## Uninstalling - -```bash -# Remove DaemonSet (includes ConfigMap and Service) -kubectl delete -f deployment/daemonset.yaml - -# Remove RBAC resources -kubectl delete -f deployment/rbac.yaml - -# Verify cleanup -kubectl get all -n kube-system -l app=node-doctor -``` - -## Production Considerations - -1. **Resource Limits**: Adjust CPU/memory limits based on cluster size -2. **Image Pull Secrets**: Add imagePullSecrets if Harbor requires authentication -3. **Node Selectors**: Add nodeSelector to limit deployment to specific nodes -4. **Monitoring Integration**: Configure Prometheus ServiceMonitor or PodMonitor -5. **Alerting**: Set up alerts for NodeDoctorHealthy condition changes -6. **Backup**: Include ConfigMap in cluster backup procedures -7. **Updates**: Use rolling updates with maxUnavailable: 25% - a percentage, not 1, so a single crashlooping pod cannot consume the whole budget and stall the rollout at 0/N - -## Support - -- GitHub: https://github.com/supporttools/node-doctor -- Documentation: See `/docs` directory -- Issues: Report at GitHub issues - -## Summary of Components - -**Completed Tasks:** -- ✅ Task #3129: DaemonSet manifest -- ✅ Task #3130: RBAC manifests -- ✅ Task #3131: ConfigMap manifest (embedded in daemonset.yaml) -- ✅ Task #3132: Service manifest (embedded in daemonset.yaml) -- ✅ Dockerfile for container image - -**What's Working:** -- CPU Monitor: 93.1% test coverage -- Memory Monitor: 95.7% test coverage -- Disk Monitor: 93.0% test coverage -- All tests passing -- Binary builds successfully -- Docker image builds (84.8 MB) -- Ready for production deployment - -**Image Registry:** -- Docker Hub: `supporttools/node-doctor:v0.1.0` -- Image: `node-doctor:v0.1.0` (84.8 MB Alpine-based) - -You're ready to deploy to a real cluster! 🚀 diff --git a/docs/development/local-cicd-validation-guide.md b/docs/development/local-cicd-validation-guide.md deleted file mode 100644 index d274d5f..0000000 --- a/docs/development/local-cicd-validation-guide.md +++ /dev/null @@ -1,734 +0,0 @@ -# Local CI/CD Validation Guide - -## Table of Contents -1. [Problem Statement](#problem-statement) -2. [CI/CD Pipeline Architecture](#cicd-pipeline-architecture) -3. [Local Validation Script](#local-validation-script) -4. [Known Issues & Fixes](#known-issues--fixes) -5. [Usage Guide](#usage-guide) -6. [Integration with Git Hooks](#integration-with-git-hooks) -7. [Troubleshooting](#troubleshooting) - ---- - -## Problem Statement - -### The Gap - -Our current local validation script (`scripts/validate-all.sh`) was designed for the original repository structure and checks these components: -- `api/` -- `agent/` -- `monitoring-agent/` -- `ui/` - -However, our **CI/CD pipeline** (`.github/workflows/pipeline-v2.yml`) now tests a completely different set of 6 components: -- `api` (via `cd api && ...`) -- `org-management-controller` (via `./pkg/controller/org-management/...`) -- `probe-controller` (via `./pkg/probe/...`) -- `ingestor` (via `./pkg/ingestor/...`) -- `customer-probe-agent` (via `./cmd/customer-probe-agent/...`) -- `pkg` (via `./pkg/...`) - -### The Cost - -This mismatch means: -- ❌ Issues that will fail CI/CD **cannot be detected locally** -- ❌ Developers push code thinking it's validated -- ❌ CI/CD fails 15-30 minutes later -- ❌ Developer context-switches back to fix the issue -- ❌ Another push, another 15-30 minute wait -- ❌ **Total time wasted per issue: 30-60 minutes** - -### The Solution - -Create a new validation script (`validate-pipeline-local.sh`) that: -- ✅ **Mirrors the exact CI/CD pipeline steps** -- ✅ Tests all 6 components the CI/CD tests -- ✅ Runs the same commands in the same order -- ✅ Provides immediate feedback (5-10 minutes vs 15-30 minutes) -- ✅ Detects 100% of CI/CD issues before pushing - ---- - -## CI/CD Pipeline Architecture - -### Pipeline File -**Location**: `.github/workflows/pipeline-v2.yml` - -### Test Matrix - -The CI/CD pipeline uses GitHub Actions matrix strategy to test 6 components in parallel: - -```yaml -strategy: - matrix: - component: - - api - - org-management-controller - - probe-controller - - ingestor - - customer-probe-agent - - pkg - fail-fast: false - max-parallel: 6 -``` - -### Test Steps for Each Component - -Every component goes through these 5 validation steps: - -#### 1. Go Format Check (gofmt) -Ensures all Go code is properly formatted using `gofmt`. - -**Per-component commands:** -```bash -# api -test -z "$(find api -name '*.go' -not -path '*/vendor/*' -exec gofmt -l {} +)" - -# pkg -test -z "$(find pkg -name '*.go' -not -path '*/vendor/*' -exec gofmt -l {} +)" - -# org-management-controller -test -z "$(find pkg/controller/org-management -name '*.go' -not -path '*/vendor/*' -exec gofmt -l {} +)" - -# probe-controller -test -z "$(find pkg/probe -name '*.go' -not -path '*/vendor/*' -exec gofmt -l {} +)" - -# ingestor -test -z "$(find pkg/ingestor -name '*.go' -not -path '*/vendor/*' -exec gofmt -l {} +)" - -# customer-probe-agent -test -z "$(find cmd/customer-probe-agent -name '*.go' -not -path '*/vendor/*' -exec gofmt -l {} +)" -``` - -**Exit code**: `0` = pass, `1` = files need formatting - -#### 2. Go Vet (static analysis) -Examines Go source code and reports suspicious constructs. - -**Per-component commands:** -```bash -# api -cd api && go vet ./... - -# pkg -go vet ./pkg/... - -# org-management-controller -go vet ./pkg/controller/org-management/... - -# probe-controller -go vet ./pkg/probe/... - -# ingestor -go vet ./pkg/ingestor/... - -# customer-probe-agent (HAS BUG - see Known Issues) -go vet ./cmd/customer-probe-agent/... # ❌ BROKEN - should cd first -``` - -#### 3. Staticcheck (advanced linting) -Staticcheck is a state-of-the-art linter for Go. It uses static analysis to find bugs and performance issues. - -**Installation** (done in CI/CD): -```bash -go install honnef.co/go/tools/cmd/staticcheck@latest -``` - -**Per-component commands:** -```bash -# api -cd api && staticcheck ./... - -# pkg -staticcheck ./pkg/... - -# org-management-controller -staticcheck ./pkg/controller/org-management/... - -# probe-controller -staticcheck ./pkg/probe/... - -# ingestor -staticcheck ./pkg/ingestor/... - -# customer-probe-agent (HAS BUG - see Known Issues) -staticcheck ./cmd/customer-probe-agent/... # ❌ BROKEN - should cd first -``` - -**Suppression syntax**: -```go -//lint:ignore SA1019 Reason for ignoring this specific check -return deprecatedFunction() - -//lint:file-ignore SA1019 AWS SDK deprecations - planned migration -package services -``` - -#### 4. Gosec (security scanning) -Inspects source code for security problems. - -**Installation** (done in CI/CD): -```bash -go install github.com/securego/gosec/v2/cmd/gosec@latest -``` - -**Per-component commands:** -```bash -# api -cd api && gosec -quiet ./... - -# pkg -gosec -quiet ./pkg/... - -# org-management-controller -gosec -quiet ./pkg/controller/org-management/... - -# probe-controller -gosec -quiet ./pkg/probe/... - -# ingestor -gosec -quiet ./pkg/ingestor/... - -# customer-probe-agent (HAS BUG - see Known Issues) -gosec -quiet ./cmd/customer-probe-agent/... # ❌ BROKEN - should cd first -``` - -**Suppression syntax**: -```go -// #nosec G115 - queueSize is always positive, validated at creation -queueCapacity := uint64(pq.queueSize) -``` - -#### 5. Unit Tests with Race Detection -Runs all unit tests with race detector and generates coverage reports. - -**Per-component commands:** -```bash -# api -cd api && go test -v -race -coverprofile=coverage.out -covermode=atomic ./... - -# org-management-controller -go test -v -race -coverprofile=coverage.out -covermode=atomic ./pkg/controller/org-management/... - -# probe-controller -go test -v -race -coverprofile=coverage.out -covermode=atomic ./pkg/probe/... - -# ingestor -go test -v -race -coverprofile=coverage.out -covermode=atomic ./pkg/ingestor/... - -# customer-probe-agent -cd cmd/customer-probe-agent && go test -v -race -coverprofile=coverage.out -covermode=atomic ./... - -# pkg -go test -v -race -coverprofile=coverage.out -covermode=atomic ./pkg/... -``` - -**Flags explained**: -- `-v`: Verbose output -- `-race`: Enable race detector (catches data races) -- `-coverprofile=coverage.out`: Generate coverage report -- `-covermode=atomic`: Coverage mode for race detector compatibility - ---- - -## Local Validation Script - -### Script Location -**File**: `scripts/validate-pipeline-local.sh` - -### Design Principles - -1. **Exact Parity with CI/CD** - - Uses identical commands - - Same execution order - - Same exit codes - -2. **Clear Output** - - Component-by-component progress - - ✓/✗ indicators for each step - - Color-coded results - -3. **Detailed Logging** - - Saves full output to `.validation-logs/` - - Timestamped log files - - Easy debugging - -4. **Fast Failure** - - Stops on first error per component (default) - - `--continue` flag to validate all components - -5. **Component Isolation** - - Each component tested independently - - Can validate single component with `--component` - -### Script Structure - -```bash -#!/bin/bash -# Validates all 6 CI/CD components locally before pushing - -set -e - -# Color codes -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' - -# Components matching CI/CD pipeline -COMPONENTS=( - "api" - "org-management-controller" - "probe-controller" - "ingestor" - "customer-probe-agent" - "pkg" -) - -# Validation functions -validate_go_fmt() { ... } -validate_go_vet() { ... } -validate_staticcheck() { ... } -validate_gosec() { ... } -validate_tests() { ... } - -# Main validation loop -for component in "${COMPONENTS[@]}"; do - echo "Validating $component..." - validate_go_fmt "$component" - validate_go_vet "$component" - validate_staticcheck "$component" - validate_gosec "$component" - validate_tests "$component" -done -``` - -### Output Format - -``` -🔍 Local CI/CD Validation -═══════════════════════════════════════════════════════════════ - -[1/6] Validating api... - ✓ go fmt check - ✓ go vet - ✓ staticcheck - ✓ gosec - ✓ tests (47 passed) - -[2/6] Validating org-management-controller... - ✓ go fmt check - ✓ go vet - ✓ staticcheck - ✓ gosec - ✓ tests (23 passed) - -... - -═══════════════════════════════════════════════════════════════ -✅ All 6 components passed validation! - -Logs saved to: .validation-logs/20251011_123045/ -``` - ---- - -## Known Issues & Fixes - -### Issue 1: customer-probe-agent Module Confusion - -**Problem**: `customer-probe-agent` has its own `go.mod` file (separate Go module), but the CI/CD workflow runs validation commands from the repository root. - -**Symptom**: -```bash -$ go vet ./cmd/customer-probe-agent/... -pattern ./cmd/customer-probe-agent/...: directory prefix cmd/customer-probe-agent -does not contain main module or its selected dependencies -``` - -**Root Cause**: When a directory has its own `go.mod`, it becomes a separate module. Commands like `go vet ./cmd/customer-probe-agent/...` from the root try to vet it as part of the root module, which fails. - -**Fix in Workflow** (`.github/workflows/pipeline-v2.yml`): - -```yaml -# BEFORE (BROKEN): -- name: Run go vet - run: | - case "${{ matrix.component }}" in - customer-probe-agent) - go vet ./cmd/customer-probe-agent/... # ❌ Fails - ;; - esac - -# AFTER (FIXED): -- name: Run go vet - run: | - case "${{ matrix.component }}" in - customer-probe-agent) - cd cmd/customer-probe-agent && go vet ./... # ✅ Works - ;; - esac -``` - -**Apply same fix to**: -- `staticcheck` step (lines 148-150) -- `gosec` step (lines 172-174) - -**Tests are already correct**: They already `cd` into the directory before running. - -### Issue 2: Type Safety - uint vs uint64 - -**Problem**: Some fields use `uint` for organization IDs but NATS functions expect `uint64`. - -**Symptom**: -```go -// pkg/ingestor/server/server.go:331 -if uint64(conn.OrganizationID) != organizationID -``` - -The fact that we need `uint64()` cast suggests a type mismatch. - -**Investigation**: -```bash -# Find all OrganizationID field definitions -grep -r "OrganizationID.*uint" pkg/ingestor/server/ - -# Check if it should be uint64 everywhere -grep -r "func.*organizationID uint64" pkg/ -``` - -**Fix**: Ensure OrganizationID is consistently `uint64` throughout ingestor package. - -### Issue 3: Staticcheck False Positives - -**Problem**: Staticcheck sometimes reports false positives for format strings. - -**Symptom**: -``` -pkg/notification/analytics/reports.go:392:21: couldn't parse format string (SA5009) -``` - -**Fix**: Use `//lint:ignore SA5009` to suppress: -```go -//lint:ignore SA5009 staticcheck false positive - format string is valid with %% for literal % -return fmt.Sprintf(html, ...) -``` - -**File-level suppression**: -```go -//lint:file-ignore SA1019 AWS SDK deprecations - planned migration - -package services -``` - ---- - -## Usage Guide - -### Basic Usage - -#### Validate All Components -```bash -./scripts/validate-pipeline-local.sh -``` - -This is the **recommended command** before every push. It validates all 6 components exactly as CI/CD will. - -**Time**: ~5-10 minutes (depends on test count) - -#### Validate Single Component -```bash -./scripts/validate-pipeline-local.sh --component ingestor -``` - -Use this when you've only changed code in one component and want quick feedback. - -**Time**: ~1-2 minutes per component - -#### Quick Mode (Skip Heavy Checks) -```bash -./scripts/validate-pipeline-local.sh --quick -``` - -Skips expensive operations like full test suites. Only runs fmt, vet, staticcheck, gosec. - -**Time**: ~2-3 minutes - -**When to use**: Pre-commit checks, quick verification - -#### Continue on Failure -```bash -./scripts/validate-pipeline-local.sh --continue -``` - -Validates all components even if some fail. Useful for getting complete picture of issues. - -**Default behavior**: Stops at first failure per component - -### Makefile Integration - -#### Add to Makefile - -```makefile -# ============================================================================= -# Local CI/CD Validation -# ============================================================================= - -.PHONY: validate-pipeline-local validate-component - -# Validate all components (mirrors CI/CD pipeline) -validate-pipeline-local: - @echo "Running full pipeline validation locally..." - @./scripts/validate-pipeline-local.sh - -# Validate single component -validate-component: - @echo "Validating component: $(COMPONENT)..." - @./scripts/validate-pipeline-local.sh --component $(COMPONENT) - -# Quick validation (skip expensive tests) -validate-quick: - @echo "Running quick validation..." - @./scripts/validate-pipeline-local.sh --quick -``` - -#### Usage via Make - -```bash -# Full validation -make validate-pipeline-local - -# Single component -make validate-component COMPONENT=ingestor - -# Quick mode -make validate-quick -``` - -### Reading Validation Logs - -Logs are saved to `.validation-logs/TIMESTAMP/` with this structure: - -``` -.validation-logs/ -└── 20251011_123045/ - ├── api_fmt.log - ├── api_vet.log - ├── api_staticcheck.log - ├── api_gosec.log - ├── api_tests.log - ├── ingestor_fmt.log - ├── ingestor_vet.log - └── ... -``` - -**To view a specific failure**: -```bash -# List recent validation runs -ls -lt .validation-logs/ - -# View specific log -cat .validation-logs/20251011_123045/ingestor_staticcheck.log - -# Search for errors -grep -r "error:" .validation-logs/20251011_123045/ -``` - -**Log cleanup**: Script automatically keeps last 10 runs, deletes older logs. - ---- - -## Integration with Git Hooks - -### Pre-Push Hook - -**File**: `.githooks/pre-push` - -```bash -#!/bin/bash -# Validate code before pushing to remote - -echo "🔍 Running local CI/CD validation before push..." -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - -# Run quick validation (skip expensive full test suites) -./scripts/validate-pipeline-local.sh --quick - -# Check exit code -if [ $? -ne 0 ]; then - echo "" - echo "❌ Validation failed! Push aborted." - echo "" - echo "To fix:" - echo " 1. Review errors above" - echo " 2. Fix issues and commit" - echo " 3. Push again" - echo "" - echo "To skip validation (NOT recommended):" - echo " git push --no-verify" - exit 1 -fi - -echo "" -echo "✅ Validation passed! Proceeding with push..." -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -exit 0 -``` - -### Enable Git Hooks - -```bash -# Configure git to use .githooks/ directory -git config core.hooksPath .githooks - -# Make hook executable -chmod +x .githooks/pre-push -``` - -### Override Hook (Emergency) - -If you **absolutely must** push without validation (production emergency, etc.): - -```bash -git push --no-verify -``` - -âš ī¸ **Warning**: Use sparingly! This bypasses all validation and CI/CD will likely fail. - ---- - -## Troubleshooting - -### Issue: "staticcheck: command not found" - -**Cause**: staticcheck not installed - -**Fix**: -```bash -go install honnef.co/go/tools/cmd/staticcheck@latest -``` - -### Issue: "gosec: command not found" - -**Cause**: gosec not installed - -**Fix**: -```bash -go install github.com/securego/gosec/v2/cmd/gosec@latest -``` - -### Issue: Tests timeout - -**Cause**: Some tests take longer than default timeout - -**Fix**: Increase timeout in test command: -```bash -go test -v -race -timeout=10m -coverprofile=coverage.out ./... -``` - -### Issue: "permission denied" when running script - -**Cause**: Script not executable - -**Fix**: -```bash -chmod +x scripts/validate-pipeline-local.sh -``` - -### Issue: customer-probe-agent validation fails - -**Cause**: Workflow bug - see [Known Issues](#known-issues--fixes) - -**Temporary Workaround**: Skip customer-probe-agent until workflow fixed: -```bash -./scripts/validate-pipeline-local.sh --skip customer-probe-agent -``` - -### Issue: Log directory fills up disk space - -**Cause**: Many validation runs, logs not cleaned up - -**Fix**: -```bash -# Manual cleanup (keeps last 10 runs) -ls -t .validation-logs | tail -n +11 | xargs -I {} rm -rf .validation-logs/{} - -# Or delete all logs -rm -rf .validation-logs -``` - -### Issue: Different results locally vs CI/CD - -**Possible causes**: -1. **Go version mismatch**: CI/CD uses Go 1.24 - check yours: - ```bash - go version # Should be 1.24.x - ``` - -2. **Stale dependencies**: Update go.mod: - ```bash - go mod tidy - go mod download - ``` - -3. **Uncommitted changes**: CI/CD tests committed code: - ```bash - git status # Check for uncommitted changes - ``` - -4. **Cache issues**: Clear Go cache: - ```bash - go clean -cache -testcache -modcache - ``` - -### Issue: Validation takes too long - -**Solutions**: - -1. **Use quick mode** for pre-commit: - ```bash - ./scripts/validate-pipeline-local.sh --quick - ``` - -2. **Validate only changed components**: - ```bash - # If you only changed ingestor - ./scripts/validate-pipeline-local.sh --component ingestor - ``` - -3. **Run in parallel** (future enhancement): - ```bash - ./scripts/validate-pipeline-local.sh --parallel - ``` - ---- - -## Appendix: Component Coverage Matrix - -| Component | fmt | vet | staticcheck | gosec | tests | Path | -|-----------|-----|-----|-------------|-------|-------|------| -| api | ✓ | ✓ | ✓ | ✓ | ✓ | `api/` | -| org-management-controller | ✓ | ✓ | ✓ | ✓ | ✓ | `pkg/controller/org-management/` | -| probe-controller | ✓ | ✓ | ✓ | ✓ | ✓ | `pkg/probe/` | -| ingestor | ✓ | ✓ | ✓ | ✓ | ✓ | `pkg/ingestor/` | -| customer-probe-agent | ✓ | âš ī¸ | âš ī¸ | âš ī¸ | ✓ | `cmd/customer-probe-agent/` | -| pkg | ✓ | ✓ | ✓ | ✓ | ✓ | `pkg/` | - -âš ī¸ = Has workflow bug, needs `cd` before command - ---- - -## Related Documentation - -- [Release Process](../release-process.md) - What the tag-triggered pipeline actually publishes -- [GitHub Actions Workflows](../../.github/workflows/README.md) - Job-by-job CI reference -- [Task Execution Workflow](./task-execution-workflow.md) -- `make help` - Authoritative list of Makefile targets for this repo - ---- - -**Last Updated**: 2025-10-11 -**Maintained By**: Platform Team -**Questions**: See #engineering-platform on Slack diff --git a/docs/development/master-task-management.md b/docs/development/master-task-management.md deleted file mode 100644 index 461897e..0000000 --- a/docs/development/master-task-management.md +++ /dev/null @@ -1,275 +0,0 @@ -# Master Task Management System - -The {{PROJECT_NAME}} project follows a prioritized task management system based on the master design documents. ALL development work must follow these rules for controlled, systematic progress. - -## Task Reference Documents - -- **Master Design Document**: `/master-design-doc.md` - System architecture and service dependencies -- **Task Management System**: TaskForge project management system with 546 tasks organized in hierarchical structure - - Tasks organized: Project → Features → Tasks (11 features, 546 total tasks) - - Accessible via MCP tools: `mcp__taskforge__getTasks`, `mcp__taskforge__createTask`, `mcp__taskforge__updateTask` - - See [TaskForge Usage Guide](taskforge-usage-guide.md) for complete documentation - - Migration from todo-mcp documented in `taskforge-migration-guide.md` - -## Priority System (MUST FOLLOW) - -### Priority 1 - Foundation Services (IMPLEMENT FIRST) - -- org-management-controller: Core orchestration system -- notification-service: Required by all alert systems -- Database consolidation: Multi-tenant and single-tenant patterns -- **BLOCKER**: No P2+ work until P1 foundation is complete - -### Priority 2 - Core Monitoring (BUSINESS CRITICAL) - -- alert-controller: Internal monitoring alerts -- health-controller: Platform health monitoring -- linux-agent: Enhanced data collection -- **DEPENDENCY**: Requires P1 foundation services - -### Priority 3 - Advanced Features (VALUE-ADD) - -- probe-controller, tag-controller, job-controller, scheduler-controller -- **DEPENDENCY**: Requires P1 + P2 core services - -### Priority 4 - Business Operations (REVENUE) - -- billing-controller, reporting-controller -- **DEPENDENCY**: Requires monitoring infrastructure from P1-P3 - -### Priority 5 - Platform Polish (UX) - -- Web-UI critical fixes (P5A) then enhancements (P5B) -- **PARALLEL**: Can work on P5A critical fixes alongside P1-P2 - -## Task Execution Rules - -### Before Starting ANY Task - -Use TaskForge to select the next task: - -```javascript -// 1. List tasks for current priority using TaskForge -mcp__taskforge__getTasks({ - projectId: 1, - featureId: 1, // P1-Foundation-Services - status: "todo", - priority: "high" -}); - -// 2. Check for blocked tasks -mcp__taskforge__getBlockedTasks({ - projectId: 1 -}); - -// 3. Use TodoWrite tool to track session progress -``` - -**Task Selection Rules:** - -1. **NEVER skip priorities** - Complete P1 before P2, P2 before P3, etc. -2. **RESPECT dependencies** - Check [DEPENDS ON: service] annotations -3. **ONE service at a time** - Complete service implementation before starting next -4. **CRITICAL first** - Within priority, do CRITICAL before HIGH before MEDIUM - -### Task Status Tracking (MANDATORY) - -Using TaskForge (PostgreSQL-backed project management system): - -```javascript -// 1. Start a task - set status to "in_progress" -mcp__taskforge__updateTask({ - taskId: 123, - status: "in_progress" -}); - -// 2. Complete a task -mcp__taskforge__updateTask({ - taskId: 123, - status: "done", - actualHours: 6.5 -}); - -// 3. Block a task (for dependencies or issues) -mcp__taskforge__updateTask({ - taskId: 123, - status: "blocked" -}); -// Or use mcp__taskforge__setDependency to track blocking tasks - -// 4. Delete irrelevant tasks (soft delete) -mcp__taskforge__deleteTask({ - taskId: 123 -}); - -// 5. Use TodoWrite tool for session-level progress tracking -``` - -## Implementation Standards - -### For Backend Services (Go) - -```bash -# ALWAYS follow this sequence: -# 1. Create service directory structure -mkdir -p controllers/[service-name]/{cmd,pkg/{handlers,models,services}} - -# 2. Implement core functionality with proper patterns -# - Single-function-per-file handlers -# - Comprehensive logging (Trace/Debug) -# - Complete Swagger documentation -# - Database models in BOTH locations (ConnectDB.go AND migrator.go) - -# 3. Test locally before committing -make build-api-local -make test-api-local -go test -v ./controllers/[service-name]/... - -# 4. Update TaskForge with completion -``` - -### For Multi-Service Integration - -```bash -# ALWAYS verify service dependencies: -# 1. notification-service running before alert-controller -# 2. org-management-controller before single-tenant services -# 3. Database schema supports both single/multi-tenant patterns - -# Test integration points: -# - API endpoints responding -# - Database connections working -# - Service-to-service communication functional -``` - -### Deployment Integration - -```bash -# Each service MUST include: -# 1. Dockerfile in service directory -# 2. Helm templates in helm/server/templates// -# 3. Helm chart integration with parent chart (values.yaml) -# 4. CI/CD pipeline integration - -# Test deployment locally: -# Build → Test → Deploy to dev → Validate -``` - -## Controlled Development Workflow - -### Phase 1: Foundation (P1 Tasks Only) - -1. org-management-controller core API -2. notification-service with multi-tenant support -3. Database schema consolidation -4. **GATE**: Foundation integration testing before P2 - -### Phase 2: Core Monitoring (P2 Tasks) - -1. alert-controller with notification integration -2. health-controller with org-management integration -3. linux-agent enhancements -4. **GATE**: End-to-end monitoring workflow before P3 - -### Phase 3: Advanced Features (P3 Tasks) - -1. probe-controller, tag-controller implementation -2. job-controller and scheduler-controller -3. **GATE**: Full platform feature testing before P4 - -### Phase 4: Business Operations (P4 Tasks) - -1. billing-controller with notification integration -2. reporting-controller with data aggregation -3. **GATE**: Business workflow testing before P5B - -### Phase 5: Platform Polish - -1. P5A: Critical web-UI fixes (parallel with P1-P2) -2. P5B: Web-UI enhancements (after P1-P4 complete) - -## Quality Gates (MANDATORY) - -### Before Moving to Next Priority - -```bash -# 1. All tasks in current priority marked [COMPLETED] -# 2. Integration tests passing for completed services -# 3. Local deployment successful -# 4. No critical bugs or blockers -# 5. Documentation updated - -# Validation commands: -make build-all-local -make test-local -./scripts/validate-all.sh -``` - -### Service Completion Checklist - -- [ ] Core functionality implemented and tested -- [ ] Database models added to both locations -- [ ] API endpoints with Swagger docs -- [ ] Service integration tests passing -- [ ] Helm chart and deployment configs -- [ ] TaskForge updated with completion -- [ ] No critical or high-priority bugs -- [ ] Ready for next service dependencies - -## Emergency Procedures - -### If Blocked on Current Priority - -1. Document blocker in TaskForge -2. Create specific blocker resolution tasks -3. **NEVER jump to next priority** - resolve blocker first -4. Consider parallel work on P5A critical fixes only - -### If Service Integration Fails - -1. Roll back to last working state -2. Create integration debugging tasks -3. Fix integration before continuing with service -4. Update master design doc if architecture changes needed - -## Task Development Workflow - -**ALL development tasks MUST follow the comprehensive workflow documented in `task-execution-workflow.md`.** - -This mandatory 8-step workflow includes: - -1. **Task Selection & Research** - Mandatory research phase before any coding -2. **Design & Planning** - Present comprehensive implementation plan -3. **Approval Gate** - Wait for explicit approval before coding -4. **Implementation** - Code with strict adherence to standards -5. **Quality Assurance** - Build verification, QA review, and testing -6. **Deployment** - Mandatory deployment with `make bump` -7. **Devils Advocate Verification** - Final validation of task completion -8. **Task Completion** - Update TaskForge and commit changes - -**Key Requirements:** -- ONE task at a time - never work on multiple tasks simultaneously -- RESEARCH before coding - no exceptions -- BUILD verification required before QA review -- QA validation required before deployment -- DEPLOYMENT mandatory before completion -- DEVILS ADVOCATE approval required before marking complete -- Multi-repository updates must be completed together - -**Multi-Repository Considerations:** - -When tasks affect multiple repositories ({{PROJECT_NAME}}, go-sdk, linux-agent, web-ui, cli): -- Update repositories in dependency order: API → SDK → Agent/UI/CLI -- Build and test ALL modified repositories -- Commit and tag ALL repositories appropriately -- NEVER leave repositories in inconsistent state - -See [Task Execution Workflow](task-execution-workflow.md) for complete details, examples, and enforcement rules. - -## See Also - -- [TaskForge Usage Guide](taskforge-usage-guide.md) - Complete TaskForge MCP tools documentation -- [Task Execution Workflow](task-execution-workflow.md) - Mandatory 8-step development workflow -- [go-sdk Synchronization Guide](go-sdk-synchronization-guide.md) - SDK update requirements -- [Web UI-API Synchronization](web-ui-api-synchronization.md) - Frontend sync procedures -- [Helm Chart Standards](helm-chart-standards.md) - Deployment configuration rules diff --git a/docs/development/task-execution-workflow.md b/docs/development/task-execution-workflow.md deleted file mode 100644 index 6f92549..0000000 --- a/docs/development/task-execution-workflow.md +++ /dev/null @@ -1,1018 +0,0 @@ -# Task Execution Workflow - -**⭐ Core Framework Document** - This workflow defines the mandatory process for executing all development tasks in the Node Doctor project. - -## 📝 About This Document - -This workflow is adapted from production-tested practices and customized for Node Doctor - a Kubernetes DaemonSet monitoring tool with auto-remediation capabilities. - -**Node Doctor Specifics:** -- Single repository, single binary architecture -- Go 1.21+ required -- Kubernetes DaemonSet deployment model -- Monitor/Remediator plugin system -- Direct node access with elevated privileges -- Safety-first remediation approach - -## Available Resources - -### Task Management System - -The project uses PostgreSQL-backed TaskForge for persistent task management: -- **Tasks organized by project** (P1-P5 priority levels, 14 features) -- **Priority-based**: Urgent (đŸ”Ĩ), High (🔴), Medium (🟡), Low (đŸŸĸ) -- **Persistent across sessions**: All Claude Code instances share the same task state -- **MCP Tools**: `mcp__taskforge__getTasks`, `mcp__taskforge__updateTask`, `mcp__taskforge__getBlockedTasks` -- See [TaskForge Usage Guide](taskforge-usage-guide.md) and [Master Task Management](master-task-management.md) - -### Development Environment - -- Local Kubernetes cluster access available via configured kubeconfig (see Makefile KUBECONFIG_* variables) -- DaemonSet testing environment for node-level operations -- Use this environment to: - - Test monitor implementations with real node metrics - - Validate remediator safety mechanisms - - Run integration tests with Kubernetes API - - Verify DaemonSet deployment and RBAC configurations - - Debug monitor/remediator interactions - - Test exporter functionality (Prometheus metrics, Kubernetes events) -- Always verify environment context before running operations requiring elevated privileges -- Use kube-system namespace for DaemonSet deployments - -### Single Repository Structure - -Node Doctor uses a single repository with the following structure: - -``` -node-doctor/ -├── cmd/node-doctor/ # Main entry point -├── pkg/ -│ ├── monitors/ # Monitor implementations -│ ├── remediators/ # Auto-remediation implementations -│ ├── exporters/ # Prometheus/Kubernetes exporters -│ ├── problemdetector/ # Core orchestration -│ ├── config/ # Configuration types -│ └── utils/ # Shared utilities -├── deploy/ # Kubernetes manifests -│ ├── daemonset.yaml # DaemonSet configuration -│ ├── rbac.yaml # ServiceAccount/ClusterRole -│ └── configmap.yaml # Monitor/remediator configs -├── docs/ # Documentation -└── tests/ # Integration/E2E tests -``` - ---- - -## Workflow Steps - -### 1. Receive and Initiate Task - -- I will provide you with a specific task from TaskForge or by name/description -- Retrieve tasks using `mcp__taskforge__getTasks` with appropriate filters (featureId, priority, status) -- Verify the task priority level and ensure no higher priority tasks are pending -- Check for task dependencies (blockedByTaskId field) -- If task has unmet dependencies or is already completed, report this and await further instruction -- Update task status to "in_progress" using `mcp__taskforge__updateTask` when starting work - -**TodoWrite Integration (MANDATORY for session tracking)**: -``` -Use TodoWrite tool to create initial task breakdown: -- Research and design phase -- Implementation phase -- Build verification -- QA review -- Testing -- Deployment -- Devils advocate verification -- Task completion - -Mark first item as "in_progress" immediately. - -NOTE: TodoWrite tracks workflow steps WITHIN the current session. -TaskForge tracks project-level task completion ACROSS all sessions. -Both are mandatory - use TodoWrite for workflow progress, TaskForge for task completion. -``` - -### 2. Research & Design Phase (MANDATORY - NO CODING WITHOUT THIS) - -**Thoroughly research the task requirements:** -- Review architecture.md for system design specifications -- Review monitors.md for monitor specifications and patterns -- Review remediation.md for safety mechanisms and remediator patterns -- Identify integration points with Problem Detector orchestrator -- Review existing monitors/remediators for similar patterns -- Understand safety requirements and failure modes - -**Present a comprehensive implementation plan that includes:** - -#### Task Overview -Clear summary aligned with Node Doctor architecture (monitors, remediators, exporters) - -#### Design Approach -Technical strategy following Node Doctor's established patterns: -- **Monitor Pattern**: Implement Monitor interface with Start()/Stop() methods -- **Channel-Based Communication**: Use Go channels for status updates to Problem Detector -- **Remediator Pattern**: Implement Remediator interface with safety mechanisms -- **Safety First**: Cooldown periods, attempt limiting, circuit breakers, rate limiting -- **Configuration-Driven**: YAML/JSON config with validation -- **Kubernetes Integration**: Use client-go for API interactions, respect RBAC boundaries - -#### Files Affected -Complete list with brief explanation: -- **Monitor implementations**: `pkg/monitors/[monitor-name].go`, `pkg/monitors/[monitor-name]_test.go` -- **Remediator implementations**: `pkg/remediators/[remediator-name].go`, `pkg/remediators/[remediator-name]_test.go` -- **Exporter updates**: `pkg/exporters/prometheus.go`, `pkg/exporters/kubernetes.go` -- **Configuration types**: `pkg/config/types.go`, `pkg/config/validation.go` -- **Problem Detector integration**: `pkg/problemdetector/orchestrator.go` -- **Deployment manifests**: `deploy/daemonset.yaml`, `deploy/rbac.yaml`, `deploy/configmap.yaml` -- **Documentation**: `docs/monitors.md`, `docs/remediation.md` - -#### Dependencies -Prerequisites and blockers: -- Node access requirements (hostPath, hostNetwork, hostPID) -- Kubernetes permissions (RBAC for ServiceAccount) -- System dependencies (systemd, network tools, disk utilities) -- Go packages (client-go, prometheus/client_golang, etc.) -- Safety mechanism implementations (circuit breaker, rate limiter) - -#### Success Criteria -Specific, measurable conditions: -- Project builds successfully: `make build-node-doctor-local` -- All tests pass: `make test-node-doctor-local` -- Validation passes: `make validate-pipeline-local` -- QA agent verification passes -- DaemonSet deploys successfully: `kubectl apply -f deploy/` -- Monitor collects data and reports problems correctly -- Remediator respects safety limits and cooldowns -- Exporters publish metrics/events correctly -- Devils advocate verification passes - -#### Risks & Mitigations -Potential issues and fallbacks: -- **Node access risks**: Privileged operations, validate inputs rigorously -- **Remediation risks**: Implement circuit breakers, cooldown periods, attempt limits -- **Performance risks**: Resource limits, efficient polling, async operations -- **Failure modes**: Graceful degradation, proper error handling, monitoring fallbacks - -**TodoWrite Update**: Mark "Research and design phase" as completed, mark "Implementation phase" as in_progress. - -### 3. Wait for Approval (MANDATORY GATE) - -**You must pause here and await explicit instruction.** Do not proceed to coding without approval. - -Based on feedback: -- `✅ Approve`: Proceed immediately to Step 4 (Implementation) -- `🔄 Request changes`: Revise plan according to feedback and re-present -- `â­ī¸ Skip`: Mark task as skipped and await new assignment - -**TodoWrite Update**: When approval received, mark "Implementation phase" as in_progress. - -### 4. Implementation Phase - -Implement the approved design with strict adherence to: - -#### Go Version -- **ALWAYS use Go 1.21+** - verify go.mod specifies "go 1.21" or higher - -#### Monitor Implementation Pattern -```go -package monitors - -import ( - "context" - "time" - "github.com/supporttools/node-doctor/pkg/problemdetector" -) - -// CPUMonitor implements the Monitor interface -type CPUMonitor struct { - config *CPUMonitorConfig - stopCh chan struct{} - statusCh chan *problemdetector.Status -} - -// NewCPUMonitor creates a new CPU monitor instance -func NewCPUMonitor(config *CPUMonitorConfig) (*CPUMonitor, error) { - // Validate config - if err := config.Validate(); err != nil { - return nil, fmt.Errorf("invalid config: %w", err) - } - - return &CPUMonitor{ - config: config, - stopCh: make(chan struct{}), - statusCh: make(chan *problemdetector.Status, 10), - }, nil -} - -// Start begins monitoring (runs in goroutine) -func (m *CPUMonitor) Start() (<-chan *problemdetector.Status, error) { - go m.monitorLoop() - return m.statusCh, nil -} - -// Stop halts monitoring -func (m *CPUMonitor) Stop() { - close(m.stopCh) -} - -func (m *CPUMonitor) monitorLoop() { - ticker := time.NewTicker(m.config.PollInterval) - defer ticker.Stop() - defer close(m.statusCh) - - for { - select { - case <-m.stopCh: - return - case <-ticker.C: - status := m.checkCPU() - if status != nil { - m.statusCh <- status - } - } - } -} -``` - -#### Remediator Implementation Pattern -```go -package remediators - -import ( - "context" - "time" - "github.com/supporttools/node-doctor/pkg/problemdetector" -) - -// SystemdRemediator handles systemd service issues -type SystemdRemediator struct { - config *SystemdRemediatorConfig - circuitBreaker *CircuitBreaker - rateLimiter *RateLimiter - lastRemediation map[string]time.Time -} - -// CanRemediate checks if this remediator handles the problem -func (r *SystemdRemediator) CanRemediate(problem problemdetector.Problem) bool { - return problem.Type == "systemd-service-failed" -} - -// Remediate attempts to fix the problem with safety checks -func (r *SystemdRemediator) Remediate(ctx context.Context, problem problemdetector.Problem) error { - // Check circuit breaker - if r.circuitBreaker.IsOpen() { - return fmt.Errorf("circuit breaker open, too many failures") - } - - // Check rate limiter - if !r.rateLimiter.Allow() { - return fmt.Errorf("rate limit exceeded") - } - - // Check cooldown period - if lastTime, ok := r.lastRemediation[problem.Resource]; ok { - if time.Since(lastTime) < r.GetCooldown() { - return fmt.Errorf("cooldown period not elapsed") - } - } - - // Perform remediation - err := r.restartService(ctx, problem.Resource) - - // Update circuit breaker and tracking - if err != nil { - r.circuitBreaker.RecordFailure() - return err - } - - r.circuitBreaker.RecordSuccess() - r.lastRemediation[problem.Resource] = time.Now() - return nil -} - -// GetCooldown returns the cooldown period for this remediator -func (r *SystemdRemediator) GetCooldown() time.Duration { - return r.config.CooldownPeriod -} -``` - -#### Configuration Pattern -```go -package config - -// NodeDoctorConfig is the root configuration -type NodeDoctorConfig struct { - Monitors []MonitorConfig `json:"monitors" yaml:"monitors"` - Remediators []RemediatorConfig `json:"remediators" yaml:"remediators"` - Exporters ExporterConfig `json:"exporters" yaml:"exporters"` - Global GlobalConfig `json:"global" yaml:"global"` -} - -// Validate performs comprehensive config validation -func (c *NodeDoctorConfig) Validate() error { - if len(c.Monitors) == 0 { - return fmt.Errorf("at least one monitor required") - } - - for i, monitor := range c.Monitors { - if err := monitor.Validate(); err != nil { - return fmt.Errorf("monitor[%d]: %w", i, err) - } - } - - // Validate remediators, exporters, global config... - return nil -} -``` - -#### DaemonSet Deployment -```yaml -# deploy/daemonset.yaml -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: node-doctor - namespace: kube-system -spec: - selector: - matchLabels: - app: node-doctor - template: - metadata: - labels: - app: node-doctor - spec: - serviceAccountName: node-doctor - hostNetwork: true # Access host networking - hostPID: true # Access host processes - containers: - - name: node-doctor - image: supporttools/node-doctor:latest - securityContext: - privileged: true # Required for remediation - volumeMounts: - - name: config - mountPath: /etc/node-doctor - - name: host-root - mountPath: /host - readOnly: true - volumes: - - name: config - configMap: - name: node-doctor-config - - name: host-root - hostPath: - path: / -``` - -#### RBAC Configuration -```yaml -# deploy/rbac.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: node-doctor - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: node-doctor -rules: -- apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: ["events"] - verbs: ["create", "patch"] -- apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: node-doctor -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: node-doctor -subjects: -- kind: ServiceAccount - name: node-doctor - namespace: kube-system -``` - -#### Repository Cleanliness -- Keep repository root clean (only essential config/docs) -- Proper file placement in pkg/, cmd/, deploy/ directories -- No generated files in repository (binaries, coverage files) -- Run `make validate-pipeline-local` to verify standards - -#### Write Tests as You Code (MANDATORY) - -**Critical Rule: NEVER write code without corresponding tests. Tests are not optional or an afterthought.** - -**Unit Tests (Go):** -- For EVERY monitor/remediator, write corresponding tests in `*_test.go` files -- Write tests BEFORE or ALONGSIDE implementation - not after -- Follow Go testing conventions with table-driven tests -- Minimum coverage requirements: - - **Happy path**: Valid inputs with expected behavior - - **Edge cases**: Boundary conditions, empty values, maximum limits - - **Error cases**: Error handling, validation failures, recovery - - **Safety mechanisms**: Circuit breaker, rate limiter, cooldown enforcement - - **Concurrent operations**: Race conditions and thread safety -- Run tests incrementally: `go test -v ./pkg/monitors/...` -- Maintain >80% code coverage -- Use mocks for system calls (systemd, disk ops, network) - -**Integration Tests:** -- Test monitor integration with Problem Detector -- Test remediator safety mechanisms under load -- Test exporter publishing to Prometheus/Kubernetes -- Verify channel communication patterns -- Place in `*_integration_test.go` files -- Run with build tags: `go test -tags=integration -v ./...` - -**End-to-End (E2E) Tests:** -- Required for complete monitoring workflows -- Test scenarios: - - Monitor detects problem → Problem Detector receives → Exporter publishes - - Problem detected → Remediator attempts fix → Safety limits enforced - - DaemonSet deployment → Configuration loading → Monitoring active -- Use realistic test scenarios with Docker/kind clusters -- Place in `tests/e2e/` directory -- Document test setup and requirements - -**Testing Workflow Integration:** -```bash -# 1. Write test first (TDD) or alongside code -# 2. Implement feature -# 3. Run unit tests -go test -v ./pkg/monitors/... -go test -v ./pkg/remediators/... - -# 4. Run integration tests -go test -tags=integration -v ./... - -# 5. Run E2E tests -go test -tags=e2e -v ./tests/e2e/... - -# 6. Check coverage -go test -cover -coverprofile=coverage.out ./... -go tool cover -html=coverage.out -``` - -**Quality Gates for Testing:** -- [ ] Every new monitor/remediator has unit tests -- [ ] All edge cases and error paths covered -- [ ] Integration tests pass for Problem Detector interaction -- [ ] E2E tests pass for complete workflows -- [ ] Code coverage >80% (target: 85%+) -- [ ] No flaky tests - all deterministic -- [ ] Tests run in reasonable time (<2min for unit, <10min for full suite) - -**If implementation reveals issues with approved plan, stop and report immediately** - -**TodoWrite Update**: Mark "Implementation phase" as completed, mark "Build verification" as in_progress. - -### 5. Quality Assurance Cycle (MANDATORY) - -#### Step 5.1: Build Verification (BLOCKER) - -```bash -# Build node-doctor binary -make build-node-doctor-local - -# Verify binary exists and is executable -./bin/node-doctor --version -``` - -- **MUST PASS**: Zero compilation errors, zero linting errors -- If build fails, fix immediately before proceeding -- Build must complete successfully before moving to QA - -**TodoWrite Update**: Mark "Build verification" as completed, mark "QA review" as in_progress. - -#### Step 5.2: QA Agent Review (BLOCKER) - -- Invoke QA agent using Task tool with qa-engineer subagent -- Provide QA agent with: - - Changed files and context - - Original task requirements - - Success criteria - - Architecture document references (architecture.md, monitors.md, remediation.md) - -- QA agent will verify: - - Code quality and Go standards compliance - - Monitor interface implementation correct - - Remediator safety mechanisms implemented - - Configuration validation comprehensive - - Channel-based communication patterns correct - - Proper error handling and logging - - DaemonSet/RBAC configurations secure - - Integration with Problem Detector correct - - Security considerations (privileged access, input validation) - - Test coverage adequate (>80%) - -- **BLOCKER**: Address ALL QA findings before proceeding - -**TodoWrite Update**: Mark "QA review" as completed, mark "Testing" as in_progress. - -#### Step 5.3: Run Full Test Suite - -```bash -# Component tests -make test-node-doctor-local - -# Full validation (mirrors CI/CD pipeline exactly) -make validate-pipeline-local - -# Quick validation (format, vet, lint only - no tests) -make validate-quick - -# Race detector -go test -race -v ./... -``` - -**Validation checks:** -- Code formatting (gofmt) -- Static analysis (go vet) -- Advanced linting (staticcheck) -- Security scanning (gosec) -- Tests with race detection -- Coverage analysis - -- Report test coverage and results -- **If tests fail**: Fix issues, re-run QA review if needed, re-test -- Test results must include: files tested, pass/fail counts, coverage % - -See [Validation Quick Reference](validation-quick-reference.md) for troubleshooting - -**TodoWrite Update**: Mark "Testing" as completed, mark "Deployment" as in_progress. - -### 6. Deployment (MANDATORY) - -For Node Doctor, deployment means applying Kubernetes manifests to the cluster: - -```bash -# Build and push container image -make build-image-node-doctor VERSION=v$(date +%s) -make push-image-node-doctor VERSION=v$(date +%s) - -# Update DaemonSet manifest with new image version -# (Or use kustomize/helm for version management) - -# Apply Kubernetes resources -kubectl apply -f deploy/rbac.yaml -kubectl apply -f deploy/configmap.yaml -kubectl apply -f deploy/daemonset.yaml - -# Verify DaemonSet rollout -kubectl rollout status daemonset/node-doctor -n kube-system - -# Check pod status on all nodes -kubectl get pods -n kube-system -l app=node-doctor -o wide - -# Verify monitoring is active (check logs) -kubectl logs -n kube-system -l app=node-doctor --tail=50 -``` - -#### Deployment Success Criteria -- DaemonSet manifest applies without errors -- Pods created on all nodes (or selected nodes if nodeSelector used) -- All pods reach Running state -- Health checks pass (if implemented) -- Monitors start and begin reporting status -- Exporters publish metrics to Prometheus -- Kubernetes events created successfully -- No CrashLoopBackOff or ImagePullBackOff errors - -**If deployment fails, enter fix cycle (fix → build → QA → deploy)** - -**TodoWrite Update**: Mark "Deployment" as completed, mark "Devils advocate verification" as in_progress. - -### 7. Devils Advocate Verification (MANDATORY) - -- Use Task tool with devils-advocate-qa subagent -- Provide complete context: - - Original task specification - - Implementation details - - Test results (coverage, pass/fail) - - Deployment status - - Monitor/remediator behavior observations - -- Devils advocate will verify: - - Original request was fully fulfilled - - Business requirements met - - No edge cases missed - - Implementation matches task specification - - All success criteria satisfied - - Safety mechanisms working correctly - - No security issues introduced - - DaemonSet deployment stable - -#### If issues found: Enter Quality Cycle -- Fix identified issues -- Build verification (`make build-node-doctor-local`) -- QA agent re-review -- Re-test (`make test-node-doctor-local`) -- Re-deploy (kubectl apply) -- Devils advocate re-verification -- **STAY IN CYCLE** until fully resolved - -**TodoWrite Update**: When devils advocate approves, mark "Devils advocate verification" as completed, mark "Task completion" as in_progress. - -### 8. Task Completion (ONLY AFTER DEVILS ADVOCATE APPROVAL) - -#### Provide completion summary: -- **Files changed/created/deleted**: - - Monitors: `pkg/monitors/[name].go` and tests - - Remediators: `pkg/remediators/[name].go` and tests - - Configuration: `pkg/config/types.go` updates - - Deployment: `deploy/*.yaml` manifests - - Documentation: `docs/*.md` updates - -- **Tests written/updated**: - - Unit tests with coverage metrics (target: >80%) - - Integration tests results - - E2E test scenarios - -- **Test results**: All passing with coverage report - -- **Deployment confirmation**: - - DaemonSet deployed successfully - - Running on X nodes - - Monitors active and reporting - - Exporters publishing metrics - -- **Safety verification** (for remediator tasks): - - Cooldown periods enforced - - Circuit breaker tested - - Rate limiting active - - Attempt limits respected - -- Any warnings or notes about behavior or limitations - -#### Update TaskForge: -```javascript -// Mark task as completed -mcp__taskforge__updateTask({ - taskId: , - status: "done", - actualHours: -}); - -// This automatically records completion timestamp in database -// Task is now marked complete across all sessions -``` - -#### Commit with proper format: -```bash -cd /home/mmattox/go/src/github.com/supporttools/node-doctor - -git add . -git commit -m "feat(monitors): add CPU throttling monitor - -- Implement CPUMonitor with throttling detection -- Add systemd remediator with safety mechanisms -- Update DaemonSet with host access permissions -- Add comprehensive unit tests (87% coverage) -- Add integration tests for Problem Detector -- Task completion: [TaskForge task description]" - -git push origin main -``` - -**TodoWrite Update**: Mark "Task completion" as completed. Clean up todo list - all workflow steps done. - -**Await next task assignment** - ---- - -## Node Doctor-Specific Requirements - -### ABSOLUTE RULES - -1. **ONE TASK AT A TIME** - Never work on multiple tasks simultaneously -2. **RESEARCH BEFORE CODING** - Mandatory research phase, no exceptions -3. **DESIGN COMPLIANCE** - Must follow architecture.md specifications -4. **GO 1.21+ ONLY** - Verify version in go.mod file -5. **BUILD VERIFICATION** - Must pass before QA review -6. **QA VALIDATION** - Must pass before deployment -7. **DEPLOYMENT MANDATORY** - Must deploy DaemonSet to cluster -8. **DEVILS ADVOCATE** - Must pass before completion -9. **QUALITY CYCLE** - Stay in cycle until fully resolved -10. **PRIORITY ENFORCEMENT** - Never skip priority levels (P1 → P2 → P3 → P4 → P5) -11. **SAFETY FIRST** - All remediators must implement safety mechanisms - -### FORBIDDEN ACTIONS - -- Starting new task before current fully completed -- Skipping workflow steps -- Proceeding with failed builds -- Ignoring QA findings -- Bypassing deployment -- Marking complete without devils advocate approval -- Creating files in repository root (except allowed config/docs) -- Using Go version < 1.21 -- Implementing remediators without safety mechanisms -- Skipping cooldown periods in remediators -- Disabling circuit breakers or rate limiters -- Running privileged operations without input validation -- Ignoring RBAC boundaries in Kubernetes operations - -### SAFETY-CRITICAL RULES (Remediators) - -**Every remediator MUST implement:** -1. **Cooldown Period**: Minimum time between remediation attempts (default: 5 minutes) -2. **Attempt Limiting**: Maximum attempts per time window (default: 3 per hour) -3. **Circuit Breaker**: Stop attempting after consecutive failures (default: 5 failures) -4. **Rate Limiting**: Global rate limit across all problems (default: 10 per minute) -5. **Input Validation**: Validate ALL inputs before system calls -6. **Audit Logging**: Log ALL remediation attempts with outcomes -7. **Dry-Run Mode**: Support dry-run testing without actual changes - -**Never:** -- Execute system commands without validation -- Disable safety mechanisms "temporarily" -- Skip cooldown periods for "urgent" fixes -- Ignore circuit breaker state -- Bypass rate limiting - -### KUBERNETES DEPLOYMENT RULES - -1. **DaemonSet Only**: Node Doctor must run as DaemonSet in kube-system -2. **RBAC Minimal**: Grant only necessary permissions, follow least privilege -3. **Resource Limits**: Always specify CPU/memory limits -4. **Node Selection**: Support nodeSelector/tolerations for selective deployment -5. **Config Management**: Use ConfigMaps for configuration, not environment variables -6. **Secrets Management**: Never hardcode credentials, use Kubernetes Secrets -7. **Health Checks**: Implement liveness and readiness probes - -### EMERGENCY PROCEDURES - -If workflow cannot be completed due to infrastructure issues: - -1. Document blocker in task description -2. Use TaskForge to track blocker: - ```javascript - mcp__taskforge__createTask({ - projectId: 41, - featureId: 250, // Core Framework or appropriate feature - name: "Blocker: [description]", - priority: "urgent", - status: "todo" - }); - ``` -3. Use `mcp__taskforge__setDependency` to link blocked task -4. Do not start new tasks until blocker resolved -5. Escalate for assistance if needed - -If safety mechanisms trigger (circuit breaker, rate limits): -1. **DO NOT disable safety mechanisms** -2. Investigate root cause of failures -3. Fix underlying issue, not the safety mechanism -4. Reset circuit breaker only after verifying fix -5. Document incident and lessons learned - ---- - -## Common Node Doctor Scenarios - -### Scenario 1: Adding New Monitor - -**Files affected**: `pkg/monitors/`, tests, config, docs - -**Workflow**: -1. Create monitor implementation (`pkg/monitors/[name].go`) -2. Implement Monitor interface (Start, Stop methods) -3. Use channels for status communication -4. Write comprehensive unit tests -5. Add configuration type and validation -6. Register monitor in orchestrator -7. Update ConfigMap with monitor config -8. Test in local cluster -9. Update docs/monitors.md - -### Scenario 2: Adding New Remediator - -**Files affected**: `pkg/remediators/`, tests, config, docs - -**Workflow**: -1. Create remediator implementation (`pkg/remediators/[name].go`) -2. Implement Remediator interface (CanRemediate, Remediate, GetCooldown) -3. **MANDATORY**: Implement ALL safety mechanisms - - Circuit breaker - - Rate limiter - - Cooldown tracker - - Attempt counter -4. Write tests for safety mechanisms -5. Write tests for remediation logic -6. Add configuration type and validation -7. Register remediator in orchestrator -8. Test in isolated environment first -9. Update deploy/rbac.yaml if new permissions needed -10. Update docs/remediation.md - -### Scenario 3: Adding New Exporter - -**Files affected**: `pkg/exporters/`, tests, config, deployment - -**Workflow**: -1. Create exporter implementation -2. Implement Exporter interface -3. Add metrics definitions (Prometheus) or event types (Kubernetes) -4. Write tests for metric accuracy -5. Update DaemonSet with necessary service ports -6. Create Service resource if needed (for Prometheus scraping) -7. Test metric publishing -8. Update documentation - -### Scenario 4: Updating DaemonSet Configuration - -**Files affected**: `deploy/daemonset.yaml`, `deploy/configmap.yaml` - -**Workflow**: -1. Modify DaemonSet manifest -2. Update resource limits if needed -3. Add volume mounts for new host access -4. Update RBAC if new permissions required -5. Test deployment in dev cluster -6. Verify backward compatibility -7. Document changes in deployment guide -8. Roll out with `kubectl apply` - -### Scenario 5: Performance Optimization - -**Files affected**: Multiple `pkg/` files, tests - -**Workflow**: -1. Profile current performance (CPU, memory, goroutines) -2. Identify bottlenecks -3. Implement optimizations -4. Verify no functionality regression -5. Run benchmark tests -6. Update resource limits in DaemonSet -7. Document performance improvements - ---- - -## Quality Gates Summary - -| Phase | Gate | Blocker | Tool | -|-------|------|---------|------| -| Research | Design approved | Yes | Manual approval | -| Build | Binary compiles | Yes | `make build-node-doctor-local` | -| QA | Agent review passes | Yes | Task tool (qa-engineer) | -| Test | All tests pass | Yes | `make validate-pipeline-local` | -| Deploy | DaemonSet healthy | Yes | `kubectl apply` + rollout status | -| Verify | Devils advocate approves | Yes | Task tool (devils-advocate-qa) | -| Complete | All gates passed | Yes | TaskForge update + commit | - ---- - -## Workflow Diagram - -``` -┌─────────────────────────────────────────────────────────────┐ -│ 1. Receive Task │ -│ - Verify priority & dependencies (TaskForge) │ -│ - Update status to "in_progress" │ -│ - Create TodoWrite breakdown │ -└────────────────────────â”Ŧ────────────────────────────────────┘ - │ - â–ŧ -┌─────────────────────────────────────────────────────────────┐ -│ 2. Research & Design Phase (MANDATORY) │ -│ - Review architecture.md, monitors.md, remediation.md │ -│ - Identify integration points │ -│ - Design safety mechanisms │ -│ - Present implementation plan │ -└────────────────────────â”Ŧ────────────────────────────────────┘ - │ - â–ŧ -┌─────────────────────────────────────────────────────────────┐ -│ 3. Wait for Approval (GATE) │ -│ ✅ Approve → Continue │ -│ 🔄 Revise → Back to Step 2 │ -│ â­ī¸ Skip → Await new task │ -└────────────────────────â”Ŧ────────────────────────────────────┘ - │ (approved) - â–ŧ -┌─────────────────────────────────────────────────────────────┐ -│ 4. Implementation Phase │ -│ - Implement monitors/remediators/exporters │ -│ - Follow coding standards │ -│ - Write tests alongside code (TDD) │ -│ - Implement safety mechanisms │ -└────────────────────────â”Ŧ────────────────────────────────────┘ - │ - â–ŧ -┌─────────────────────────────────────────────────────────────┐ -│ 5.1 Build Verification (BLOCKER) │ -│ make build-node-doctor-local │ -│ ❌ Failed → Fix → Retry │ -└────────────────────────â”Ŧ────────────────────────────────────┘ - │ (passed) - â–ŧ -┌─────────────────────────────────────────────────────────────┐ -│ 5.2 QA Agent Review (BLOCKER) │ -│ Review code quality, safety, security │ -│ ❌ Issues found → Fix → Retry │ -└────────────────────────â”Ŧ────────────────────────────────────┘ - │ (passed) - â–ŧ -┌─────────────────────────────────────────────────────────────┐ -│ 5.3 Run Test Suite (BLOCKER) │ -│ make validate-pipeline-local │ -│ ❌ Tests failed → Fix → Back to 5.2 │ -└────────────────────────â”Ŧ────────────────────────────────────┘ - │ (passed) - â–ŧ -┌─────────────────────────────────────────────────────────────┐ -│ 6. Deployment (MANDATORY) │ -│ kubectl apply -f deploy/ │ -│ kubectl rollout status daemonset/node-doctor │ -│ ❌ Failed → Fix → Back to 5.1 │ -└────────────────────────â”Ŧ────────────────────────────────────┘ - │ (deployed) - â–ŧ -┌─────────────────────────────────────────────────────────────┐ -│ 7. Devils Advocate Verification (MANDATORY) │ -│ Verify task completion and safety │ -│ ❌ Issues found → Back to 5.1 (Quality Cycle) │ -└────────────────────────â”Ŧ────────────────────────────────────┘ - │ (approved) - â–ŧ -┌─────────────────────────────────────────────────────────────┐ -│ 8. Task Completion │ -│ - Provide completion summary │ -│ - Update TaskForge status to "done" │ -│ - Commit with proper message format │ -│ - Clean up TodoWrite │ -│ - Await next task │ -└─────────────────────────────────────────────────────────────┘ -``` - -### Monitor/Remediator Development Flow - -``` -New Feature Request - │ - â–ŧ -┌──────────────────────────────────────────────────────────────┐ -│ Research & Design Phase │ -│ - Review similar monitors/remediators │ -│ - Design safety mechanisms (for remediators) │ -│ - Plan channel communication │ -│ - Design configuration schema │ -└───────────────────────â”Ŧ──────────────────────────────────────┘ - │ - â–ŧ -┌──────────────────────────────────────────────────────────────┐ -│ Implementation │ -│ - pkg/monitors/[name].go OR pkg/remediators/[name].go │ -│ - pkg/monitors/[name]_test.go (unit tests) │ -│ - pkg/config/types.go (config schema) │ -│ - tests/integration/[name]_test.go │ -└───────────────────────â”Ŧ──────────────────────────────────────┘ - │ - â–ŧ -┌──────────────────────────────────────────────────────────────┐ -│ Quality Assurance │ -│ - Build verification │ -│ - QA agent review │ -│ - Full test suite (unit, integration, e2e) │ -│ - Safety mechanism validation (remediators) │ -└───────────────────────â”Ŧ──────────────────────────────────────┘ - │ - â–ŧ -┌──────────────────────────────────────────────────────────────┐ -│ Deployment to Cluster │ -│ - Update deploy/configmap.yaml with new config │ -│ - Update deploy/rbac.yaml if new permissions needed │ -│ - kubectl apply -f deploy/ │ -│ - Verify DaemonSet rollout │ -│ - Monitor logs for new feature activity │ -└───────────────────────â”Ŧ──────────────────────────────────────┘ - │ - â–ŧ -┌──────────────────────────────────────────────────────────────┐ -│ Devils Advocate Verification │ -│ - Verify feature works as designed │ -│ - Verify safety mechanisms active (remediators) │ -│ - Verify no regressions in existing monitors │ -│ - Check resource usage impact │ -└───────────────────────â”Ŧ──────────────────────────────────────┘ - │ - â–ŧ - Feature Complete - Monitor active on all nodes -``` - ---- - -This workflow ensures systematic, quality-driven development aligned with Node Doctor architecture and Kubernetes best practices. All work follows the **Safety First** principle, especially for remediators with privileged access to nodes. diff --git a/docs/development/validation-quick-reference.md b/docs/development/validation-quick-reference.md deleted file mode 100644 index cd60680..0000000 --- a/docs/development/validation-quick-reference.md +++ /dev/null @@ -1,445 +0,0 @@ -# Local CI/CD Validation - Quick Reference - -Quick reference guide for using the local validation system to catch issues before pushing. - -## âš ī¸ CRITICAL: Validation Mode Selection - -**ALWAYS use FULL validation before marking tasks complete or pushing to remote!** - -- **Quick Mode** (`--quick`): Skips gosec security scans and tests - - ✅ Use during development for fast feedback - - ❌ **NEVER** use as final validation before commit/push - -- **Full Mode** (default): Runs ALL checks including gosec and tests - - ✅ **MANDATORY** before pushing to remote - - ✅ **MANDATORY** before marking task complete - - ✅ Mirrors CI/CD pipeline exactly - -**Lesson Learned**: Using quick mode as final validation led to CI/CD failures that could have been caught locally. Always run full validation as the last step! - -## Before You Push - -### Full Validation (Recommended) - -```bash -# Validate everything - mirrors CI/CD completely -make validate-pipeline-local - -# Or directly -./scripts/validate-pipeline-local.sh -``` - -**When to use:** Before pushing to main/master, before creating PR - -**Time:** ~2-5 minutes (depends on changed components) - -### Quick Validation - -```bash -# Skip tests and gosec - fast feedback -make validate-quick - -# Or directly -./scripts/validate-pipeline-local.sh --quick -``` - -**When to use:** Before committing, during development, frequent checks - -**Time:** ~30 seconds - -### Component-Specific - -```bash -# Validate only one component -make validate-component COMPONENT=api - -# Or directly -./scripts/validate-pipeline-local.sh --component=ingestor -``` - -**Valid components:** api, org-management-controller, probe-controller, ingestor, customer-probe-agent, pkg - -**When to use:** Working on specific component, debugging validation failures - -**Time:** ~20 seconds per component - -## Common Commands - -### Makefile Targets - -```bash -make validate-pipeline-local # Full validation -make validate-quick # Quick check (no tests) -make validate-pre-push # Alias for full validation -make validate-component COMPONENT=api # Single component -``` - -### Script Options - -```bash -./scripts/validate-pipeline-local.sh --quick # Skip tests -./scripts/validate-pipeline-local.sh --component=api # One component -./scripts/validate-pipeline-local.sh --sequential # Run one at a time -./scripts/validate-pipeline-local.sh --fail-fast # Stop on first error -./scripts/validate-pipeline-local.sh --help # Show all options -``` - -## Common Issues and Fixes - -### Format Issues - -**Error:** -``` -✗ [component] format FAILED -pkg/foo/bar.go -``` - -**Fix:** -```bash -# Auto-fix formatting -gofmt -w . - -# Or specific file -gofmt -w pkg/foo/bar.go -``` - -### Go Vet Errors - -**Error:** -``` -✗ [component] go-vet FAILED -composite literal uses unkeyed fields -``` - -**Fix:** Review the error, fix the code issue. These are real bugs that need fixing. - -### Staticcheck Warnings - -**Error:** -``` -✗ [component] staticcheck FAILED -pkg/foo/bar.go:42:2: field token is unused (U1000) -``` - -**Fix Option 1 - Suppress (if intentional):** -```go -type Foo struct { - token string //lint:ignore U1000 Reserved for future JWT auth -} -``` - -**Fix Option 2 - Remove unused code:** -```go -// Delete the unused field/function -``` - -### Gosec Security Warnings - -**Error:** -``` -✗ [component] gosec FAILED -[G115] integer overflow conversion (CWE-190) -``` - -**Fix Option 1 - Suppress (if validated):** -```go -//nolint:gosec // G115 - Validated: orgID range checked before conversion -orgID := uint64(unsafeValue) -``` - -**Fix Option 2 - Use safe conversion:** -```go -import "github.com/{{PROJECT_NAME}}/{{PROJECT_NAME}}/pkg/utils" - -orgID := utils.SafeUintToUint64(value) // Handles overflow -``` - -### Test Failures - -**Error:** -``` -✗ [component] tests FAILED -FAIL: TestFoo (0.00s) -``` - -**Fix:** Review test output, fix the failing test or code - -**Run specific test:** -```bash -go test -v ./pkg/path/to/package -run TestName -``` - -### Race Detector Failures - -**Error:** -``` -WARNING: DATA RACE -Write at 0x... by goroutine X -``` - -**Fix:** This is a real concurrency bug - MUST be fixed - -**Never suppress race detector warnings!** - -## Component-Specific Commands - -### API - -```bash -cd /path/to/{{PROJECT_NAME}} -gofmt -l api/ -cd api && go vet ./... -cd api && staticcheck ./... -cd api && gosec -quiet ./... -cd api && go test -v -race ./... -``` - -### pkg - -```bash -gofmt -l pkg/ -go vet ./pkg/... -staticcheck ./pkg/... -gosec -quiet ./pkg/... -go test -v -race ./pkg/... -``` - -### org-management-controller - -```bash -gofmt -l pkg/controller/org-management/ -go vet ./pkg/controller/org-management/... -staticcheck ./pkg/controller/org-management/... -gosec -quiet ./pkg/controller/org-management/... -go test -v -race ./pkg/controller/org-management/... -``` - -### probe-controller - -```bash -gofmt -l pkg/probe/ -go vet ./pkg/probe/... -staticcheck ./pkg/probe/... -gosec -quiet ./pkg/probe/... -go test -v -race ./pkg/probe/... -``` - -### ingestor - -```bash -gofmt -l pkg/ingestor/ -go vet ./pkg/ingestor/... -staticcheck ./pkg/ingestor/... -gosec -quiet ./pkg/ingestor/... -go test -v -race ./pkg/ingestor/... -``` - -### customer-probe-agent - -**Note:** Has separate go.mod - must cd into directory - -```bash -gofmt -l cmd/customer-probe-agent/ -cd cmd/customer-probe-agent && go vet ./... -cd cmd/customer-probe-agent && staticcheck ./... -cd cmd/customer-probe-agent && gosec -quiet ./... -cd cmd/customer-probe-agent && go test -v -race ./... -``` - -## Enable Pre-Push Hook - -### Automatic Validation on Push - -```bash -# Run setup script -./.githooks/setup.sh - -# Or manually configure -git config core.hooksPath .githooks -``` - -### Verify Hook Enabled - -```bash -git config --get core.hooksPath -# Should output: .githooks -``` - -### Bypass Hook (Emergency Only) - -```bash -git push --no-verify -``` - -**Warning:** Only use --no-verify for emergencies! - -## Troubleshooting - -### "directory prefix does not contain main module" - -**Issue:** customer-probe-agent has separate go.mod - -**Fix:** Always cd into directory: -```bash -cd cmd/customer-probe-agent && go vet ./... -``` - -### "couldn't parse format string" - -**Issue:** Staticcheck false positive on `%.2f%%` - -**Fix:** Add suppression: -```go -//lint:ignore SA5009 staticcheck false positive - format string valid -return fmt.Sprintf("%.2f%%", value) -``` - -### Race Detector Takes Too Long - -**Solution:** Use quick mode during development: -```bash -make validate-quick # Skips race detector -``` - -Run full validation before push: -```bash -make validate-pre-push # Includes race detector -``` - -### Validation Hangs - -**Solution:** Run sequential instead of parallel: -```bash -./scripts/validate-pipeline-local.sh --sequential -``` - -### Need to Validate Only Changed Code - -**Solution:** Validate specific component: -```bash -# If you only changed ingestor code -make validate-component COMPONENT=ingestor -``` - -## Best Practices - -### Development Workflow - -1. **During development:** `make validate-quick` (frequent checks) -2. **Before commit:** `make validate-quick` (ensure no format/lint issues) -3. **Before push:** `make validate-pre-push` (full validation with tests - **MANDATORY**) -4. **Before marking task complete:** Full validation must pass - no exceptions! - -**âš ī¸ WARNING**: Quick mode skips gosec security scans and tests. These WILL fail in CI/CD if you don't run full validation locally! - -### Time-Saving Tips - -1. **Use quick mode frequently** - catches 80% of issues in 30 seconds -2. **Validate specific components** - faster feedback when working on one area -3. **Enable pre-push hook** - prevents accidental pushes with issues -4. **Fix format issues first** - fastest to fix, unblocks other checks - -### When to Bypass Validation - -**Only bypass in these situations:** -- Emergency hotfix (non-code changes) -- Infrastructure-only changes (Helm charts, docs) -- You've already validated manually -- CI/CD pipeline will catch the issues anyway (not recommended!) - -**Never bypass for:** -- Code changes -- "It works on my machine" -- "CI/CD is slow" -- Time pressure (fix the issues!) - -## Examples - -### Example 1: Quick Development Check - -```bash -# Make some code changes -vim pkg/api/handlers/foo.go - -# Quick check before committing -make validate-quick - -# If passes, commit -git add . -git commit -m "feat: add new handler" -``` - -### Example 2: Full Pre-Push Validation - -```bash -# Finished feature, ready to push -make validate-pre-push - -# All checks pass -git push origin feature-branch -``` - -### Example 3: Fix Validation Failures - -```bash -# Validation fails on format -make validate-quick -# ✗ [api] format FAILED - -# Fix formatting -gofmt -w api/ - -# Re-validate -make validate-quick -# ✓ All checks pass - -# Push -git push origin main -``` - -### Example 4: Component-Specific Development - -```bash -# Working on ingestor only -vim pkg/ingestor/server/metrics.go - -# Validate just ingestor -make validate-component COMPONENT=ingestor - -# Passes, commit and continue -git commit -am "feat(ingestor): improve metrics collection" -``` - -## Validation Checklist - -Before pushing code, ensure: - -- [ ] Code is properly formatted (`gofmt`) -- [ ] No vet warnings (real bugs) -- [ ] No staticcheck issues (or properly suppressed) -- [ ] No security issues (gosec) -- [ ] All tests passing -- [ ] No race conditions detected -- [ ] Validation script returns exit code 0 - -## Need More Help? - -- **Full documentation:** [local-cicd-validation-guide.md](local-cicd-validation-guide.md) -- **Git hooks:** [.githooks/README.md](../../.githooks/README.md) -- **Task workflow:** [task-execution-workflow.md](task-execution-workflow.md) -- **CI/CD pipeline:** [.github/workflows/pipeline-v2.yml](../../.github/workflows/pipeline-v2.yml) - -## Quick Commands Summary - -```bash -# Most common commands -make validate-quick # Fast check -make validate-pipeline-local # Full check -make validate-component COMPONENT=api # One component -./.githooks/setup.sh # Enable pre-push hook -git push --no-verify # Bypass hook (emergency) -``` - ---- - -**Remember:** Validation saves time by catching issues early. A few minutes of local validation prevents hours of CI/CD debugging! diff --git a/docs/release-process.md b/docs/release-process.md index 8b2834b..1583d7d 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -754,4 +754,4 @@ Listed explicitly because previous versions of this document claimed otherwise: - [Helm rollback](https://helm.sh/docs/helm/helm_rollback/) - [GitHub Actions Workflows](../.github/workflows/) - [Node Doctor Architecture](./architecture.md) -- [Deployment Guide](./deployment.md) +- [Deployment Guide](../deployment/README.md) diff --git a/scripts/check-doc-links.sh b/scripts/check-doc-links.sh new file mode 100755 index 0000000..0b60405 --- /dev/null +++ b/scripts/check-doc-links.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Reports relative markdown links whose target does not exist. Exits 1 if any are broken. +set -u +cd "$(dirname "$0")/.." || exit 2 + +status=0 +while IFS= read -r file; do + dir=$(dirname "$file") + while IFS= read -r target; do + target=${target%% \"*} + target=${target%%#*} + [ -z "$target" ] && continue + case "$target" in http://*|https://*|mailto:*) continue ;; esac + if [ ! -e "$dir/$target" ]; then + echo "$file: broken -> $target" + status=1 + fi + done < <(grep -o '](<\{0,1\}[^)]*)' "$file" | sed 's/^](<\{0,1\}//; s/>\{0,1\})$//') +done < <(find . -name '*.md' -not -path './.git/*' -not -path './.claude/*' | sort) + +exit $status From a177bda669079ee745ff775d8717473f73f2e445 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Wed, 7 Oct 2026 09:39:07 -0500 Subject: [PATCH 3/6] build: make local validation real, align coverage, remove template leftovers scripts/validate-pipeline-local.sh only had case branches for another project's components, so it ran zero checks and reported success. It now runs gofmt, GOOS=linux go vet, golangci-lint (SKIP_LINT=1 to allow it missing), gosec when installed, go test -short -race, helm lint and helm-verify-generated, and exits non-zero on any failure. Makefile: validate-component passes --component=; test-e2e uses -tags=e2e and checks for kind; COVERAGE_THRESHOLD is 70 and ci.yml calls make coverage-threshold so there is one source; docker-push and push-*-image require an explicit VERSION and never tag or push :latest. Removed the template header, helm-publish stub, deploy-*, bump*, qa-check, devils-advocate, check-kubeconfig-* and their variables. Deleted dead template files: init-from-template.sh, substitute-variables.sh, validate-api.sh, the GPG scripts, .githooks pre-push and setup.sh, hooks/, .validation.json, .version-rc. validate-repo-cleanliness.sh no longer rejects the repo's own Dockerfile.overlay-test. Co-Authored-By: Claude Fable 5.1 --- .githooks/pre-push | 44 --- .githooks/setup.sh | 63 ---- .github/workflows/ci.yml | 24 +- .validation.json | 149 --------- .version-rc | 1 - CONTRIBUTING.md | 2 +- Makefile | 324 +++---------------- docs/release-process.md | 62 +--- hooks/coding/basic-automation.toml | 24 -- hooks/writing/content-automation.toml | 39 --- scripts/README.md | 364 +-------------------- scripts/init-from-template.sh | 121 ------- scripts/setup-gpg-signing-subkey.sh | 253 --------------- scripts/substitute-variables.sh | 135 -------- scripts/validate-api.sh | 5 - scripts/validate-pipeline-local.sh | 435 ++++++-------------------- scripts/validate-repo-cleanliness.sh | 17 - scripts/verify-gpg-setup.sh | 185 ----------- 18 files changed, 158 insertions(+), 2089 deletions(-) delete mode 100755 .githooks/pre-push delete mode 100755 .githooks/setup.sh delete mode 100644 .validation.json delete mode 100644 .version-rc delete mode 100644 hooks/coding/basic-automation.toml delete mode 100644 hooks/writing/content-automation.toml delete mode 100755 scripts/init-from-template.sh delete mode 100755 scripts/setup-gpg-signing-subkey.sh delete mode 100755 scripts/substitute-variables.sh delete mode 100755 scripts/validate-api.sh delete mode 100755 scripts/verify-gpg-setup.sh diff --git a/.githooks/pre-push b/.githooks/pre-push deleted file mode 100755 index 69ffaeb..0000000 --- a/.githooks/pre-push +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -# Pre-push hook that validates all components before pushing -# This prevents CI/CD failures by catching issues locally - -# TEMPORARILY DISABLED - blocking large data transfers -exit 0 - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -echo "" -echo -e "${BLUE}🔍 Running pre-push validation...${NC}" -echo "" - -# Run full pipeline validation -./scripts/validate-pipeline-local.sh - -RESULT=$? - -echo "" - -if [ $RESULT -ne 0 ]; then - echo -e "${RED}╔════════════════════════════════════════════════════════════════╗${NC}" - echo -e "${RED}║ ❌ Pre-push validation FAILED ║${NC}" - echo -e "${RED}╚════════════════════════════════════════════════════════════════╝${NC}" - echo "" - echo -e "${YELLOW}Fix the issues above before pushing.${NC}" - echo "" - echo -e "${YELLOW}To bypass this check (NOT RECOMMENDED):${NC}" - echo -e "${YELLOW} git push --no-verify${NC}" - echo "" - exit 1 -fi - -echo -e "${GREEN}╔════════════════════════════════════════════════════════════════╗${NC}" -echo -e "${GREEN}║ ✅ Pre-push validation passed - proceeding with push ║${NC}" -echo -e "${GREEN}╚════════════════════════════════════════════════════════════════╝${NC}" -echo "" - -exit 0 diff --git a/.githooks/setup.sh b/.githooks/setup.sh deleted file mode 100755 index 10ecfc9..0000000 --- a/.githooks/setup.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -# Setup script to enable git hooks for validation - -# Colors for output -GREEN='\033[0;32m' -BLUE='\033[0;34m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -echo "" -echo -e "${BLUE}╔════════════════════════════════════════════════════════════════╗${NC}" -echo -e "${BLUE}║ Git Hooks Setup for Nexmonyx ║${NC}" -echo -e "${BLUE}╚════════════════════════════════════════════════════════════════╝${NC}" -echo "" - -# Check if already configured -CURRENT_HOOKS_PATH=$(git config --get core.hooksPath) - -if [ "$CURRENT_HOOKS_PATH" = ".githooks" ]; then - echo -e "${GREEN}✓ Git hooks are already configured!${NC}" - echo "" - echo "Current configuration:" - echo " core.hooksPath = .githooks" - echo "" - exit 0 -fi - -echo -e "${BLUE}This will configure Git to use local validation hooks:${NC}" -echo "" -echo " â€ĸ Pre-push hook: Validates all components before pushing" -echo " â€ĸ Prevents CI/CD failures by catching issues locally" -echo " â€ĸ Can be bypassed with: git push --no-verify" -echo "" - -read -p "Enable git hooks? (y/n): " -n 1 -r -echo "" - -if [[ ! $REPLY =~ ^[Yy]$ ]]; then - echo -e "${YELLOW}Setup cancelled.${NC}" - echo "" - exit 0 -fi - -# Configure git to use .githooks directory -git config core.hooksPath .githooks - -echo "" -echo -e "${GREEN}╔════════════════════════════════════════════════════════════════╗${NC}" -echo -e "${GREEN}║ ✓ Git hooks configured successfully! ║${NC}" -echo -e "${GREEN}╚════════════════════════════════════════════════════════════════╝${NC}" -echo "" -echo "Configuration:" -echo " core.hooksPath = .githooks" -echo "" -echo "Active hooks:" -echo " â€ĸ pre-push: Full validation before push" -echo "" -echo -e "${BLUE}To disable hooks:${NC}" -echo " git config --unset core.hooksPath" -echo "" -echo -e "${BLUE}To bypass on specific push:${NC}" -echo " git push --no-verify" -echo "" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08aafe9..a7b5dac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,29 +107,9 @@ jobs: go tool cover -func=coverage-integration.out | grep total | awk '{print "Integration test coverage: " $3}' fi + # Threshold lives in the Makefile (COVERAGE_THRESHOLD) so local and CI gates agree. - name: Check coverage threshold - run: | - if [ ! -f coverage.out ]; then - echo "::error::Coverage file not found" - exit 1 - fi - - COVERAGE=$(go tool cover -func=coverage.out | grep total | awk '{print $3}' | sed 's/%//') - - if [ -z "$COVERAGE" ]; then - echo "::error::Failed to extract coverage percentage" - exit 1 - fi - - THRESHOLD=70 - echo "Current coverage: ${COVERAGE}%" - echo "Minimum threshold: ${THRESHOLD}%" - - if (( $(echo "$COVERAGE < $THRESHOLD" | bc -l) )); then - echo "::error::Coverage ${COVERAGE}% is below minimum threshold of ${THRESHOLD}%" - exit 1 - fi - echo "Coverage check passed!" + run: make coverage-threshold COVERAGE_PROFILE=coverage.out - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 diff --git a/.validation.json b/.validation.json deleted file mode 100644 index e7cfc1e..0000000 --- a/.validation.json +++ /dev/null @@ -1,149 +0,0 @@ -{ - "_comment": "Validation configuration for node-doctor local CI/CD validation", - "_info": "Adapted from Nexmonyx repo-template for single-binary Kubernetes DaemonSet project", - "validation": { - "enabled": true, - "parallel_jobs": 2, - "fail_fast": false, - "quick_mode": { - "enabled": true, - "skip_tests": true, - "skip_gosec": true, - "skip_coverage": true - } - }, - "components": { - "_comment": "node-doctor is a single binary Kubernetes monitoring tool", - "node-doctor": { - "enabled": true, - "path": "cmd/node-doctor", - "test_timeout": "10m", - "coverage_threshold": 50, - "coverage_exclude": [ - "cmd/node-doctor/main.go" - ] - } - }, - "tools": { - "gofmt": { - "enabled": true, - "check_only": true, - "fix": false - }, - "goimports": { - "enabled": false, - "local": "github.com/supporttools/node-doctor" - }, - "go_vet": { - "enabled": true, - "flags": [ - "-all" - ] - }, - "staticcheck": { - "enabled": true, - "checks": [ - "all", - "-ST1000", - "-ST1003" - ], - "fail_on": [ - "error", - "warning" - ] - }, - "gosec": { - "enabled": true, - "severity": "medium", - "confidence": "medium", - "exclude_rules": [ - "G104" - ], - "scan_tests": false - }, - "golangci_lint": { - "enabled": false, - "config": ".golangci.yml", - "timeout": "5m" - }, - "go_test": { - "enabled": true, - "flags": [ - "-v", - "-race", - "-cover" - ], - "timeout": "10m", - "coverage_mode": "atomic", - "coverage_profile": "coverage.txt" - } - }, - "repository": { - "go_version": "1.21", - "go_mod_tidy_check": true, - "cleanliness_check": true, - "allowed_root_files": [ - ".gitignore", - ".gitattributes", - ".validation.json", - "README.md", - "LICENSE", - "CLAUDE.md", - "CONTRIBUTING.md", - "PROJECT_SETUP_CHECKLIST.md", - "makefile", - "Makefile", - "Dockerfile", - "go.mod", - "go.sum" - ], - "forbidden_patterns": [ - "*.swp", - "*.swo", - "*~", - ".DS_Store", - "*.key", - "*.pem", - "credentials.json", - "secrets.yaml", - "kubeconfig" - ] - }, - "ci_cd": { - "mirror_github_actions": true, - "cache_enabled": true, - "cache_paths": [ - "~/.cache/go-build", - "~/go/pkg/mod" - ] - }, - "reporting": { - "verbose": true, - "show_passed": true, - "show_skipped": true, - "color_output": true, - "summary": true - }, - "pre_commit": { - "enabled": true, - "auto_fix": false, - "checks": [ - "gofmt", - "go_vet" - ] - }, - "pre_push": { - "enabled": true, - "run_full_validation": true, - "allow_skip": false, - "checks": [ - "go_version", - "cleanliness", - "gofmt", - "go_vet", - "staticcheck", - "gosec", - "tests" - ] - } -} diff --git a/.version-rc b/.version-rc deleted file mode 100644 index 1df767c..0000000 --- a/.version-rc +++ /dev/null @@ -1 +0,0 @@ -v1.5.5-rc3 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 635c5d3..137846e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -337,7 +337,7 @@ func (r *ServiceRemediator) GetCooldown() time.Duration { ### Test Coverage Expectations -- **Minimum Coverage**: 80% overall code coverage +- **Minimum Coverage**: 70% unit test coverage (`make coverage-check`; CI enforces the same threshold) - **Target Coverage**: 85%+ for production code - **New Code**: Must include tests for all new functionality diff --git a/Makefile b/Makefile index b3f2a8e..ae62296 100644 --- a/Makefile +++ b/Makefile @@ -1,32 +1,17 @@ -# ================================================================================================ -# Nexmonyx Repository Template - Makefile -# ================================================================================================ -# -# This makefile provides a complete build, test, and deployment framework for Go/Kubernetes projects. -# -# CUSTOMIZATION REQUIRED: -# 1. Replace {{PROJECT_NAME}} with your project name -# 2. Replace {{REGISTRY}} with your container registry (e.g., ghcr.io/myorg, harbor.mycompany.com/myproject) -# 3. Replace {{HELM_REPO}} with your Helm chart repository URL -# 4. Update COMPONENTS list with your actual components -# 5. Customize KUBECONFIG paths for your environments -# 6. Adjust build targets for your project structure -# -# ================================================================================================ +# node-doctor build, test, and local validation targets. +# Images and charts are published only by .github/workflows/release.yml. -.PHONY: help all test build deploy \ - build-local test-local validate-local validate-pipeline-local validate-quick \ - deploy-dev deploy-stg deploy-prd \ - bump bump-with-monitoring \ - qa-check devils-advocate workflow-status \ +.PHONY: help all test build \ + build-local test-local validate-local validate-pipeline-local validate-quick validate-component \ + workflow-status workflow-help gh-help \ gh-status gh-watch gh-logs gh-builds \ - check-prerequisites check-docker check-kubectl \ + check-prerequisites check-docker check-kubectl check-go-version require-version \ build test test-integration test-e2e test-all test-ci \ test-net-icmp-integration \ lint fmt clean install-deps \ docker-build docker-push \ - helm-lint helm-package helm-publish helm-generate helm-verify-generated \ - coverage-check + helm-lint helm-package helm-generate helm-verify-generated \ + coverage-check coverage-threshold # ================================================================================================ # Project Configuration @@ -44,38 +29,11 @@ GIT_COMMIT := $(shell git rev-parse HEAD) BUILD_TIME := $(shell date -u +"%Y-%m-%dT%H:%M:%SZ") GO_VERSION := 1.25 -# RC Version - for release candidate builds -RC_VERSION_FILE := .version-rc -RC_VERSION := $(shell [ -f $(RC_VERSION_FILE) ] && cat $(RC_VERSION_FILE) || echo "v0.1.0-rc.1") - export VERSION export GIT_COMMIT export BUILD_TIME export GO_VERSION -# ================================================================================================ -# Environment Configuration -# ================================================================================================ - -# CUSTOMIZE: Update these paths for your kubeconfig locations -KUBECONFIG_DEV := $(HOME)/.kube/dev-cluster -KUBECONFIG_STG := $(HOME)/.kube/staging-cluster -KUBECONFIG_PRD := $(HOME)/.kube/config - -# Kubernetes context for production deployment -KUBECTL_CONTEXT_PRD := a1-ops-prd - -export KUBECONFIG_DEV -export KUBECONFIG_STG -export KUBECONFIG_PRD - -# Kubernetes namespaces (node-doctor runs in kube-system typically) -NAMESPACE_DEV := kube-system -NAMESPACE_STG := kube-system -NAMESPACE_PRD := node-doctor - -# Helm Chart settings -HELM_REPO_URL := https://charts.support.tools CHART_VERSION := v$(shell git rev-list --count HEAD) # ================================================================================================ @@ -143,16 +101,11 @@ check-go-version: @./scripts/validate-go-version.sh @$(call print_success,"Go version check passed") -check-kubeconfig-%: - @$(call print_status,"Checking kubeconfig for $*...") - @ENV_UPPER=$$(echo $* | tr '[:lower:]' '[:upper:]') && \ - KUBECONFIG_VAR=KUBECONFIG_$$ENV_UPPER && \ - eval KUBECONFIG_PATH=\$$$$KUBECONFIG_VAR && \ - if [ ! -f "$$KUBECONFIG_PATH" ]; then \ - $(call print_error,"Kubeconfig not found for $*: $$KUBECONFIG_PATH"); \ - exit 1; \ - fi - @$(call print_success,"Kubeconfig validated for $*") +# Pushing images needs a real tag; the epoch default would litter the registry. +require-version: +ifeq ($(origin VERSION),file) + @$(call print_error,"Set VERSION explicitly: make docker-push VERSION=1.2.3"); exit 1 +endif # ================================================================================================ # Build Targets @@ -239,14 +192,12 @@ test-ci: fi @$(call print_success,"CI test gate passed") -# End-to-end tests +# E2E tests create a kind cluster; they need docker, kind, and kubectl. +# Env: E2E_KEEP_CLUSTER=1, E2E_EXPORT_LOGS=1, E2E_DEBUG=1 (see test/e2e/README.md). test-e2e: - @$(call print_status,"Running E2E tests...") - @if [ -d "test/e2e" ]; then \ - go test ./test/e2e/... -v -timeout 10m; \ - else \ - $(call print_warning,"E2E tests not yet implemented (test/e2e/ does not exist)"); \ - fi + @$(call print_status,"Running E2E tests (kind cluster)...") + @command -v kind >/dev/null 2>&1 || ($(call print_error,"kind not found") && exit 1) + @go test -tags=e2e ./test/e2e/... -v -timeout 30m @$(call print_success,"E2E tests completed") # Run the real ICMP pinger integration test under privilege. @@ -272,13 +223,21 @@ test-all: @go tool cover -func=coverage/coverage.out | grep total | awk '{print "Total coverage: " $$3}' @$(call print_success,"All tests passed - coverage report: coverage/coverage.html") -# Check coverage meets minimum threshold (80%) -COVERAGE_THRESHOLD := 80 +# Single source for the coverage gate; ci.yml calls coverage-threshold with its own profile. +COVERAGE_THRESHOLD := 70 +COVERAGE_PROFILE ?= coverage/coverage.out + coverage-check: - @$(call print_status,"Checking coverage threshold (minimum $(COVERAGE_THRESHOLD)%)...") - @mkdir -p coverage - @go test ./... -covermode=atomic -coverprofile=coverage/coverage.out -short > /dev/null 2>&1 - @COVERAGE=$$(go tool cover -func=coverage/coverage.out | grep total | awk '{print $$3}' | sed 's/%//'); \ + @$(call print_status,"Generating unit test coverage profile...") + @mkdir -p $(dir $(COVERAGE_PROFILE)) + @go test ./pkg/... ./cmd/... -short -covermode=atomic -coverprofile=$(COVERAGE_PROFILE) > /dev/null + @$(MAKE) coverage-threshold COVERAGE_PROFILE=$(COVERAGE_PROFILE) + +coverage-threshold: + @$(call print_status,"Checking $(COVERAGE_PROFILE) against minimum $(COVERAGE_THRESHOLD)%...") + @[ -f $(COVERAGE_PROFILE) ] || { $(call print_error,"$(COVERAGE_PROFILE) not found"); exit 1; } + @COVERAGE=$$(go tool cover -func=$(COVERAGE_PROFILE) | grep total | awk '{print $$3}' | sed 's/%//'); \ + [ -n "$$COVERAGE" ] || { $(call print_error,"Failed to extract coverage percentage"); exit 1; }; \ echo "Current coverage: $${COVERAGE}%"; \ if [ $$(echo "$${COVERAGE} < $(COVERAGE_THRESHOLD)" | bc -l) -eq 1 ]; then \ $(call print_error,"Coverage $${COVERAGE}% is below minimum threshold of $(COVERAGE_THRESHOLD)%"); \ @@ -327,8 +286,8 @@ install-deps: # Docker build shorthand docker-build: build-all-images -# Docker push shorthand -docker-push: push-all-images +# Docker push shorthand; never tags :latest, release.yml owns that +docker-push: require-version push-all-images # ================================================================================================ # Validation Targets (mirrors CI/CD pipeline) @@ -346,7 +305,7 @@ validate-quick: check-prerequisites check-go-version validate-component: @$(call print_status,"Validating component: $(COMPONENT)...") - @./scripts/validate-pipeline-local.sh --component $(COMPONENT) + @./scripts/validate-pipeline-local.sh --component=$(COMPONENT) @$(call print_success,"Component validation passed") validate-local: validate-pipeline-local @@ -359,14 +318,12 @@ validate-local: validate-pipeline-local build-node-doctor-image: check-docker @$(call print_status,"Building node-doctor Docker image...") @docker build -f Dockerfile -t $(DOCKER_IMAGE_node-doctor):$(VERSION) . - @docker tag $(DOCKER_IMAGE_node-doctor):$(VERSION) $(DOCKER_IMAGE_node-doctor):latest @$(call print_success,"node-doctor image built: $(DOCKER_IMAGE_node-doctor):$(VERSION)") # Build Docker image for overlay-test-server build-overlay-test-server-image: check-docker @$(call print_status,"Building overlay-test-server Docker image...") @docker build -f Dockerfile.overlay-test -t $(DOCKER_IMAGE_overlay-test-server):$(VERSION) . - @docker tag $(DOCKER_IMAGE_overlay-test-server):$(VERSION) $(DOCKER_IMAGE_overlay-test-server):latest @$(call print_success,"overlay-test-server image built: $(DOCKER_IMAGE_overlay-test-server):$(VERSION)") # Build all images @@ -374,17 +331,15 @@ build-all-images: build-node-doctor-image build-overlay-test-server-image @$(call print_success,"Docker images built successfully") # Push node-doctor image to registry -push-node-doctor-image: +push-node-doctor-image: require-version @$(call print_status,"Pushing node-doctor image to registry...") @docker push $(DOCKER_IMAGE_node-doctor):$(VERSION) - @docker push $(DOCKER_IMAGE_node-doctor):latest @$(call print_success,"node-doctor image pushed") # Push overlay-test-server image to registry -push-overlay-test-server-image: +push-overlay-test-server-image: require-version @$(call print_status,"Pushing overlay-test-server image to registry...") @docker push $(DOCKER_IMAGE_overlay-test-server):$(VERSION) - @docker push $(DOCKER_IMAGE_overlay-test-server):latest @$(call print_success,"overlay-test-server image pushed") push-all-images: push-node-doctor-image push-overlay-test-server-image @@ -446,143 +401,6 @@ helm-package: @helm package ./helm/$(PROJECT_NAME) --version $(CHART_VERSION) @$(call print_success,"Helm chart packaged") -helm-publish: helm-package - @$(call print_status,"Publishing Helm chart to repository...") - @# CUSTOMIZE: Add your Helm chart publish command - @$(call print_success,"Helm chart published") - -# ================================================================================================ -# Deployment Targets -# ================================================================================================ - -deploy-dev: check-kubeconfig-dev - @$(call print_status,"Deploying to development environment...") - @KUBECONFIG=$(KUBECONFIG_DEV) helm upgrade --install $(PROJECT_NAME) ./helm/$(PROJECT_NAME) \ - --namespace $(NAMESPACE_DEV) \ - --set image.tag=$(VERSION) \ - --set environment=dev \ - --wait - @$(call print_success,"Deployed to development") - -deploy-stg: check-kubeconfig-stg - @$(call print_status,"Deploying to staging environment...") - @KUBECONFIG=$(KUBECONFIG_STG) helm upgrade --install $(PROJECT_NAME) ./helm/$(PROJECT_NAME) \ - --namespace $(NAMESPACE_STG) \ - --set image.tag=$(VERSION) \ - --set environment=staging \ - --wait - @$(call print_success,"Deployed to staging") - -deploy-prd: check-kubeconfig-prd - @echo "âš ī¸ CRITICAL DECISION: Deploy to Production" - @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - @echo "What: Deploy $(PROJECT_NAME) version $(VERSION) to production" - @echo "Why: Approved for production deployment" - @echo "Risk: May impact production users" - @echo "" - @read -p "Do you approve? (y/n): " approve; \ - if [ "$$approve" != "y" ] && [ "$$approve" != "Y" ]; then \ - $(call print_error,"Production deployment cancelled"); \ - exit 1; \ - fi - @$(call print_status,"Deploying to production environment...") - @KUBECONFIG=$(KUBECONFIG_PRD) helm upgrade --install $(PROJECT_NAME) ./helm/$(PROJECT_NAME) \ - --namespace $(NAMESPACE_PRD) \ - --set image.tag=$(VERSION) \ - --set environment=production \ - --wait - @$(call print_success,"Deployed to production") - -# ================================================================================================ -# Version Bump and Deployment Workflow -# ================================================================================================ - -bump: validate-pipeline-local - @$(call print_status,"Creating new version bump...") - @$(call print_status,"Current VERSION: $(VERSION)") - @git add . && git commit -m "bump: automated version bump to $(VERSION) [skip ci]" || true - @git push origin $$(git rev-parse --abbrev-ref HEAD) - @$(call print_success,"Version bumped and pushed - CI/CD will handle deployment") - -bump-with-monitoring: bump - @$(call print_status,"Monitoring GitHub Actions workflow...") - @$(MAKE) gh-watch - -# Increment RC version (e.g., v0.1.0-rc.1 -> v0.1.0-rc.2) -increment-rc-version: - @$(call print_status,"Incrementing RC version...") - @CURRENT_RC=$$(cat $(RC_VERSION_FILE)); \ - BASE_VERSION=$$(echo $$CURRENT_RC | sed 's/-rc\.[0-9]*$$//'); \ - RC_NUM=$$(echo $$CURRENT_RC | sed 's/.*-rc\.//'); \ - NEW_RC_NUM=$$((RC_NUM + 1)); \ - NEW_RC_VERSION="$$BASE_VERSION-rc.$$NEW_RC_NUM"; \ - echo $$NEW_RC_VERSION > $(RC_VERSION_FILE); \ - echo "$(GREEN)✅ RC version updated: $$CURRENT_RC -> $$NEW_RC_VERSION$(NC)" - -# Deploy to production cluster using kubectl -deploy-prd-kubectl: check-kubectl - @$(call print_status,Deploying to production cluster $(KUBECTL_CONTEXT_PRD)...) - @$(call print_status,Checking namespace $(NAMESPACE_PRD)...) - @KUBECONFIG=$(KUBECONFIG_PRD) kubectl config use-context $(KUBECTL_CONTEXT_PRD) - @KUBECONFIG=$(KUBECONFIG_PRD) kubectl get namespace $(NAMESPACE_PRD) > /dev/null 2>&1 || \ - (echo "Creating namespace $(NAMESPACE_PRD)..." && \ - KUBECONFIG=$(KUBECONFIG_PRD) kubectl create namespace $(NAMESPACE_PRD)) - @$(call print_status,Applying RBAC resources...) - @KUBECONFIG=$(KUBECONFIG_PRD) kubectl apply -f deployment/rbac.yaml -n $(NAMESPACE_PRD) - @$(call print_status,Applying DaemonSet...) - @KUBECONFIG=$(KUBECONFIG_PRD) kubectl apply -f deployment/daemonset.yaml -n $(NAMESPACE_PRD) - @$(call print_status,Waiting for rollout to complete...) - @KUBECONFIG=$(KUBECONFIG_PRD) kubectl rollout status daemonset/node-doctor -n $(NAMESPACE_PRD) --timeout=5m - @$(call print_success,Deployed to production cluster $(KUBECTL_CONTEXT_PRD)) - -# Build and push RC release to $(REGISTRY) and deploy to a1-ops-prd cluster. -# -# This target deliberately does NOT tag :latest. It used to, which meant a workstation -# build of a *release candidate* could overwrite the :latest that release.yml publishes -# only for non-rc tags. Only .github/workflows/release.yml moves :latest. -# -# The git steps below are also not `|| true`: a failed commit/tag/push used to be -# swallowed, leaving an image pushed and a cluster deployed from a revision that was -# never tagged or pushed anywhere. -bump-rc: validate-pipeline-local increment-rc-version - @$(call print_status,Building RC release: $(RC_VERSION)) - @$(call print_status,Registry: $(REGISTRY)/$(PROJECT_NAME)) - @$(call print_status,Cluster: $(KUBECTL_CONTEXT_PRD) Namespace: $(NAMESPACE_PRD)) - @echo "" - @$(call print_status,Building multi-arch Docker image for amd64 and arm64...) - @docker buildx build \ - --platform linux/amd64,linux/arm64 \ - --build-arg VERSION=$(RC_VERSION) \ - --build-arg GIT_COMMIT=$(GIT_COMMIT) \ - --build-arg BUILD_TIME=$(BUILD_TIME) \ - -t $(REGISTRY)/$(PROJECT_NAME):$(RC_VERSION) \ - -f Dockerfile \ - --push \ - . - @$(call print_success,Multi-arch image built and pushed: $(REGISTRY)/$(PROJECT_NAME):$(RC_VERSION)) - @echo "" - @$(call print_status,Updating DaemonSet image tag...) - @sed -i.bak 's|image: .*node-doctor:.*|image: $(REGISTRY)/$(PROJECT_NAME):$(RC_VERSION)|' deployment/daemonset.yaml - @rm -f deployment/daemonset.yaml.bak - @echo "" - @$(MAKE) deploy-prd-kubectl - @echo "" - @$(call print_status,Committing RC release...) - @git add $(RC_VERSION_FILE) deployment/daemonset.yaml - @git commit -m "bump-rc: Release candidate $(RC_VERSION) deployed to $(KUBECTL_CONTEXT_PRD)" - @git tag -a $(RC_VERSION) -m "Release candidate $(RC_VERSION)" - @git push origin main --tags - @echo "" - @echo "🎉 RC Release $(RC_VERSION) complete!" - @echo " - Built and pushed to $(REGISTRY)/$(PROJECT_NAME):$(RC_VERSION)" - @echo " - Deployed to $(KUBECTL_CONTEXT_PRD) cluster" - @echo " - Tagged as $(RC_VERSION)" - @echo "" - @echo "Next steps:" - @echo " - Verify deployment: kubectl --context=$(KUBECTL_CONTEXT_PRD) -n $(NAMESPACE_PRD) get pods -l app=node-doctor" - @echo " - Check logs: kubectl --context=$(KUBECTL_CONTEXT_PRD) -n $(NAMESPACE_PRD) logs -l app=node-doctor" - @echo " - Monitor health: kubectl --context=$(KUBECTL_CONTEXT_PRD) -n $(NAMESPACE_PRD) get pods -l app=node-doctor -w" - # ================================================================================================ # Quality Workflow Targets # ================================================================================================ @@ -591,21 +409,6 @@ workflow-status: @$(call print_status,"Checking workflow status...") @echo "TODO: Check TaskForge or project management system" -qa-check: - @$(call print_status,"Running QA validation...") - @echo "QA Check: Running validation suite" - @$(MAKE) validate-pipeline-local - @$(MAKE) test-local - @$(call print_success,"QA validation passed") - -devils-advocate: - @$(call print_status,"Running Devils Advocate validation...") - @echo "Devils Advocate: Challenging implementation..." - @echo "- Testing edge cases" - @echo "- Verifying error handling" - @echo "- Checking security implications" - @$(call print_success,"Devils Advocate review complete") - # ================================================================================================ # GitHub Actions Monitoring # ================================================================================================ @@ -624,14 +427,14 @@ gh-logs: gh-builds: @$(call print_status,"Showing recent builds...") - @gh run list --workflow=pipeline-v2.yml --limit 10 + @gh run list --workflow=ci.yml --limit 10 # ================================================================================================ # Help System # ================================================================================================ help: - @echo "$(PROJECT_NAME) Makefile - Repository Template" + @echo "$(PROJECT_NAME) Makefile" @echo "" @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" @echo "QUICK START" @@ -652,7 +455,7 @@ help: @echo " make build Compile binary (shorthand for build-local)" @echo " make build-local Build node-doctor binary" @echo " make docker-build Build Docker image" - @echo " make docker-push Push Docker image to registry" + @echo " make docker-push VERSION=x.y.z Push Docker image (VERSION required, never :latest)" @echo " make clean Remove build artifacts and caches" @echo "" @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" @@ -661,9 +464,9 @@ help: @echo "" @echo " make test Run unit tests (fast)" @echo " make test-integration Run integration tests" - @echo " make test-e2e Run end-to-end tests" + @echo " make test-e2e Run end-to-end tests (creates a kind cluster)" @echo " make test-all Run all tests with coverage report" - @echo " make coverage-check Verify coverage >= 80% threshold" + @echo " make coverage-check Verify unit coverage >= $(COVERAGE_THRESHOLD)% threshold" @echo " make lint Run golangci-lint" @echo " make fmt Format code with gofmt and goimports" @echo "" @@ -672,36 +475,17 @@ help: @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" @echo "" @echo " make install-deps Install golangci-lint, goimports, etc." - @echo " make validate-quick Quick validation (format, vet, staticcheck)" - @echo " make validate-pipeline-local Full validation (mirrors CI/CD)" @echo "" @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" @echo "VALIDATION COMMANDS (Pre-push)" @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" @echo "" - @echo " make validate-pipeline-local Full validation (mirrors CI/CD exactly)" - @echo " make validate-quick Quick validation (format, vet, staticcheck)" - @echo " make validate-component COMPONENT=api Validate specific component" - @echo "" - @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - @echo "DEPLOYMENT COMMANDS" - @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - @echo "" - @echo " make bump Bump version and trigger CI/CD" - @echo " make bump-rc Build RC, push to $(REGISTRY), deploy to a1-ops-prd" - @echo " make bump-with-monitoring Bump and watch deployment" - @echo " make deploy-dev Deploy to development" - @echo " make deploy-stg Deploy to staging" - @echo " make deploy-prd Deploy to production (requires approval)" - @echo " make deploy-prd-kubectl Deploy to a1-ops-prd using kubectl" + @echo " make validate-pipeline-local gofmt, vet, golangci-lint, gosec, tests, helm (mirrors CI)" + @echo " make validate-quick Same without tests" + @echo " make validate-component COMPONENT=node-doctor Same, explicit component" + @echo " SKIP_LINT=1 Allow a missing golangci-lint" @echo "" - @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - @echo "QUALITY WORKFLOW COMMANDS" - @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - @echo "" - @echo " make qa-check Run QA validation" - @echo " make devils-advocate Run adversarial testing" - @echo " make workflow-status Check workflow status" + @echo " Deployment is done by .github/workflows/release.yml on a v* tag; see docs/release-process.md" @echo "" @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" @echo "GITHUB ACTIONS MONITORING" @@ -716,9 +500,10 @@ help: @echo "HELM COMMANDS" @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" @echo "" + @echo " make helm-generate Render Chart.yaml/values.yaml from their templates" + @echo " make helm-verify-generated Fail if Chart.yaml/values.yaml drifted from templates" @echo " make helm-lint Lint Helm chart" @echo " make helm-package Package Helm chart" - @echo " make helm-publish Publish Helm chart to repository" @echo "" @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" @echo "UTILITY COMMANDS" @@ -726,20 +511,15 @@ help: @echo "" @echo " make check-prerequisites Check all required tools installed" @echo " make check-go-version Verify Go version matches requirements" - @echo " make check-kubeconfig-dev Verify development kubeconfig" + @echo " make workflow-status Check workflow status" @echo " make help Show this help message" @echo "" - @echo "For more information, see docs/development/" + @echo "For more information, see CONTRIBUTING.md" workflow-help: - @echo "Quality Workflow Commands" + @echo "Workflow Commands" @echo "" @echo " make workflow-status - Show current workflow status" - @echo " make qa-check - Run QA validation gate" - @echo " make devils-advocate - Run adversarial testing gate" - @echo " make workflow-validate - Full workflow validation" - @echo "" - @echo "See docs/development/task-execution-workflow.md for complete workflow documentation" gh-help: @echo "GitHub Actions Monitoring Commands" diff --git a/docs/release-process.md b/docs/release-process.md index 1583d7d..a281cfe 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -338,7 +338,7 @@ prerelease marker — is treated as a prerelease. `ci.yml` used to defeat this exclusion entirely: it had no prerelease gate and pushed `:latest` (amd64-only) on **every** `v*` tag, so cutting an RC could leave `latest` pointing at RC content, on one architecture. `ci.yml` no longer pushes anything, so `release.yml`'s exclusion -is now the only thing that decides `:latest`. `make bump-rc` likewise no longer tags `:latest`. +is now the only thing that decides `:latest`. The Makefile never tags or pushes `:latest`. ## Creating a Release @@ -460,9 +460,7 @@ step, no `crazy-max/ghaction-import-gpg`, and no `.asc` output. If you previousl `GPG_PRIVATE_KEY` repository secret, nothing consumes it — it should be removed and the key material revoked, because an unused secret is pure attack surface. -`scripts/setup-gpg-signing-subkey.sh` and `scripts/verify-gpg-setup.sh` are left over from a -signing scheme that was never wired into the workflow. Running them configures nothing. -Likewise `.goreleaser.yml` exists in the repo root but **is never invoked** — no workflow runs +`.goreleaser.yml` exists in the repo root but **is never invoked** — no workflow runs GoReleaser, so it produces no binaries, no archives, and no `checksums.txt`. ### Verifying an image @@ -529,8 +527,7 @@ helm -n node-doctor get manifest node-doctor | grep -A6 'Minimal resources' ### If the cluster was deployed with raw manifests -Deployments made with `kubectl apply -f deployment/daemonset.yaml` (or `make deploy-prd-kubectl`) -are not Helm releases and have no `helm history`. Roll those back with: +Deployments made with `kubectl apply -f deployment/daemonset.yaml` are not Helm releases and have no `helm history`. Roll those back with: ```bash kubectl -n node-doctor rollout undo daemonset/node-doctor @@ -570,53 +567,14 @@ version must be unreachable, delete the tags from Docker Hub and revert the char ### `VERSION` defaults to an epoch timestamp -`Makefile:42` sets `VERSION := $(shell date +%s)`. Every target that interpolates `$(VERSION)` -without an override gets an integer like `1786... `, which is never a published image tag. +`VERSION := $(shell date +%s)` in the Makefile, so local `make build-*-image` tags images with an +integer that is never a published tag. `make docker-push` / `push-*-image` refuse to run unless +`VERSION` is set explicitly (`make docker-push VERSION=1.8.7`), and no Makefile target tags or +pushes `:latest`; only `release.yml` does. -```bash -make deploy-dev # helm upgrade --install ... --set image.tag=1786132891 -make deploy-stg # same -make deploy-prd # same, behind a y/n prompt -``` - -**These three targets are broken as written.** They deploy the local chart with an image tag that -does not exist in any registry, producing `ImagePullBackOff`. If you use them, always override: - -```bash -make deploy-dev VERSION=1.8.7 -``` - -They also pass `--set environment=`, a value the chart templates do not consume. - -`make build-*-image` / `push-*-image` have the same `VERSION` default; locally built images are -tagged with the epoch and with `latest`. Pushing those tags to `docker.io/supporttools` -overwrites the registry `latest` with a local build — avoid `make docker-push` unless that is -explicitly what you want. - -### `make helm-publish` does nothing - -The `helm-publish` target packages the chart and then hits a `# CUSTOMIZE: Add your Helm chart publish -command` placeholder before printing "Helm chart published". It publishes nothing. The only path -to `charts.support.tools` is the `helm-publish` job in `release.yml`, i.e. a tag push. - -### `make bump-rc` is a direct-to-production path - -It increments `.version-rc`, builds and pushes a multi-arch image, rewrites the image line in -`deployment/daemonset.yaml` with `sed`, applies it to the `a1-ops-prd` cluster, then commits, -tags and **pushes to `main` with `--tags`**. Pushing the tag re-triggers the full `release.yml`. -Understand all of that before running it. - -Two things about it were fixed alongside the tag-naming work, worth knowing if you remember the -old behaviour: - -- It used to also tag `:latest`, so a workstation build of a *release candidate* could overwrite - the `:latest` that `release.yml` publishes only for stable tags. It now pushes only the RC tag. -- Its `git commit` / `git tag` / `git push` steps used to end in `|| true`, silently swallowing - failures — leaving an image pushed and a cluster deployed from a revision that was never - tagged or pushed anywhere. They now fail loudly. - -Its console output used to claim it pushes "to Harbor"; it pushes to Docker Hub -(`REGISTRY := docker.io/supporttools`), and the strings now say so. +The former `deploy-dev/stg/prd`, `deploy-prd-kubectl`, `bump`, `bump-rc` and `helm-publish` +targets were removed. Deploy with Helm from `charts.support.tools` (see Rollback Procedures for +the equivalent `helm upgrade` commands); publishing is a `v*` tag push. ### Useful and accurate diff --git a/hooks/coding/basic-automation.toml b/hooks/coding/basic-automation.toml deleted file mode 100644 index 24e2d65..0000000 --- a/hooks/coding/basic-automation.toml +++ /dev/null @@ -1,24 +0,0 @@ -# Basic Coding Project Automation Hooks -# Customize based on project needs - -# Optional: Auto-save reminder -[[hooks]] -event = "Notification" -[hooks.matcher] -time = "every 30 minutes" -command = "echo '💾 Auto-save reminder: Your work is being saved automatically'" - -# Optional: Simple test reminder (if user mentioned testing) -[[hooks]] -event = "UserPromptSubmit" -[hooks.matcher] -content_regex = "done|finished|complete" -command = "echo 'đŸ§Ē Remember to test your changes!'" - -# Optional: Basic formatting (customize per language) -# [[hooks]] -# event = "PostToolUse" -# [hooks.matcher] -# tool_name = "Write" -# file_paths = ["*.py"] # Adjust for project language -# command = "echo '🎨 Formatting code...' && black $CLAUDE_FILE_PATH 2>/dev/null || true" \ No newline at end of file diff --git a/hooks/writing/content-automation.toml b/hooks/writing/content-automation.toml deleted file mode 100644 index 89f5dcb..0000000 --- a/hooks/writing/content-automation.toml +++ /dev/null @@ -1,39 +0,0 @@ -# Writing Project Automation Hooks -# For blogs, articles, books, etc. - -# Optional: Word count tracking -[[hooks]] -event = "Stop" -command = """ -if [ -d "content" ]; then - count=$(wc -w content/*.md 2>/dev/null | tail -1 | awk '{print $1}') - echo "📝 Total word count: $count words" - echo "$(date): $count words" >> .writing-progress.log -fi -""" - -# Optional: Writing session timer -[[hooks]] -event = "UserPromptSubmit" -[hooks.matcher] -content_regex = "start writing|begin writing" -command = "echo '⏰ Writing session started at $(date +%H:%M)' && echo $(date +%s) > .session-start" - -[[hooks]] -event = "Stop" -command = """ -if [ -f .session-start ]; then - start=$(cat .session-start) - duration=$(( ($(date +%s) - start) / 60 )) - echo "âąī¸ Writing session duration: $duration minutes" - rm .session-start -fi -""" - -# Optional: Backup reminder -[[hooks]] -event = "Notification" -[hooks.matcher] -time = "17:00" -days = ["monday", "wednesday", "friday"] -command = "echo 'â˜ī¸ Remember to backup your writing to cloud storage!'" \ No newline at end of file diff --git a/scripts/README.md b/scripts/README.md index ed2ec5c..38320eb 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,359 +1,7 @@ -# Validation Scripts +# Scripts -This directory contains validation scripts that mirror CI/CD pipeline checks, allowing local validation before pushing code. - -## Core Validation Scripts - -### validate-pipeline-local.sh ⭐ PRIMARY SCRIPT - -**Purpose**: Mirrors the complete CI/CD pipeline validation locally - -**Usage**: -```bash -# Full validation (mirrors CI/CD exactly) -./scripts/validate-pipeline-local.sh - -# Quick validation (format, vet, staticcheck only - no tests) -./scripts/validate-pipeline-local.sh --quick - -# Validate specific component -./scripts/validate-pipeline-local.sh --component api -``` - -**What it validates**: -- Go version compliance -- Repository cleanliness -- Code formatting (gofmt) -- Go vet checks -- Static analysis (staticcheck) -- Security scanning (gosec) -- Unit tests with coverage -- Component-specific validations - -**Exit codes**: -- `0` - All validations passed -- `1` - One or more validations failed - ---- - -### validate-go-version.sh - -**Purpose**: Ensures Go version consistency across project - -**Usage**: -```bash -./scripts/validate-go-version.sh -``` - -**Checks**: -- Installed Go version matches go.mod requirement -- go.mod specifies correct Go version -- Dockerfiles use correct Go version - -**Customization**: -Edit the script to change the required Go version (currently 1.24) - ---- - -### validate-repo-cleanliness.sh - -**Purpose**: Enforces repository structure rules - -**Usage**: -```bash -./scripts/validate-repo-cleanliness.sh -``` - -**Checks**: -- No Go files in repository root -- No build artifacts in repository -- No temporary files committed -- No secrets or credentials -- Proper directory organization - -**Violations result in**: -- Detailed error messages -- Suggested fixes -- Non-zero exit code - ---- - -### validate-api.sh (Example Component Validation) - -**Purpose**: Component-specific validation for API - -**Usage**: -```bash -./scripts/validate-api.sh -``` - -**What it does**: -- Runs API-specific tests -- Validates API configuration -- Checks OpenAPI/Swagger documentation -- Verifies handler structure - -**Customization**: Copy this pattern for other components - ---- - -## Integration with Workflow - -### Pre-Push Validation - -Enable automatic pre-push validation: - -```bash -# Setup git hooks (in .githooks/) -git config core.hooksPath .githooks -``` - -### Makefile Integration - -These scripts are integrated into the makefile: - -```bash -# Full validation -make validate-pipeline-local - -# Quick validation -make validate-quick - -# Component validation -make validate-component COMPONENT=api -``` - -### CI/CD Integration - -The CI/CD pipeline uses these same scripts: - -```yaml -# .github/workflows/pipeline-v2.yml -validation: - - run: ./scripts/validate-go-version.sh - - run: ./scripts/validate-repo-cleanliness.sh - - run: ./scripts/validate-pipeline-local.sh -``` - -## Creating Component Validation Scripts - -To add validation for a new component: - -### 1. Create Component Script - -```bash -cp scripts/validate-api.sh scripts/validate-mycomponent.sh -``` - -### 2. Customize for Component - -```bash -#!/bin/bash -# validate-mycomponent.sh - -set -e - -echo "Validating my component..." - -# Component-specific checks -cd cmd/mycomponent -go test ./... -v -go vet ./... - -# Additional validation -# ... - -echo "✅ My component validation passed" -``` - -### 3. Make Executable - -```bash -chmod +x scripts/validate-mycomponent.sh -``` - -### 4. Add to Pipeline Script - -Edit `validate-pipeline-local.sh` to include your component: - -```bash -COMPONENTS=("api" "agent" "mycomponent") -``` - -### 5. Add Makefile Target - -```makefile -validate-mycomponent-local: - @./scripts/validate-mycomponent.sh -``` - -## Validation Configuration - -Configuration is stored in `.validation.json`: - -```json -{ - "validation": { - "enabled": true, - "parallel_jobs": 4, - "quick_mode": { - "skip_tests": true, - "skip_gosec": true - } - }, - "components": { - "api": { - "enabled": true, - "test_timeout": "10m", - "coverage_threshold": 80 - } - }, - "tools": { - "staticcheck": { - "enabled": true, - "checks": ["all", "-ST1000"] - }, - "gosec": { - "enabled": true, - "severity": "medium" - } - } -} -``` - -## Exit Codes - -All validation scripts follow these exit code conventions: - -- `0` - Success, all checks passed -- `1` - Failure, one or more checks failed -- `2` - Configuration error or missing dependencies - -## Best Practices - -### 1. Always Validate Locally Before Pushing - -```bash -make validate-pipeline-local -``` - -This prevents CI/CD failures and saves time. - -### 2. Use Quick Mode During Development - -```bash -make validate-quick -``` - -Run quick validation frequently, full validation before push. - -### 3. Fix Issues Systematically - -When validation fails: -1. Read the error message carefully -2. Fix the reported issue -3. Re-run validation -4. Don't commit until validation passes - -### 4. Keep Scripts Updated - -When changing: -- Go version → Update validate-go-version.sh -- Repository structure → Update validate-repo-cleanliness.sh -- Component structure → Update component validation scripts - -### 5. Test Scripts Locally - -Before committing script changes: -```bash -# Test success case -./scripts/validate-pipeline-local.sh - -# Test failure case (introduce an error) -echo "bad code" > test.go -./scripts/validate-pipeline-local.sh -git checkout test.go -``` - -## Troubleshooting - -### Script Permission Denied - -```bash -chmod +x scripts/*.sh -``` - -### Go Version Mismatch - -```bash -# Check installed version -go version - -# Check required version -grep "^go" go.mod - -# Update Go if needed -``` - -### Validation Takes Too Long - -Use quick mode during development: -```bash -./scripts/validate-pipeline-local.sh --quick -``` - -### Component Not Found - -Add component to validate-pipeline-local.sh: -```bash -COMPONENTS=("api" "agent" "your-component") -``` - -## Integration Testing - -Test that validation correctly catches issues: - -```bash -# Test 1: Should pass -make validate-pipeline-local - -# Test 2: Should fail on formatting -echo "package main;func main(){}" > cmd/test/bad.go -make validate-pipeline-local -git checkout cmd/test/bad.go - -# Test 3: Should fail on cleanliness -touch root-file.go -make validate-pipeline-local -rm root-file.go -``` - -## Performance - -### Validation Times (Typical) - -- **Quick mode**: 30-60 seconds -- **Full validation**: 2-5 minutes -- **Component validation**: 30-120 seconds per component - -### Optimization Tips - -1. **Use parallel execution** (already configured) -2. **Skip tests in quick mode** for rapid iteration -3. **Cache Go modules** to speed up builds -4. **Use component validation** when working on specific areas - -## Documentation - -For complete validation workflow documentation, see: -- [Local CI/CD Validation Guide](../docs/development/local-cicd-validation-guide.md) -- [Task Execution Workflow](../docs/development/task-execution-workflow.md) - -## Contributing - -When adding new validation: -1. Create a focused validation script -2. Add to validate-pipeline-local.sh -3. Document in this README -4. Test both success and failure cases -5. Update .validation.json if needed +- `validate-pipeline-local.sh` - runs the CI checks locally (gofmt, go vet, golangci-lint, gosec, go test, helm lint, helm-verify-generated). `--quick` skips tests; `SKIP_LINT=1` allows a missing golangci-lint. Wired to `make validate-pipeline-local` and `make validate-quick`. +- `validate-go-version.sh` - checks that go.mod and the Dockerfiles use the required Go minor version. Wired to `make check-go-version`. +- `validate-repo-cleanliness.sh` - checks for stray Go files in the repo root, compiled binaries outside `bin/`, and editor temp files. +- `test-monitors.sh` - exercises the monitors on a real cluster node, grouped by risk level (`--safe`, `--caution`, `--destructive`). Run as root on a dedicated test node. +- `test-vmxnet3-detection.sh` - injects synthetic vmxnet3 kernel messages on a lab node to verify pattern detection. Run as root. diff --git a/scripts/init-from-template.sh b/scripts/init-from-template.sh deleted file mode 100755 index 808bf31..0000000 --- a/scripts/init-from-template.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/bin/bash -# Template Initialization Script -# -# This script initializes a new project from the Nexmonyx repository template. -# It performs variable substitution, sets up git hooks, and validates the setup. - -set -e - -# Colors -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' - -echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" -echo -e "${BLUE} Nexmonyx Template Initialization${NC}" -echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" -echo "" -echo "This script will:" -echo " 1. Substitute template variables with your project values" -echo " 2. Set up git hooks for validation" -echo " 3. Validate the setup" -echo " 4. Provide next steps" -echo "" - -# Check prerequisites -echo -e "${BLUE}[1/5] Checking prerequisites...${NC}" - -command -v git >/dev/null 2>&1 || { echo -e "${RED}Error: git not found${NC}"; exit 1; } -command -v go >/dev/null 2>&1 || { echo -e "${YELLOW}Warning: go not found (required for Go projects)${NC}"; } - -echo -e "${GREEN}✓ Prerequisites checked${NC}" -echo "" - -# Run variable substitution -echo -e "${BLUE}[2/5] Variable substitution...${NC}" - -if [ ! -f "scripts/substitute-variables.sh" ]; then - echo -e "${RED}Error: substitute-variables.sh not found${NC}" - exit 1 -fi - -chmod +x scripts/substitute-variables.sh -./scripts/substitute-variables.sh - -echo "" - -# Set up git hooks -echo -e "${BLUE}[3/5] Setting up git hooks...${NC}" - -if [ -f ".githooks/setup.sh" ]; then - chmod +x .githooks/setup.sh - cd .githooks && ./setup.sh && cd .. - echo -e "${GREEN}✓ Git hooks configured${NC}" -else - echo -e "${YELLOW}Warning: Git hooks not found, skipping${NC}" -fi - -echo "" - -# Validate setup -echo -e "${BLUE}[4/5] Validating setup...${NC}" - -# Check for remaining template variables -remaining=$(grep -r "{{.*}}" . \ - --exclude-dir=".git" \ - --exclude-dir="node_modules" \ - --exclude-dir="bin" \ - --exclude="*.png" --exclude="*.jpg" --exclude="*.jpeg" \ - 2>/dev/null | wc -l | tr -d ' ') - -if [ "$remaining" -gt 0 ]; then - echo -e "${YELLOW}Warning: $remaining files still contain template variables${NC}" - echo " Run: grep -r '{{.*}}' . to find them" -else - echo -e "${GREEN}✓ No remaining template variables found${NC}" -fi - -echo "" - -# Provide next steps -echo -e "${BLUE}[5/5] Next steps...${NC}" -echo "" -echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" -echo -e "${GREEN} Initialization Complete!${NC}" -echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" -echo "" -echo "Your project has been initialized. Next steps:" -echo "" -echo "1. Review the changes:" -echo " git diff" -echo "" -echo "2. Customize for your project:" -echo " - Update CLAUDE.md with project-specific instructions" -echo " - Add your specialized agents to .claude/agents/specialized/" -echo " - Update makefile with your components" -echo " - Configure .validation.json for your components" -echo "" -echo "3. Initialize or update git repository:" -echo " git init # If new repository" -echo " git add ." -echo " git commit -m \"Initialize from Nexmonyx template\"" -echo "" -echo "4. Set up remote repository (optional):" -echo " git remote add origin https://github.com/YOUR_ORG/YOUR_REPO.git" -echo " git push -u origin main" -echo "" -echo "5. Read the documentation:" -echo " - docs/README.md # Documentation hub" -echo " - docs/development/task-execution-workflow.md # Development process" -echo " - .claude/AGENT_GUIDE.md # How to use agents" -echo "" -echo "6. Start development:" -echo " make help # See available commands" -echo " make build-local # Build your project" -echo " make test-local # Run tests" -echo " make validate-pipeline-local # Validate before pushing" -echo "" -echo -e "${BLUE}For complete setup instructions, see: TEMPLATE_USAGE.md${NC}" -echo "" diff --git a/scripts/setup-gpg-signing-subkey.sh b/scripts/setup-gpg-signing-subkey.sh deleted file mode 100755 index 5679957..0000000 --- a/scripts/setup-gpg-signing-subkey.sh +++ /dev/null @@ -1,253 +0,0 @@ -#!/bin/bash -set -e - -# setup-gpg-signing-subkey.sh -# Creates a dedicated signing subkey for GitHub Actions (safer than using main key) - -echo "🔐 Setting up GPG Signing Subkey for Release Automation" -echo "================================================================" -echo "" -echo "WHY USE A SUBKEY?" -echo "- Protects your main GPG key from compromise" -echo "- Subkey can be revoked independently if GitHub is compromised" -echo "- Best practice for automated signing systems" -echo "" -echo "================================================================" -echo "" - -# Colors -GREEN='\033[0;32m' -RED='\033[0;31m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -MAIN_KEY_ID="1AF32A8B0481A7F3" - -# Step 1: Verify main key exists -echo "1ī¸âƒŖ Verifying main GPG key..." -if ! gpg --list-secret-keys "$MAIN_KEY_ID" > /dev/null 2>&1; then - echo -e "${RED}❌ Main key $MAIN_KEY_ID not found${NC}" - exit 1 -fi -echo -e "${GREEN}✅ Main key found${NC}" -echo "" - -# Step 2: Check if signing subkey already exists -echo "2ī¸âƒŖ Checking for existing signing subkey..." -EXISTING_SUBKEY=$(gpg --list-secret-keys --with-subkey-fingerprints "$MAIN_KEY_ID" | grep -A 1 "\[S\]" | tail -1 | awk '{print $1}' || echo "") - -if [ -n "$EXISTING_SUBKEY" ]; then - echo -e "${YELLOW}âš ī¸ Found existing signing subkey: $EXISTING_SUBKEY${NC}" - echo "" - read -p "Do you want to use this existing subkey? (y/n): " USE_EXISTING - if [ "$USE_EXISTING" = "y" ]; then - SUBKEY_ID="$EXISTING_SUBKEY" - echo -e "${GREEN}✅ Using existing subkey${NC}" - SKIP_CREATION=true - else - echo "Will create a new signing subkey..." - SKIP_CREATION=false - fi -else - echo "No existing signing subkey found. Will create new one." - SKIP_CREATION=false -fi -echo "" - -# Step 3: Create signing subkey (if needed) -if [ "$SKIP_CREATION" != "true" ]; then - echo "3ī¸âƒŖ Creating dedicated signing subkey..." - echo "" - echo -e "${BLUE}â„šī¸ You will be prompted to:${NC}" - echo " 1. Enter your main key passphrase" - echo " 2. Choose subkey type: Select '(4) RSA (sign only)'" - echo " 3. Choose key size: Recommend 4096" - echo " 4. Choose expiration: Recommend 2y (2 years)" - echo "" - read -p "Press Enter to continue..." - echo "" - - # Interactive subkey creation - gpg --expert --edit-key "$MAIN_KEY_ID" < "$EXPORT_FILE" 2>/dev/null; then - echo -e "${GREEN}✅ Subkey exported to: $EXPORT_FILE${NC}" - echo "" - - # Verify it's only the subkey (not main key) - echo "5ī¸âƒŖ Verifying export contains only subkey..." - TEMP_KEYRING=$(mktemp -d) - if gpg --homedir "$TEMP_KEYRING" --import "$EXPORT_FILE" 2>/dev/null; then - # Check if main key secret is present - if gpg --homedir "$TEMP_KEYRING" --list-secret-keys | grep -q "sec#"; then - echo -e "${GREEN}✅ Export verified: Contains subkey only (main key stub)${NC}" - echo " The '#' after 'sec' means main key is NOT included (safe!)" - else - echo -e "${YELLOW}âš ī¸ Warning: Main key might be included${NC}" - fi - fi - rm -rf "$TEMP_KEYRING" -else - echo -e "${RED}❌ Failed to export subkey${NC}" - exit 1 -fi -echo "" - -# Step 5: Create passphrase for automated signing -echo "6ī¸âƒŖ Subkey passphrase setup..." -echo "" -echo -e "${BLUE}â„šī¸ The subkey currently uses your main key's passphrase.${NC}" -echo "For GitHub Actions, you have two options:" -echo "" -echo "Option A (RECOMMENDED): Remove passphrase from subkey" -echo " - Safer because subkey can be revoked independently" -echo " - No passphrase to manage in GitHub Secrets" -echo " - Still requires passphrase for main key operations" -echo "" -echo "Option B: Use main key passphrase in GitHub Secrets" -echo " - Keep subkey passphrase protection" -echo " - Must store passphrase in GitHub Secrets" -echo "" -read -p "Remove passphrase from subkey? (recommended: y/n): " REMOVE_PASS - -if [ "$REMOVE_PASS" = "y" ]; then - echo "" - echo "Removing passphrase from signing subkey..." - echo -e "${YELLOW}You will be prompted for your CURRENT passphrase, then press Enter (blank) for new passphrase${NC}" - - gpg --edit-key "$MAIN_KEY_ID" < "$EXPORT_FILE" - NEEDS_PASSPHRASE=false - echo -e "${GREEN}✅ Passphrase removed from subkey${NC}" -else - NEEDS_PASSPHRASE=true - echo -e "${YELLOW}âš ī¸ Subkey still requires passphrase${NC}" -fi -echo "" - -# Step 6: GitHub configuration instructions -echo "================================================================" -echo "7ī¸âƒŖ GitHub Secrets Configuration" -echo "================================================================" -echo "" -echo -e "${GREEN}✅ Subkey ready for GitHub Actions!${NC}" -echo "" -echo "Next steps:" -echo "" -echo "1. Add the subkey to GitHub Secrets:" -echo "" -echo -e " ${BLUE}Via GitHub CLI:${NC}" -echo " gh secret set GPG_PRIVATE_KEY < $EXPORT_FILE" -echo "" -echo -e " ${BLUE}Via GitHub UI:${NC}" -echo " - Go to: https://github.com/supporttools/node-doctor/settings/secrets/actions" -echo " - Click 'New repository secret'" -echo " - Name: GPG_PRIVATE_KEY" -echo " - Value: " -echo "" - -if [ "$NEEDS_PASSPHRASE" = "true" ]; then - echo "2. Add passphrase to GitHub Secrets:" - echo "" - echo -e " ${BLUE}Via GitHub CLI:${NC}" - echo " gh secret set GPG_PASSPHRASE # will prompt for value" - echo "" - echo -e " ${BLUE}Via GitHub UI:${NC}" - echo " - Click 'New repository secret'" - echo " - Name: GPG_PASSPHRASE" - echo " - Value: " - echo "" -else - echo "2. No passphrase needed (you removed it from the subkey)" - echo "" -fi - -echo "3. Publish public key for verification:" -echo "" -echo " gpg --armor --export $MAIN_KEY_ID > node-doctor-signing-key.asc" -echo " # Upload to: docs/signing-keys/node-doctor-signing-key.asc" -echo " # Or publish to keyserver:" -echo " gpg --send-keys $MAIN_KEY_ID" -echo "" - -echo "4. Update release documentation with key ID:" -echo " Main key ID: $MAIN_KEY_ID" -echo " Signing subkey: $SUBKEY_ID" -echo "" - -echo "================================================================" -echo "🔐 Security Notes" -echo "================================================================" -echo "" -echo -e "${GREEN}✅ Your main GPG key is safe!${NC}" -echo " - Main key never leaves your machine" -echo " - Only signing subkey is uploaded to GitHub" -echo " - Subkey can be revoked independently if needed" -echo "" -echo -e "${BLUE}â„šī¸ To revoke the subkey later (if GitHub is compromised):${NC}" -echo " gpg --edit-key $MAIN_KEY_ID" -echo " > key $SUBKEY_ID" -echo " > revkey" -echo " > save" -echo "" -echo -e "${YELLOW}âš ī¸ Remember to securely delete the exported file after uploading:${NC}" -echo " shred -u $EXPORT_FILE" -echo "" - -# Step 7: Verification -echo "================================================================" -echo "8ī¸âƒŖ Verification" -echo "================================================================" -echo "" -echo "To verify the setup works:" -echo "" -echo "1. Run the verification script:" -echo " ./scripts/verify-gpg-setup.sh" -echo "" -echo "2. Test with a pre-release tag:" -echo " git tag v0.1.0-rc.1" -echo " git push origin v0.1.0-rc.1" -echo "" -echo "3. Check GitHub Actions workflow for GPG signing steps" -echo "" -echo "4. Verify release artifacts have both signatures:" -echo " - *.tar.gz.cosign.sig (Cosign/GitHub OIDC)" -echo " - *.tar.gz.asc (GPG subkey signature)" -echo "" diff --git a/scripts/substitute-variables.sh b/scripts/substitute-variables.sh deleted file mode 100755 index b36edec..0000000 --- a/scripts/substitute-variables.sh +++ /dev/null @@ -1,135 +0,0 @@ -#!/bin/bash -# Variable Substitution Script -# -# This script replaces template variables with your project values. -# It searches for {{VARIABLE}} patterns and replaces them with your values. - -set -e - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" -echo -e "${BLUE} Nexmonyx Template Variable Substitution${NC}" -echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" -echo "" - -# Check if running in repository root -if [ ! -f "CLAUDE.md" ] || [ ! -d ".claude" ]; then - echo -e "${RED}Error: Must run from repository root${NC}" - exit 1 -fi - -# Function to prompt for variable -prompt_variable() { - local var_name=$1 - local description=$2 - local default_value=$3 - local value - - echo -e "${YELLOW}${var_name}${NC}" - echo -e " ${description}" - if [ -n "$default_value" ]; then - read -p " Enter value (default: ${default_value}): " value - value=${value:-$default_value} - else - read -p " Enter value: " value - fi - - echo "$value" -} - -# Collect variables from user -echo -e "${BLUE}Please provide values for template variables:${NC}" -echo "" - -PROJECT_NAME=$(prompt_variable "PROJECT_NAME" "Your project name (lowercase, no spaces)" "my-project") -GITHUB_ORG=$(prompt_variable "GITHUB_ORG" "Your GitHub organization or username" "") -REGISTRY=$(prompt_variable "REGISTRY" "Container registry (e.g., ghcr.io/myorg, harbor.example.com/myproject)" "ghcr.io/${GITHUB_ORG}") -HELM_REPO=$(prompt_variable "HELM_REPO" "Helm chart repository URL" "https://charts.example.com") -PRIMARY_DOMAIN=$(prompt_variable "PRIMARY_DOMAIN" "Primary domain for your project" "example.com") -GO_VERSION=$(prompt_variable "GO_VERSION" "Go version" "1.24") - -echo "" -echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" -echo -e "${BLUE} Summary of Values${NC}" -echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" -echo "PROJECT_NAME: $PROJECT_NAME" -echo "GITHUB_ORG: $GITHUB_ORG" -echo "REGISTRY: $REGISTRY" -echo "HELM_REPO: $HELM_REPO" -echo "PRIMARY_DOMAIN: $PRIMARY_DOMAIN" -echo "GO_VERSION: $GO_VERSION" -echo "" - -read -p "Proceed with substitution? (y/n): " confirm -if [ "$confirm" != "y" ] && [ "$confirm" != "Y" ]; then - echo -e "${YELLOW}Substitution cancelled${NC}" - exit 0 -fi - -echo "" -echo -e "${BLUE}Performing substitutions...${NC}" - -# Find all files that might contain variables (excluding binary files, .git, and node_modules) -files=$(find . -type f \ - -not -path "*/\.git/*" \ - -not -path "*/node_modules/*" \ - -not -path "*/bin/*" \ - -not -path "*/.claude/settings.local.json" \ - -not -name "*.png" \ - -not -name "*.jpg" \ - -not -name "*.jpeg" \ - -not -name "*.gif" \ - -not -name "*.ico" \ - -not -name "*.pdf" \ - -not -name "*.woff*" \ - -not -name "*.ttf" \ - -not -name "*.eot") - -count=0 - -for file in $files; do - # Check if file contains any template variables - if grep -q "{{.*}}" "$file" 2>/dev/null; then - echo -e " ${GREEN}✓${NC} $file" - - # Perform substitutions (macOS and Linux compatible) - if [[ "$OSTYPE" == "darwin"* ]]; then - # macOS - sed -i '' "s|{{PROJECT_NAME}}|$PROJECT_NAME|g" "$file" - sed -i '' "s|{{GITHUB_ORG}}|$GITHUB_ORG|g" "$file" - sed -i '' "s|{{REGISTRY}}|$REGISTRY|g" "$file" - sed -i '' "s|{{HELM_REPO}}|$HELM_REPO|g" "$file" - sed -i '' "s|{{PRIMARY_DOMAIN}}|$PRIMARY_DOMAIN|g" "$file" - sed -i '' "s|{{GO_VERSION}}|$GO_VERSION|g" "$file" - else - # Linux - sed -i "s|{{PROJECT_NAME}}|$PROJECT_NAME|g" "$file" - sed -i "s|{{GITHUB_ORG}}|$GITHUB_ORG|g" "$file" - sed -i "s|{{REGISTRY}}|$REGISTRY|g" "$file" - sed -i "s|{{HELM_REPO}}|$HELM_REPO|g" "$file" - sed -i "s|{{PRIMARY_DOMAIN}}|$PRIMARY_DOMAIN|g" "$file" - sed -i "s|{{GO_VERSION}}|$GO_VERSION|g" "$file" - fi - - ((count++)) - fi -done - -echo "" -echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" -echo -e "${GREEN} Substitution Complete!${NC}" -echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" -echo -e "${GREEN} Updated $count files${NC}" -echo "" -echo "Next steps:" -echo "1. Review the changes: git diff" -echo "2. Update additional project-specific files as needed" -echo "3. Initialize git: git init (if not already initialized)" -echo "4. Commit the changes: git add . && git commit -m \"Initialize from template\"" -echo "" diff --git a/scripts/validate-api.sh b/scripts/validate-api.sh deleted file mode 100755 index f569b2b..0000000 --- a/scripts/validate-api.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -# API-specific validation script - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -"$SCRIPT_DIR/validate-all.sh" --component api \ No newline at end of file diff --git a/scripts/validate-pipeline-local.sh b/scripts/validate-pipeline-local.sh index ad23355..410f48d 100755 --- a/scripts/validate-pipeline-local.sh +++ b/scripts/validate-pipeline-local.sh @@ -1,383 +1,122 @@ -#!/bin/bash -# validate-pipeline-local.sh -# Mirrors CI/CD pipeline validation exactly for local testing -# This script validates node-doctor with the same checks as .github/workflows/ci.yml - -set -e - -# Color codes for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Counters for summary -TOTAL_CHECKS=0 -PASSED_CHECKS=0 -FAILED_CHECKS=0 - -# Components to validate (node-doctor is a single binary) -COMPONENTS=("node-doctor") - -# Parse command line arguments -SPECIFIC_COMPONENT="" -QUICK_MODE=false -PARALLEL_MODE=true -FAIL_FAST=false +#!/usr/bin/env bash +# Runs the checks from .github/workflows/ci.yml against the local tree. +set -uo pipefail usage() { - cat << EOF -Usage: $0 [OPTIONS] - -Validates node-doctor using the same checks as CI/CD pipeline. - -OPTIONS: - --component=NAME Validate only specific component (node-doctor) - --quick Skip tests and gosec (format, vet, staticcheck only) - --sequential Run components sequentially instead of parallel - --fail-fast Stop on first failure - -h, --help Show this help message - -EXAMPLES: - $0 # Validate node-doctor - $0 --component=node-doctor # Validate node-doctor (explicit) - $0 --quick # Quick validation (no tests) - $0 --fail-fast # Stop on first error + cat <&2; exit 1 ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $arg" >&2; usage >&2; exit 1 ;; esac done -# Validate specific component if provided -if [ -n "$SPECIFIC_COMPONENT" ]; then - VALID_COMPONENT=false - for comp in "${COMPONENTS[@]}"; do - if [ "$comp" = "$SPECIFIC_COMPONENT" ]; then - VALID_COMPONENT=true - break - fi - done - - if [ "$VALID_COMPONENT" = false ]; then - echo -e "${RED}Error: Invalid component '$SPECIFIC_COMPONENT'${NC}" - echo "Valid components: ${COMPONENTS[*]}" - exit 1 - fi - - COMPONENTS=("$SPECIFIC_COMPONENT") -fi - -# Print header -echo -e "${BLUE}╔════════════════════════════════════════════════════════════════╗${NC}" -echo -e "${BLUE}║ Local CI/CD Pipeline Validation ║${NC}" -echo -e "${BLUE}╚════════════════════════════════════════════════════════════════╝${NC}" -echo "" -echo -e "${BLUE}Components:${NC} ${COMPONENTS[*]}" -echo -e "${BLUE}Quick Mode:${NC} $QUICK_MODE" -echo -e "${BLUE}Parallel:${NC} $PARALLEL_MODE" -echo -e "${BLUE}Fail Fast:${NC} $FAIL_FAST" -echo "" - -# Check for required tools -echo -e "${BLUE}Checking required tools...${NC}" -MISSING_TOOLS=() - -if ! command -v gofmt &> /dev/null; then - MISSING_TOOLS+=("gofmt") -fi - -if ! command -v go &> /dev/null; then - MISSING_TOOLS+=("go") -else - GO_VERSION=$(go version | awk '{print $3}' | sed 's/go//') - if [[ ! "$GO_VERSION" =~ ^1\.24 ]]; then - echo -e "${YELLOW}Warning: Go version $GO_VERSION detected. CI/CD uses Go 1.24${NC}" - fi -fi - -if ! command -v staticcheck &> /dev/null; then - echo -e "${YELLOW}Installing staticcheck...${NC}" - go install honnef.co/go/tools/cmd/staticcheck@latest -fi - -if [ "$QUICK_MODE" = false ] && ! command -v gosec &> /dev/null; then - echo -e "${YELLOW}Installing gosec...${NC}" - go install github.com/securego/gosec/v2/cmd/gosec@latest -fi - -if [ ${#MISSING_TOOLS[@]} -gt 0 ]; then - echo -e "${RED}Error: Missing required tools: ${MISSING_TOOLS[*]}${NC}" - exit 1 -fi - -echo -e "${GREEN}✓ All required tools available${NC}" -echo "" - -# Function to run a validation check -run_check() { - local component=$1 - local check_name=$2 - local command=$3 +cd "$(dirname "$0")/.." - TOTAL_CHECKS=$((TOTAL_CHECKS + 1)) +PASSED=() +FAILED=() +SKIPPED=() - echo -e "${BLUE}[$component]${NC} Running $check_name..." - - if eval "$command" > /tmp/validate_${component}_${check_name}.log 2>&1; then - echo -e "${GREEN}✓ [$component] $check_name passed${NC}" - PASSED_CHECKS=$((PASSED_CHECKS + 1)) - return 0 +run() { + local name=$1 + shift + echo "==> $name" + if "$@"; then + PASSED+=("$name") else - echo -e "${RED}✗ [$component] $check_name FAILED${NC}" - cat /tmp/validate_${component}_${check_name}.log - FAILED_CHECKS=$((FAILED_CHECKS + 1)) - - if [ "$FAIL_FAST" = true ]; then - echo -e "${RED}Fail-fast enabled. Stopping validation.${NC}" - exit 1 - fi - - return 1 + FAILED+=("$name") + echo "FAILED: $name" >&2 fi } -# Function to validate a single component -validate_component() { - local component=$1 - local component_failed=false - - echo -e "${BLUE}════════════════════════════════════════════════════════════════${NC}" - echo -e "${BLUE}Validating: $component${NC}" - echo -e "${BLUE}════════════════════════════════════════════════════════════════${NC}" - - # Step 1: Format check (gofmt) - case "$component" in - api) - run_check "$component" "format" "test -z \"\$(find api -name '*.go' -not -path '*/vendor/*' -exec gofmt -l {} +)\"" || component_failed=true - ;; - pkg) - run_check "$component" "format" "test -z \"\$(find pkg -name '*.go' -not -path '*/vendor/*' -exec gofmt -l {} +)\"" || component_failed=true - ;; - org-management-controller) - run_check "$component" "format" "test -z \"\$(find pkg/controller/org-management -name '*.go' -not -path '*/vendor/*' -exec gofmt -l {} +)\"" || component_failed=true - ;; - probe-controller) - run_check "$component" "format" "test -z \"\$(find pkg/probe -name '*.go' -not -path '*/vendor/*' -exec gofmt -l {} +)\"" || component_failed=true - ;; - ingestor) - run_check "$component" "format" "test -z \"\$(find pkg/ingestor -name '*.go' -not -path '*/vendor/*' -exec gofmt -l {} +)\"" || component_failed=true - ;; - customer-probe-agent) - run_check "$component" "format" "test -z \"\$(find cmd/customer-probe-agent -name '*.go' -not -path '*/vendor/*' -exec gofmt -l {} +)\"" || component_failed=true - ;; - esac - - # Step 2: Go vet - case "$component" in - api) - run_check "$component" "go-vet" "(cd api && go vet ./...)" || component_failed=true - ;; - pkg) - run_check "$component" "go-vet" "go vet ./pkg/..." || component_failed=true - ;; - org-management-controller) - run_check "$component" "go-vet" "go vet ./pkg/controller/org-management/..." || component_failed=true - ;; - probe-controller) - run_check "$component" "go-vet" "go vet ./pkg/probe/..." || component_failed=true - ;; - ingestor) - run_check "$component" "go-vet" "go vet ./pkg/ingestor/..." || component_failed=true - ;; - customer-probe-agent) - # NOTE: This is the CORRECT way - CI/CD workflow has a bug here - run_check "$component" "go-vet" "(cd cmd/customer-probe-agent && go vet ./...)" || component_failed=true - ;; - esac - - # Step 3: Staticcheck - case "$component" in - api) - run_check "$component" "staticcheck" "(cd api && staticcheck ./...)" || component_failed=true - ;; - pkg) - run_check "$component" "staticcheck" "staticcheck ./pkg/..." || component_failed=true - ;; - org-management-controller) - run_check "$component" "staticcheck" "staticcheck ./pkg/controller/org-management/..." || component_failed=true - ;; - probe-controller) - run_check "$component" "staticcheck" "staticcheck ./pkg/probe/..." || component_failed=true - ;; - ingestor) - run_check "$component" "staticcheck" "staticcheck ./pkg/ingestor/..." || component_failed=true - ;; - customer-probe-agent) - # NOTE: This is the CORRECT way - CI/CD workflow has a bug here - run_check "$component" "staticcheck" "(cd cmd/customer-probe-agent && staticcheck ./...)" || component_failed=true - ;; - esac - - # Step 4: Gosec (skip in quick mode) - if [ "$QUICK_MODE" = false ]; then - case "$component" in - api) - run_check "$component" "gosec" "(cd api && gosec -quiet ./...)" || component_failed=true - ;; - pkg) - run_check "$component" "gosec" "gosec -quiet ./pkg/..." || component_failed=true - ;; - org-management-controller) - run_check "$component" "gosec" "gosec -quiet ./pkg/controller/org-management/..." || component_failed=true - ;; - probe-controller) - run_check "$component" "gosec" "gosec -quiet ./pkg/probe/..." || component_failed=true - ;; - ingestor) - run_check "$component" "gosec" "gosec -quiet ./pkg/ingestor/..." || component_failed=true - ;; - customer-probe-agent) - # NOTE: This is the CORRECT way - CI/CD workflow has a bug here - run_check "$component" "gosec" "(cd cmd/customer-probe-agent && gosec -quiet ./...)" || component_failed=true - ;; - esac - fi +skip() { + SKIPPED+=("$1 ($2)") + echo "==> $1 skipped: $2" +} - # Step 5: Unit tests with race detection (skip in quick mode) - if [ "$QUICK_MODE" = false ]; then - case "$component" in - api) - run_check "$component" "tests" "(cd api && go test -v -race -coverprofile=coverage.out -covermode=atomic ./...)" || component_failed=true - ;; - org-management-controller) - run_check "$component" "tests" "go test -v -race -coverprofile=coverage.out -covermode=atomic ./pkg/controller/org-management/..." || component_failed=true - ;; - probe-controller) - run_check "$component" "tests" "go test -v -race -coverprofile=coverage.out -covermode=atomic ./pkg/probe/..." || component_failed=true - ;; - ingestor) - run_check "$component" "tests" "go test -v -race -coverprofile=coverage.out -covermode=atomic ./pkg/ingestor/..." || component_failed=true - ;; - customer-probe-agent) - run_check "$component" "tests" "(cd cmd/customer-probe-agent && go test -v -race -coverprofile=coverage.out -covermode=atomic ./...)" || component_failed=true - ;; - pkg) - run_check "$component" "tests" "go test -v -race -coverprofile=coverage.out -covermode=atomic ./pkg/..." || component_failed=true - ;; - esac - fi +check_gofmt() { + local unformatted + unformatted=$(find . -name '*.go' -not -path './.*' -not -path './vendor/*' -print0 | xargs -0 gofmt -l) + [ -z "$unformatted" ] && return 0 + echo "gofmt -l reports unformatted files:" + echo "$unformatted" + return 1 +} - echo "" +check_vet() { + GOOS=linux go vet ./... +} - if [ "$component_failed" = true ]; then - return 1 - fi - return 0 +check_lint() { + golangci-lint run --timeout=5m } -# Main validation loop -echo -e "${BLUE}Starting validation...${NC}" -echo "" +check_gosec() { + gosec -quiet ./... +} -COMPONENT_RESULTS=() +check_tests() { + go test -short -race ./pkg/... ./cmd/... ./config/... +} -if [ "$PARALLEL_MODE" = true ] && [ ${#COMPONENTS[@]} -gt 1 ]; then - echo -e "${YELLOW}Running components in parallel (max 6 workers)...${NC}" - echo "" +check_helm_lint() { + helm lint helm/node-doctor +} - # Run components in parallel using background jobs - PIDS=() - for component in "${COMPONENTS[@]}"; do - validate_component "$component" & - PIDS+=($!) +check_helm_generated() { + make -s helm-verify-generated +} - # Limit to 6 parallel workers - if [ ${#PIDS[@]} -ge 6 ]; then - wait -n - PIDS=($(jobs -p)) - fi - done +run gofmt check_gofmt +run "go vet (GOOS=linux)" check_vet - # Wait for all remaining jobs - for pid in "${PIDS[@]}"; do - wait "$pid" - COMPONENT_RESULTS+=($?) - done +if command -v golangci-lint >/dev/null 2>&1; then + run golangci-lint check_lint +elif [ "${SKIP_LINT:-0}" = "1" ]; then + skip golangci-lint "not installed, SKIP_LINT=1" else - # Sequential execution - for component in "${COMPONENTS[@]}"; do - validate_component "$component" - COMPONENT_RESULTS+=($?) - done + echo "==> golangci-lint" + echo "golangci-lint is not installed. Install it (make install-deps) or set SKIP_LINT=1." >&2 + FAILED+=("golangci-lint") fi -# Cleanup temp files -rm -f /tmp/validate_*.log +if command -v gosec >/dev/null 2>&1; then + run gosec check_gosec +else + skip gosec "not installed" +fi -# Print summary -echo -e "${BLUE}════════════════════════════════════════════════════════════════${NC}" -echo -e "${BLUE}Validation Summary${NC}" -echo -e "${BLUE}════════════════════════════════════════════════════════════════${NC}" -echo "" -echo -e "Total checks: $TOTAL_CHECKS" -echo -e "${GREEN}Passed: $PASSED_CHECKS${NC}" -echo -e "${RED}Failed: $FAILED_CHECKS${NC}" -echo "" +if [ "$QUICK" = true ]; then + skip "go test" "--quick" +else + run "go test" check_tests +fi -# Component-level results -echo -e "${BLUE}Component Results:${NC}" -for i in "${!COMPONENTS[@]}"; do - component="${COMPONENTS[$i]}" - result="${COMPONENT_RESULTS[$i]}" +run "helm lint" check_helm_lint +run helm-verify-generated check_helm_generated - if [ "$result" -eq 0 ]; then - echo -e " ${GREEN}✓ $component${NC}" - else - echo -e " ${RED}✗ $component${NC}" - fi -done -echo "" +echo +echo "Summary" +for name in "${PASSED[@]}"; do echo " PASS $name"; done +for name in "${SKIPPED[@]}"; do echo " SKIP $name"; done +for name in "${FAILED[@]}"; do echo " FAIL $name"; done -# Exit with appropriate code -if [ $FAILED_CHECKS -eq 0 ]; then - echo -e "${GREEN}╔════════════════════════════════════════════════════════════════╗${NC}" - echo -e "${GREEN}║ ✓ All validations passed! Safe to push. ║${NC}" - echo -e "${GREEN}╚════════════════════════════════════════════════════════════════╝${NC}" - exit 0 -else - echo -e "${RED}╔════════════════════════════════════════════════════════════════╗${NC}" - echo -e "${RED}║ ✗ Validation failed! Fix issues before pushing. ║${NC}" - echo -e "${RED}╚════════════════════════════════════════════════════════════════╝${NC}" - echo "" - echo -e "${YELLOW}See logs above for details.${NC}" - echo -e "${YELLOW}Run with --component=NAME to validate specific component.${NC}" - echo -e "${YELLOW}Run with --quick for faster feedback (skips tests).${NC}" +if [ ${#FAILED[@]} -gt 0 ]; then + echo + echo "${#FAILED[@]} check(s) failed." >&2 exit 1 fi +echo +echo "All checks passed." diff --git a/scripts/validate-repo-cleanliness.sh b/scripts/validate-repo-cleanliness.sh index b0d75b6..ddc179d 100755 --- a/scripts/validate-repo-cleanliness.sh +++ b/scripts/validate-repo-cleanliness.sh @@ -13,31 +13,16 @@ echo "Checking repository cleanliness..." ROOT_BLACKLIST=( "*.go" # Go files belong in proper directories "test-*" # Test files - "*test*" # Any test files (except .gitignore, etc) "*.tmp" # Temporary files "*.log" # Log files "*.bak" # Backup files "*~" # Editor backup files "*.swp" # Vim swap files "*.swo" # Vim swap files - "Dockerfile.*" # Test Dockerfiles "*.out" # Output files "*.exe" # Executables ) -# Allowed files in root (whitelist) -ROOT_WHITELIST=( - ".gitignore" - ".gitattributes" - ".validation.json" - "README.md" - "LICENSE" - "CLAUDE.md" - "Makefile" - "makefile" - "makefile.windows" -) - echo "" echo "Checking root directory..." @@ -130,8 +115,6 @@ echo "" if [ $ERRORS -gt 0 ]; then echo "❌ Repository cleanliness check FAILED with $ERRORS errors!" echo "" - echo "To fix these issues, run: ./scripts/cleanup-repo.sh" - echo "" echo "Rules for keeping the repository clean:" echo "1. No Go files in the root directory - use proper package structure" echo "2. No test files or temporary files in the root" diff --git a/scripts/verify-gpg-setup.sh b/scripts/verify-gpg-setup.sh deleted file mode 100755 index c09082b..0000000 --- a/scripts/verify-gpg-setup.sh +++ /dev/null @@ -1,185 +0,0 @@ -#!/bin/bash -set -e - -# verify-gpg-setup.sh -# Verifies GPG signing configuration for GitHub Actions - -echo "🔍 Verifying GPG Setup for Node Doctor Release Automation" -echo "================================================================" -echo "" - -# Colors -GREEN='\033[0;32m' -RED='\033[0;31m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -ERRORS=0 -WARNINGS=0 - -# Check 1: Verify GPG key exists locally -echo "1ī¸âƒŖ Checking local GPG key..." -if gpg --list-secret-keys 1AF32A8B0481A7F3 > /dev/null 2>&1; then - echo -e " ${GREEN}✅ GPG key 1AF32A8B0481A7F3 found locally${NC}" - - # Get key details - KEY_CREATED=$(gpg --list-secret-keys --with-colons 1AF32A8B0481A7F3 | grep ^sec | cut -d: -f6) - KEY_EMAIL=$(gpg --list-secret-keys 1AF32A8B0481A7F3 | grep uid | grep -oP '<.*>' | tr -d '<>') - echo " 📧 Email: $KEY_EMAIL" - echo " 📅 Created: $(date -d @$KEY_CREATED 2>/dev/null || date -r $KEY_CREATED 2>/dev/null || echo 'Unknown')" -else - echo -e " ${RED}❌ GPG key 1AF32A8B0481A7F3 NOT found locally${NC}" - ERRORS=$((ERRORS + 1)) -fi -echo "" - -# Check 2: Verify GitHub secrets are configured -echo "2ī¸âƒŖ Checking GitHub repository secrets..." -if command -v gh &> /dev/null; then - SECRET_LIST=$(gh secret list 2>&1) - if echo "$SECRET_LIST" | grep -q "404"; then - echo -e " ${YELLOW}âš ī¸ Cannot verify secrets (permission issue)${NC}" - echo " Run manually: gh secret list" - WARNINGS=$((WARNINGS + 1)) - else - # Check for GPG_PRIVATE_KEY - if echo "$SECRET_LIST" | grep -q "GPG_PRIVATE_KEY"; then - echo -e " ${GREEN}✅ GPG_PRIVATE_KEY secret is configured${NC}" - else - echo -e " ${RED}❌ GPG_PRIVATE_KEY secret NOT configured${NC}" - ERRORS=$((ERRORS + 1)) - fi - - # Check for GPG_PASSPHRASE - if echo "$SECRET_LIST" | grep -q "GPG_PASSPHRASE"; then - echo -e " ${GREEN}✅ GPG_PASSPHRASE secret is configured${NC}" - else - echo -e " ${RED}❌ GPG_PASSPHRASE secret NOT configured${NC}" - ERRORS=$((ERRORS + 1)) - fi - fi -else - echo -e " ${YELLOW}âš ī¸ GitHub CLI (gh) not installed - skipping secret check${NC}" - WARNINGS=$((WARNINGS + 1)) -fi -echo "" - -# Check 3: Verify key can be exported (test without actually exporting) -echo "3ī¸âƒŖ Checking if key can be exported..." -if gpg --list-secret-keys 1AF32A8B0481A7F3 > /dev/null 2>&1; then - echo -e " ${GREEN}✅ Key is accessible for export${NC}" - echo " â„šī¸ To export: gpg --armor --export-secret-key 1AF32A8B0481A7F3 > /tmp/gpg-private-key.asc" -else - echo -e " ${RED}❌ Cannot access key for export${NC}" - ERRORS=$((ERRORS + 1)) -fi -echo "" - -# Check 4: Verify release workflow configuration -echo "4ī¸âƒŖ Checking release workflow GPG configuration..." -RELEASE_WORKFLOW=".github/workflows/release.yml" -if [ -f "$RELEASE_WORKFLOW" ]; then - if grep -q "GPG_PRIVATE_KEY" "$RELEASE_WORKFLOW"; then - echo -e " ${GREEN}✅ Release workflow references GPG_PRIVATE_KEY${NC}" - else - echo -e " ${RED}❌ Release workflow missing GPG_PRIVATE_KEY reference${NC}" - ERRORS=$((ERRORS + 1)) - fi - - if grep -q "GPG_PASSPHRASE" "$RELEASE_WORKFLOW"; then - echo -e " ${GREEN}✅ Release workflow references GPG_PASSPHRASE${NC}" - else - echo -e " ${RED}❌ Release workflow missing GPG_PASSPHRASE reference${NC}" - ERRORS=$((ERRORS + 1)) - fi - - # Check for GPG signing step - if grep -q "gpg --batch --import" "$RELEASE_WORKFLOW"; then - echo -e " ${GREEN}✅ GPG key import step configured${NC}" - else - echo -e " ${YELLOW}âš ī¸ GPG key import step not found${NC}" - WARNINGS=$((WARNINGS + 1)) - fi - - # Check for GPG signing - if grep -q "gpg.*--detach-sign" "$RELEASE_WORKFLOW"; then - echo -e " ${GREEN}✅ GPG artifact signing configured${NC}" - else - echo -e " ${YELLOW}âš ī¸ GPG artifact signing not found${NC}" - WARNINGS=$((WARNINGS + 1)) - fi -else - echo -e " ${RED}❌ Release workflow not found at $RELEASE_WORKFLOW${NC}" - ERRORS=$((ERRORS + 1)) -fi -echo "" - -# Check 5: Test GPG signing capability (if key is available) -echo "5ī¸âƒŖ Testing GPG signing capability..." -TEST_FILE="/tmp/gpg-test-$$" -echo "Test content for GPG signing verification" > "$TEST_FILE" - -if gpg --list-secret-keys 1AF32A8B0481A7F3 > /dev/null 2>&1; then - # Try to sign with --batch (will fail if passphrase needed) - if echo "test-passphrase" | gpg --batch --yes --passphrase-fd 0 --armor --detach-sign --output "${TEST_FILE}.asc" "$TEST_FILE" 2>/dev/null; then - echo -e " ${YELLOW}âš ī¸ GPG signing works but test passphrase was accepted${NC}" - echo " â„šī¸ Make sure to use your actual passphrase in GitHub secrets" - WARNINGS=$((WARNINGS + 1)) - rm -f "${TEST_FILE}.asc" - else - echo -e " ${GREEN}✅ GPG key requires passphrase (as expected)${NC}" - echo " â„šī¸ GitHub Actions will use GPG_PASSPHRASE secret during release" - fi -else - echo -e " ${YELLOW}âš ī¸ Cannot test signing - key not accessible${NC}" - WARNINGS=$((WARNINGS + 1)) -fi -rm -f "$TEST_FILE" -echo "" - -# Summary -echo "================================================================" -echo "📊 Verification Summary" -echo "================================================================" -if [ $ERRORS -eq 0 ] && [ $WARNINGS -eq 0 ]; then - echo -e "${GREEN}✅ ALL CHECKS PASSED!${NC}" - echo "" - echo "GPG signing is properly configured. Next steps:" - echo "1. Test with a pre-release: git tag v0.1.0-rc.1 && git push origin v0.1.0-rc.1" - echo "2. Monitor the release workflow in GitHub Actions" - echo "3. Verify dual signatures are created (.cosign.sig and .asc files)" -elif [ $ERRORS -eq 0 ]; then - echo -e "${YELLOW}âš ī¸ PASSED WITH WARNINGS ($WARNINGS warnings)${NC}" - echo "" - echo "GPG configuration is mostly ready, but review warnings above." -else - echo -e "${RED}❌ VERIFICATION FAILED ($ERRORS errors, $WARNINGS warnings)${NC}" - echo "" - echo "Please address the errors above before proceeding with releases." - exit 1 -fi - -echo "" -echo "================================================================" -echo "📚 Quick Reference" -echo "================================================================" -echo "" -echo "Export GPG private key:" -echo " gpg --armor --export-secret-key 1AF32A8B0481A7F3 > /tmp/gpg-private-key.asc" -echo "" -echo "Add GitHub secrets (manual via UI):" -echo " 1. Go to: https://github.com/supporttools/node-doctor/settings/secrets/actions" -echo " 2. Click 'New repository secret'" -echo " 3. Name: GPG_PRIVATE_KEY" -echo " Value: " -echo " 4. Click 'New repository secret'" -echo " 5. Name: GPG_PASSPHRASE" -echo " Value: " -echo "" -echo "Or add via CLI:" -echo " gh secret set GPG_PRIVATE_KEY < /tmp/gpg-private-key.asc" -echo " gh secret set GPG_PASSPHRASE # will prompt for value" -echo "" -echo "Verify secrets:" -echo " gh secret list" -echo "" From 8966a2a071f8a472bfc5680b180570d8e423065b Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Wed, 7 Oct 2026 09:39:08 -0500 Subject: [PATCH 4/6] build(controller): build and publish the controller image The chart and deploy manifests referenced supporttools/node-doctor-controller, but nothing built or published it, so enabling the controller always ended in ImagePullBackOff. Add Dockerfile.controller (static Go binary on scratch, non-root, writable /data for SQLite), wire it into the Makefile image targets, build it in ci.yml, and publish, sign and guard it in release.yml alongside the other two images. The chart's controller tag now follows image.tag instead of appVersion, which carries a v prefix that no published image tag has. deploy/controller pins the current release instead of :latest, and the e2e harness builds and loads the controller image kind expects. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 21 +++++ .github/workflows/release.yml | 86 ++++++++++++++++--- Dockerfile.controller | 46 ++++++++++ Makefile | 27 +++++- deploy/controller/deployment.yaml | 2 +- .../templates/controller-deployment.yaml | 2 +- helm/node-doctor/values.yaml | 4 +- helm/node-doctor/values.yaml.template | 4 +- test/e2e/cluster/image.go | 41 +++++---- 9 files changed, 195 insertions(+), 38 deletions(-) create mode 100644 Dockerfile.controller diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7b5dac..dc58c08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -199,6 +199,12 @@ jobs: cd cmd/overlay-test-server go build -o ../../bin/overlay-test-server + - name: Build node-doctor-controller binary + run: | + mkdir -p bin + cd cmd/node-doctor-controller + go build -o ../../bin/node-doctor-controller + - name: Test binary run: ./bin/node-doctor --version @@ -273,6 +279,21 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max + - name: Build controller Docker image (no push) + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile.controller + platforms: linux/amd64 + push: false + tags: ${{ env.REGISTRY }}/node-doctor-controller:ci-${{ github.sha }} + build-args: | + VERSION=${{ steps.meta.outputs.VERSION }} + GIT_COMMIT=${{ steps.meta.outputs.GIT_COMMIT }} + BUILD_TIME=${{ steps.meta.outputs.BUILD_TIME }} + cache-from: type=gha,scope=controller + cache-to: type=gha,mode=max,scope=controller + # Summary job - provides overall status # # New gating jobs are wired in here rather than added as new required status checks: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7692003..d79e59f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -171,11 +171,72 @@ jobs: cache-from: type=gha,scope=overlay-test cache-to: type=gha,mode=max,scope=overlay-test + # Build, push and sign the controller Docker image (multi-arch) + docker-controller: + name: Build & Push Controller Image + runs-on: self-hosted-linux + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/supporttools/node-doctor-controller + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=latest,enable=${{ !contains(github.ref_name, 'rc') && !contains(github.ref_name, 'beta') && !contains(github.ref_name, 'alpha') }} + + - name: Build and push controller image + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile.controller + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + VERSION=${{ github.ref_name }} + GIT_COMMIT=${{ github.sha }} + BUILD_TIME=${{ github.event.head_commit.timestamp }} + cache-from: type=gha,scope=controller + cache-to: type=gha,mode=max,scope=controller + + - name: Verify pre-baked cosign + run: | + set -euo pipefail + command -v cosign >/dev/null 2>&1 || { echo "::error::cosign is missing from this self-hosted runner. It must be pre-baked into the runner image - there is deliberately no download fallback. Fix the runner image."; exit 1; } + cosign version + + - name: Sign controller image + env: + COSIGN_EXPERIMENTAL: 1 + IMAGE_REF: ${{ env.REGISTRY }}/supporttools/node-doctor-controller:${{ steps.meta.outputs.version }} + run: | + echo "Signing controller image with cosign (keyless)..." + echo "Tag: ${IMAGE_REF}" + cosign sign --yes "${IMAGE_REF}" + # Create GitHub Release github-release: name: Create GitHub Release runs-on: self-hosted-linux - needs: [docker, docker-overlay-test] + needs: [docker, docker-overlay-test, docker-controller] steps: - name: Checkout code uses: actions/checkout@v5 @@ -242,13 +303,13 @@ jobs: # Helm Chart Publishing # - # Depends on docker-overlay-test as well as docker: the chart's default values reference - # BOTH images, and the post-package guard below asserts those references actually resolve - # in the registry. That assertion is only meaningful once both images have been pushed. + # Depends on every image job: the chart's default values reference all three images, and + # the post-package guard below asserts those references actually resolve in the registry. + # That assertion is only meaningful once all of them have been pushed. helm-publish: name: Publish Helm Chart runs-on: self-hosted-linux - needs: [docker, docker-overlay-test] + needs: [docker, docker-overlay-test, docker-controller] outputs: chart-version: ${{ steps.get-chart-version.outputs.version }} @@ -289,7 +350,7 @@ jobs: # docker/metadata-action in the `docker` job publishes # `type=semver,pattern={{version}}`, which strips the leading v: the image lands on # Docker Hub as `1.8.7`, never `v1.8.7`. values.yaml.template feeds ${IMAGE_TAG} into - # BOTH image.tag and overlayTest.image.tag, so when IMAGE_TAG carried the v, every + # image.tag, overlayTest.image.tag and controller.image.tag, so when IMAGE_TAG carried the v, every # published chart defaulted to an image tag that was never pushed — # `helm install supporttools/node-doctor` with default values gave ImagePullBackOff # on both DaemonSets. node-doctor-overlay-test has never had a v-prefixed tag for any @@ -354,7 +415,7 @@ jobs: with open('/tmp/chart-verify/node-doctor/values.yaml') as fh: v = yaml.safe_load(fh) refs = [] - for path in (('image',), ('overlayTest', 'image')): + for path in (('image',), ('overlayTest', 'image'), ('controller', 'image')): node = v for key in path: node = (node or {}).get(key) @@ -509,7 +570,7 @@ jobs: release-complete: name: Release Complete runs-on: self-hosted-linux - needs: [docker, docker-overlay-test, github-release, helm-publish] + needs: [docker, docker-overlay-test, docker-controller, github-release, helm-publish] if: always() steps: - name: Check release status @@ -525,6 +586,10 @@ jobs: echo "❌ Overlay test Docker build job failed" exit 1 fi + if [[ "${{ needs.docker-controller.result }}" != "success" ]]; then + echo "❌ Controller Docker build job failed" + exit 1 + fi if [[ "${{ needs.github-release.result }}" != "success" ]]; then echo "❌ GitHub release job failed" exit 1 @@ -556,7 +621,7 @@ jobs: notify-failure: name: Notify Release Failure runs-on: self-hosted-linux - needs: [docker, docker-overlay-test, github-release, helm-publish, grype-scan, release-complete] + needs: [docker, docker-overlay-test, docker-controller, github-release, helm-publish, grype-scan, release-complete] if: failure() steps: # A tag name may legally contain characters that would break out of the JSON @@ -580,6 +645,7 @@ jobs: echo "| --- | --- |" echo "| Docker images | ${{ needs.docker.result }} |" echo "| Overlay test image | ${{ needs.docker-overlay-test.result }} |" + echo "| Controller image | ${{ needs.docker-controller.result }} |" echo "| GitHub release | ${{ needs.github-release.result }} |" echo "| Helm chart publish | ${{ needs.helm-publish.result }} |" echo "| Grype scan | ${{ needs.grype-scan.result }} |" @@ -600,7 +666,7 @@ jobs: "type": "section", "text": { "type": "mrkdwn", - "text": "❌ *node-doctor* release *${{ steps.ctx.outputs.ref }}* *FAILED*\n\n*Docker images:* `${{ needs.docker.result }}`\n*Overlay test image:* `${{ needs.docker-overlay-test.result }}`\n*GitHub release:* `${{ needs.github-release.result }}`\n*Helm chart publish:* `${{ needs.helm-publish.result }}`\n\nA failed chart publish means charts.support.tools still serves the previous version.\n\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Workflow Run>" + "text": "❌ *node-doctor* release *${{ steps.ctx.outputs.ref }}* *FAILED*\n\n*Docker images:* `${{ needs.docker.result }}`\n*Overlay test image:* `${{ needs.docker-overlay-test.result }}`\n*Controller image:* `${{ needs.docker-controller.result }}`\n*GitHub release:* `${{ needs.github-release.result }}`\n*Helm chart publish:* `${{ needs.helm-publish.result }}`\n\nA failed chart publish means charts.support.tools still serves the previous version.\n\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Workflow Run>" } } ] diff --git a/Dockerfile.controller b/Dockerfile.controller new file mode 100644 index 0000000..202c74f --- /dev/null +++ b/Dockerfile.controller @@ -0,0 +1,46 @@ +# Dockerfile for node-doctor-controller - cluster-wide coordination API for node-doctor agents +# Pure-Go SQLite (modernc.org/sqlite), so the binary is fully static and runs from scratch. + +FROM golang:1.25-alpine AS builder + +RUN apk add --no-cache git + +WORKDIR /build + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +ARG VERSION=dev +ARG GIT_COMMIT=unknown +ARG BUILD_TIME=unknown +ARG TARGETARCH + +RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build \ + -ldflags="-X main.Version=${VERSION} -X main.GitCommit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME} -w -s" \ + -trimpath \ + -o node-doctor-controller \ + ./cmd/node-doctor-controller + +# SQLite needs a writable data dir; in-cluster this is replaced by the PVC mount. +RUN mkdir -p /data && chown 65534:65534 /data + +FROM scratch + +COPY --from=builder /build/node-doctor-controller /usr/local/bin/node-doctor-controller +COPY --from=builder --chown=65534:65534 /data /data + +USER 65534:65534 + +# 8080: HTTP API, health and metrics endpoint +EXPOSE 8080 + +ENTRYPOINT ["/usr/local/bin/node-doctor-controller"] + +LABEL org.opencontainers.image.title="Node Doctor Controller" \ + org.opencontainers.image.description="Cluster-wide coordination controller for node-doctor agents" \ + org.opencontainers.image.vendor="SupportTools" \ + org.opencontainers.image.url="https://github.com/supporttools/node-doctor" \ + org.opencontainers.image.source="https://github.com/supporttools/node-doctor" \ + org.opencontainers.image.licenses="Apache-2.0" diff --git a/Makefile b/Makefile index ae62296..f38e132 100644 --- a/Makefile +++ b/Makefile @@ -41,11 +41,12 @@ CHART_VERSION := v$(shell git rev-list --count HEAD) # ================================================================================================ # Node Doctor binaries -COMPONENTS := node-doctor overlay-test-server +COMPONENTS := node-doctor overlay-test-server node-doctor-controller # Docker images DOCKER_IMAGE_node-doctor := $(REGISTRY)/$(PROJECT_NAME) DOCKER_IMAGE_overlay-test-server := $(REGISTRY)/$(PROJECT_NAME)-overlay-test +DOCKER_IMAGE_node-doctor-controller := $(REGISTRY)/$(PROJECT_NAME)-controller # ================================================================================================ # Color Output Functions @@ -134,6 +135,12 @@ build-overlay-test-server-local: @cd cmd/overlay-test-server && go build -o ../../bin/overlay-test-server @$(call print_success,"overlay-test-server built: bin/overlay-test-server") +build-node-doctor-controller-local: + @$(call print_status,"Building node-doctor-controller locally...") + @mkdir -p bin + @cd cmd/node-doctor-controller && go build -ldflags="-X main.Version=$(VERSION) -X main.GitCommit=$(GIT_COMMIT) -X main.BuildTime=$(BUILD_TIME)" -o ../../bin/node-doctor-controller + @$(call print_success,"node-doctor-controller built: bin/node-doctor-controller") + build-local: build-all-local # ================================================================================================ @@ -326,8 +333,16 @@ build-overlay-test-server-image: check-docker @docker build -f Dockerfile.overlay-test -t $(DOCKER_IMAGE_overlay-test-server):$(VERSION) . @$(call print_success,"overlay-test-server image built: $(DOCKER_IMAGE_overlay-test-server):$(VERSION)") +build-node-doctor-controller-image: check-docker + @$(call print_status,"Building node-doctor-controller Docker image...") + @docker build -f Dockerfile.controller \ + --build-arg VERSION=$(VERSION) --build-arg GIT_COMMIT=$(GIT_COMMIT) --build-arg BUILD_TIME=$(BUILD_TIME) \ + -t $(DOCKER_IMAGE_node-doctor-controller):$(VERSION) . + @docker tag $(DOCKER_IMAGE_node-doctor-controller):$(VERSION) $(DOCKER_IMAGE_node-doctor-controller):latest + @$(call print_success,"node-doctor-controller image built: $(DOCKER_IMAGE_node-doctor-controller):$(VERSION)") + # Build all images -build-all-images: build-node-doctor-image build-overlay-test-server-image +build-all-images: build-node-doctor-image build-overlay-test-server-image build-node-doctor-controller-image @$(call print_success,"Docker images built successfully") # Push node-doctor image to registry @@ -342,7 +357,13 @@ push-overlay-test-server-image: require-version @docker push $(DOCKER_IMAGE_overlay-test-server):$(VERSION) @$(call print_success,"overlay-test-server image pushed") -push-all-images: push-node-doctor-image push-overlay-test-server-image +push-node-doctor-controller-image: + @$(call print_status,"Pushing node-doctor-controller image to registry...") + @docker push $(DOCKER_IMAGE_node-doctor-controller):$(VERSION) + @docker push $(DOCKER_IMAGE_node-doctor-controller):latest + @$(call print_success,"node-doctor-controller image pushed") + +push-all-images: push-node-doctor-image push-overlay-test-server-image push-node-doctor-controller-image @$(call print_success,"Images pushed to registry") # ================================================================================================ diff --git a/deploy/controller/deployment.yaml b/deploy/controller/deployment.yaml index a546dd5..b9df478 100644 --- a/deploy/controller/deployment.yaml +++ b/deploy/controller/deployment.yaml @@ -34,7 +34,7 @@ spec: fsGroup: 65534 containers: - name: controller - image: supporttools/node-doctor-controller:latest + image: supporttools/node-doctor-controller:1.9.0 imagePullPolicy: IfNotPresent args: - --config=/etc/node-doctor/controller.yaml diff --git a/helm/node-doctor/templates/controller-deployment.yaml b/helm/node-doctor/templates/controller-deployment.yaml index e4678bb..eeaf92b 100644 --- a/helm/node-doctor/templates/controller-deployment.yaml +++ b/helm/node-doctor/templates/controller-deployment.yaml @@ -34,7 +34,7 @@ spec: {{- end }} containers: - name: controller - image: "{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag | default .Chart.AppVersion }}" + image: "{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag | default .Values.image.tag }}" imagePullPolicy: {{ .Values.controller.image.pullPolicy }} args: - --config=/etc/node-doctor/controller.yaml diff --git a/helm/node-doctor/values.yaml b/helm/node-doctor/values.yaml index 8803a81..47bbe98 100644 --- a/helm/node-doctor/values.yaml +++ b/helm/node-doctor/values.yaml @@ -590,8 +590,8 @@ controller: image: repository: supporttools/node-doctor-controller pullPolicy: IfNotPresent - # Defaults to chart appVersion when empty - tag: "" + # Falls back to image.tag when empty + tag: "1.0.0" # Service account for the controller serviceAccount: diff --git a/helm/node-doctor/values.yaml.template b/helm/node-doctor/values.yaml.template index f4f71f9..23a7e32 100644 --- a/helm/node-doctor/values.yaml.template +++ b/helm/node-doctor/values.yaml.template @@ -590,8 +590,8 @@ controller: image: repository: supporttools/node-doctor-controller pullPolicy: IfNotPresent - # Defaults to chart appVersion when empty - tag: "" + # Falls back to image.tag when empty + tag: "${IMAGE_TAG}" # Service account for the controller serviceAccount: diff --git a/test/e2e/cluster/image.go b/test/e2e/cluster/image.go index 601efc8..8c1c30b 100644 --- a/test/e2e/cluster/image.go +++ b/test/e2e/cluster/image.go @@ -18,42 +18,45 @@ import ( "time" ) -// BuildAndLoadImage builds the Node Doctor Docker image and loads it into KIND cluster +var e2eImages = []struct { + name string + dockerfile string +}{ + {name: "node-doctor:e2e-test", dockerfile: "Dockerfile"}, + {name: "node-doctor-controller:e2e-test", dockerfile: "Dockerfile.controller"}, +} + +// BuildAndLoadImage builds the agent and controller images and loads them into the KIND cluster func BuildAndLoadImage(ctx context.Context, clusterName string) error { - // Get project root (should be 3 levels up from test/e2e/cluster) projectRoot, err := getProjectRoot() if err != nil { return fmt.Errorf("failed to find project root: %w", err) } - imageName := "node-doctor:e2e-test" + for _, img := range e2eImages { + fmt.Printf("Building Docker image: %s\n", img.name) + if err := buildDockerImage(ctx, projectRoot, img.dockerfile, img.name); err != nil { + return fmt.Errorf("failed to build image %s: %w", img.name, err) + } - // Step 1: Build Docker image - fmt.Printf("Building Docker image: %s\n", imageName) - if err := buildDockerImage(ctx, projectRoot, imageName); err != nil { - return fmt.Errorf("failed to build image: %w", err) - } + fmt.Printf("Loading image into KIND cluster: %s\n", clusterName) + if err := loadImageIntoKIND(ctx, clusterName, img.name); err != nil { + return fmt.Errorf("failed to load image %s into KIND: %w", img.name, err) + } - // Step 2: Load image into KIND cluster - fmt.Printf("Loading image into KIND cluster: %s\n", clusterName) - if err := loadImageIntoKIND(ctx, clusterName, imageName); err != nil { - return fmt.Errorf("failed to load image into KIND: %w", err) + fmt.Printf("Image %s built and loaded successfully\n", img.name) } - fmt.Printf("Image %s built and loaded successfully\n", imageName) return nil } -// buildDockerImage builds the Node Doctor Docker image -func buildDockerImage(ctx context.Context, projectRoot, imageName string) error { - // Create timeout for build operation +func buildDockerImage(ctx context.Context, projectRoot, dockerfile, imageName string) error { buildCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) defer cancel() - // Check if Dockerfile exists - dockerfilePath := filepath.Join(projectRoot, "Dockerfile") + dockerfilePath := filepath.Join(projectRoot, dockerfile) if _, err := os.Stat(dockerfilePath); err != nil { - return fmt.Errorf("Dockerfile not found at %s: %w", dockerfilePath, err) + return fmt.Errorf("%s not found at %s: %w", dockerfile, dockerfilePath, err) } // Build the image From fed4d31d44d241b4ca7dc38bc487ebb8ed438a10 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Wed, 7 Oct 2026 09:39:09 -0500 Subject: [PATCH 5/6] fix(remediation): apply circuit breaker at startup, fix races, fix reload strategy validation The registry only received the circuit breaker config on hot reload, so a fresh process ran with the package default (5 / 5m / 2) instead of the configured values, and circuitBreaker.enabled was read nowhere outside defaults and validation. NewRegistryFromConfig now wires the registry at startup through the same conversion ApplyConfig uses, and a disabled breaker bypasses trip/open checks while reporting Closed. Circuit breaker defaults apply regardless of enabled so a disabled block no longer leaves zeros the reload guard rejects. Remediate read dryRun without the lock and recordRateLimitEntry read maxPerHour before taking it; both raced with the setters reachable from hot reload. The reload validator accepted only restart/recreate/script/webhook, none of which exist, so every real strategy failed reload validation. It now uses the exported strategy list from pkg/types, and the config error message builds from the same list. Co-Authored-By: Claude Fable 5.1 --- cmd/node-doctor/main.go | 21 ++-- pkg/reload/validator.go | 12 +- pkg/reload/validator_test.go | 39 +++++- pkg/remediators/registry.go | 97 ++++++++++++--- pkg/remediators/registry_applyconfig_test.go | 119 +++++++++++++++++++ pkg/remediators/registry_test.go | 43 +++++++ pkg/types/config.go | 48 +++++--- pkg/types/config_test.go | 35 ++++++ 8 files changed, 354 insertions(+), 60 deletions(-) diff --git a/cmd/node-doctor/main.go b/cmd/node-doctor/main.go index 6b39afc..42ee3db 100644 --- a/cmd/node-doctor/main.go +++ b/cmd/node-doctor/main.go @@ -281,21 +281,14 @@ func main() { // /remediation/history immediately on start (no race window). var remediatorRegistry *remediators.RemediatorRegistry if config.Remediation.Enabled { - maxPerHour := config.Remediation.MaxRemediationsPerHour - if maxPerHour == 0 { - maxPerHour = 10 // sensible default - } - historySize := config.Remediation.HistorySize - if historySize == 0 { - historySize = 100 + var err error + remediatorRegistry, err = remediators.NewRegistryFromConfig(&config.Remediation, config.Settings.DryRunMode) + if err != nil { + log.Fatalf("Failed to initialize remediator registry: %v", err) } - remediatorRegistry = remediators.NewRegistry(maxPerHour, historySize) - remediatorRegistry.SetDryRun(config.Remediation.DryRun || config.Settings.DryRunMode) - // Wire the per-minute token-bucket burst limit. A value of 0 leaves the - // per-minute check disabled (only the per-hour window applies). - remediatorRegistry.SetMaxRemediationsPerMinute(config.Remediation.MaxRemediationsPerMinute) - log.Printf("[INFO] Remediator registry initialized (dry-run=%v, maxPerHour=%d, maxPerMinute=%d)", - remediatorRegistry.IsDryRun(), maxPerHour, config.Remediation.MaxRemediationsPerMinute) + stats := remediatorRegistry.GetStats() + log.Printf("[INFO] Remediator registry initialized (dry-run=%v, maxPerHour=%d, maxPerMinute=%d, circuitBreaker=%+v)", + stats.DryRun, stats.MaxPerHour, config.Remediation.MaxRemediationsPerMinute, remediatorRegistry.GetCircuitBreakerConfig()) // Register the built-in remediator strategies so the detector's dispatch // (which addresses a remediator by its strategy type) can find one. diff --git a/pkg/reload/validator.go b/pkg/reload/validator.go index 403788f..a4964ba 100644 --- a/pkg/reload/validator.go +++ b/pkg/reload/validator.go @@ -444,7 +444,7 @@ func (v *ConfigValidator) validateRemediation(remediation *types.RemediationConf func (v *ConfigValidator) validateMonitorRemediation(remediation *types.MonitorRemediationConfig, prefix string, result *ValidationResult) { if remediation.Strategy == "" { v.addError(result, prefix+".strategy", "remediation strategy is required") - } else if !v.isValidRemediationStrategy(remediation.Strategy) { + } else if !types.IsValidRemediationStrategy(remediation.Strategy) { v.addError(result, prefix+".strategy", fmt.Sprintf("unsupported remediation strategy '%s'", remediation.Strategy)) } @@ -613,16 +613,6 @@ func (v *ConfigValidator) isValidMonitorType(monitorType string) bool { return false } -func (v *ConfigValidator) isValidRemediationStrategy(strategy string) bool { - validStrategies := []string{"restart", "recreate", "script", "webhook"} - for _, validStrategy := range validStrategies { - if strategy == validStrategy { - return true - } - } - return false -} - func (v *ConfigValidator) validatePort(port interface{}) error { var portNum int switch p := port.(type) { diff --git a/pkg/reload/validator_test.go b/pkg/reload/validator_test.go index 5bca040..d51df43 100644 --- a/pkg/reload/validator_test.go +++ b/pkg/reload/validator_test.go @@ -615,7 +615,7 @@ func TestValidate_RemediationConsistency(t *testing.T) { config.Remediation.Enabled = false config.Monitors[0].Remediation = &types.MonitorRemediationConfig{ Enabled: true, - Strategy: "restart", + Strategy: "systemd-restart", } validator := NewConfigValidator() @@ -1853,3 +1853,40 @@ func TestValidate_ReloadRejectsUnregisteredMonitorType(t *testing.T) { t.Errorf("Expected monitors[0].type error, got: %v", result.Errors) } } + +func TestValidateMonitorRemediation_StrategyNames(t *testing.T) { + validator := NewConfigValidator() + tests := []struct { + strategy string + valid bool + }{ + {"systemd-restart", true}, + {"webhook", false}, + } + for _, tt := range tests { + t.Run(tt.strategy, func(t *testing.T) { + result := &ValidationResult{Valid: true} + validator.validateMonitorRemediation(&types.MonitorRemediationConfig{ + Enabled: true, + Strategy: tt.strategy, + }, "monitors[0].remediation", result) + if result.Valid != tt.valid { + t.Errorf("strategy %q valid = %v, want %v (errors: %v)", tt.strategy, result.Valid, tt.valid, result.Errors) + } + }) + } +} + +func TestValidate_MonitorRemediationSystemdRestartPasses(t *testing.T) { + config := createValidConfig() + config.Monitors[0].Remediation = &types.MonitorRemediationConfig{ + Enabled: true, + Strategy: "systemd-restart", + Service: "kubelet", + } + + result := NewConfigValidator().Validate(config) + if !result.Valid { + t.Fatalf("reload validation must accept systemd-restart, got errors: %v", result.Errors) + } +} diff --git a/pkg/remediators/registry.go b/pkg/remediators/registry.go index 1540c83..dad5294 100644 --- a/pkg/remediators/registry.go +++ b/pkg/remediators/registry.go @@ -194,6 +194,9 @@ func (r RemediationRecord) MarshalJSON() ([]byte, error) { // CircuitBreakerConfig contains configuration for the circuit breaker. type CircuitBreakerConfig struct { + // Disabled makes every check pass and stops failures from tripping the circuit + Disabled bool + // Threshold is the number of consecutive failures before opening the circuit Threshold int @@ -309,6 +312,31 @@ func NewRegistry(maxPerHour, maxHistory int) *RemediatorRegistry { } } +// NewRegistryFromConfig builds a registry wired exactly as a hot reload would +// leave it, so startup and reload share one source of truth. +func NewRegistryFromConfig(cfg *types.RemediationConfig, dryRunMode bool) (*RemediatorRegistry, error) { + if cfg == nil { + return nil, fmt.Errorf("remediation config cannot be nil") + } + + maxPerHour := cfg.MaxRemediationsPerHour + if maxPerHour == 0 { + maxPerHour = types.DefaultMaxRemediationsPerHour + } + historySize := cfg.HistorySize + if historySize == 0 { + historySize = types.DefaultHistorySize + } + + r := NewRegistry(maxPerHour, historySize) + r.SetDryRun(cfg.DryRun || dryRunMode) + r.SetMaxRemediationsPerMinute(cfg.MaxRemediationsPerMinute) + if err := r.applyCircuitBreakerConfig(cfg.CircuitBreaker); err != nil { + return nil, err + } + return r, nil +} + // Register adds a new remediator type to the registry. // This function is typically called during initialization to register // available remediators. @@ -443,18 +471,8 @@ func (r *RemediatorRegistry) ApplyConfig(cfg *types.RemediationConfig, dryRunMod r.SetMaxRemediationsPerHour(cfg.MaxRemediationsPerHour) r.SetMaxRemediationsPerMinute(cfg.MaxRemediationsPerMinute) - // Only push a circuit-breaker update when the new values are actually - // usable. A zero/absent circuitBreaker block must not clobber the running - // configuration with invalid values, so treat it as "leave as-is". - cb := CircuitBreakerConfig{ - Threshold: cfg.CircuitBreaker.Threshold, - Timeout: cfg.CircuitBreaker.Timeout, - SuccessThreshold: cfg.CircuitBreaker.SuccessThreshold, - } - if cb.Threshold > 0 && cb.Timeout > 0 && cb.SuccessThreshold > 0 { - if err := r.SetCircuitBreakerConfig(cb); err != nil { - return fmt.Errorf("apply circuit breaker config: %w", err) - } + if err := r.applyCircuitBreakerConfig(cfg.CircuitBreaker); err != nil { + return err } r.logInfof("Remediation config reloaded in place (dryRun=%v maxPerHour=%d maxPerMinute=%d)", @@ -462,6 +480,25 @@ func (r *RemediatorRegistry) ApplyConfig(cfg *types.RemediationConfig, dryRunMod return nil } +// applyCircuitBreakerConfig converts the typed config and applies it. A +// zero/absent circuitBreaker block is treated as "leave as-is" so it cannot +// clobber the running configuration with invalid values. +func (r *RemediatorRegistry) applyCircuitBreakerConfig(cfg types.CircuitBreakerConfig) error { + cb := CircuitBreakerConfig{ + Disabled: !cfg.Enabled, + Threshold: cfg.Threshold, + Timeout: cfg.Timeout, + SuccessThreshold: cfg.SuccessThreshold, + } + if cb.Threshold <= 0 || cb.Timeout <= 0 || cb.SuccessThreshold <= 0 { + return nil + } + if err := r.SetCircuitBreakerConfig(cb); err != nil { + return fmt.Errorf("apply circuit breaker config: %w", err) + } + return nil +} + // SetCircuitStateObserver registers an observer that is notified of circuit // breaker state changes. The observer is called once immediately with the // current state (so a backing metric is correct from the start) and then on @@ -537,10 +574,23 @@ func (r *RemediatorRegistry) SetCircuitBreakerConfig(config CircuitBreakerConfig r.mu.Lock() defer r.mu.Unlock() r.circuitConfig = config + if config.Disabled && r.circuitState != CircuitClosed { + r.circuitState = CircuitClosed + r.circuitOpenedAt = time.Time{} + r.circuitLastStateChange = time.Now() + r.notifyCircuitStateObserver(r.circuitState) + } r.logInfof("Circuit breaker config updated: %+v", config) return nil } +// GetCircuitBreakerConfig returns the active circuit breaker configuration. +func (r *RemediatorRegistry) GetCircuitBreakerConfig() CircuitBreakerConfig { + r.mu.RLock() + defer r.mu.RUnlock() + return r.circuitConfig +} + // GetRegisteredTypes returns a sorted list of all registered remediator types. func (r *RemediatorRegistry) GetRegisteredTypes() []string { r.mu.RLock() @@ -757,11 +807,12 @@ func (r *RemediatorRegistry) Remediate(ctx context.Context, remediatorType strin return remediationErr } + dryRun := r.IsDryRun() r.logInfof("Executing remediation: type=%s problem=%s dry-run=%v", - remediatorType, GenerateProblemKey(problem), r.dryRun) + remediatorType, GenerateProblemKey(problem), dryRun) // Phase 5: Execute remediation (or skip in dry-run mode) - if r.dryRun { + if dryRun { r.logInfof("[DRY-RUN] Would execute remediation for %s", GenerateProblemKey(problem)) success = true r.recordCircuitBreakerSuccess() @@ -843,6 +894,10 @@ func (r *RemediatorRegistry) RemediateWithStrategies(ctx context.Context, strate // checkCircuitBreaker checks if the circuit breaker allows remediation. // This must be called with the lock held. func (r *RemediatorRegistry) checkCircuitBreaker() error { + if r.circuitConfig.Disabled { + return nil + } + switch r.circuitState { case CircuitClosed: // Normal operation @@ -925,13 +980,13 @@ func (r *RemediatorRegistry) checkPerMinuteRate() error { // recordRateLimitEntry records a successful remediation for rate limiting. func (r *RemediatorRegistry) recordRateLimitEntry() { + r.mu.Lock() + defer r.mu.Unlock() + if r.maxPerHour == 0 { return // Rate limiting disabled } - r.mu.Lock() - defer r.mu.Unlock() - r.remediationTimes = append(r.remediationTimes, time.Now()) } @@ -940,6 +995,10 @@ func (r *RemediatorRegistry) recordCircuitBreakerSuccess() { r.mu.Lock() defer r.mu.Unlock() + if r.circuitConfig.Disabled { + return + } + r.consecutiveFailures = 0 r.consecutiveSuccesses++ @@ -960,6 +1019,10 @@ func (r *RemediatorRegistry) recordCircuitBreakerFailure() { r.mu.Lock() defer r.mu.Unlock() + if r.circuitConfig.Disabled { + return + } + r.consecutiveSuccesses = 0 r.consecutiveFailures++ diff --git a/pkg/remediators/registry_applyconfig_test.go b/pkg/remediators/registry_applyconfig_test.go index b33bfc5..663cd05 100644 --- a/pkg/remediators/registry_applyconfig_test.go +++ b/pkg/remediators/registry_applyconfig_test.go @@ -1,6 +1,8 @@ package remediators import ( + "context" + "strings" "testing" "time" @@ -131,3 +133,120 @@ func TestApplyConfigRejectsNil(t *testing.T) { t.Error("ApplyConfig(nil) must return an error") } } + +func TestNewRegistryFromConfigAppliesCircuitBreakerAtStartup(t *testing.T) { + r, err := NewRegistryFromConfig(&types.RemediationConfig{ + Enabled: true, + CircuitBreaker: types.CircuitBreakerConfig{ + Enabled: true, + Threshold: 5, + Timeout: 7 * time.Minute, + SuccessThreshold: 2, + }, + }, false) + if err != nil { + t.Fatalf("NewRegistryFromConfig: %v", err) + } + + got := r.GetCircuitBreakerConfig() + if got.Timeout != 7*time.Minute { + t.Errorf("circuit breaker timeout = %v, want 7m applied at startup without a reload", got.Timeout) + } + if got.Disabled { + t.Error("circuit breaker must be enabled when config says enabled") + } +} + +func TestNewRegistryFromConfigDefaults(t *testing.T) { + r, err := NewRegistryFromConfig(&types.RemediationConfig{Enabled: true}, true) + if err != nil { + t.Fatalf("NewRegistryFromConfig: %v", err) + } + + stats := r.GetStats() + if stats.MaxPerHour != types.DefaultMaxRemediationsPerHour { + t.Errorf("MaxPerHour = %d, want %d", stats.MaxPerHour, types.DefaultMaxRemediationsPerHour) + } + if stats.MaxHistory != types.DefaultHistorySize { + t.Errorf("MaxHistory = %d, want %d", stats.MaxHistory, types.DefaultHistorySize) + } + if !stats.DryRun { + t.Error("process-wide dry-run must be honored at startup") + } + if r.GetCircuitBreakerConfig() != DefaultCircuitBreakerConfig { + t.Errorf("an absent circuitBreaker block must leave the default, got %+v", r.GetCircuitBreakerConfig()) + } +} + +func TestNewRegistryFromConfigRejectsNil(t *testing.T) { + if _, err := NewRegistryFromConfig(nil, false); err == nil { + t.Error("NewRegistryFromConfig(nil) must return an error") + } +} + +func TestCircuitBreakerDisabledNeverBlocks(t *testing.T) { + r, err := NewRegistryFromConfig(&types.RemediationConfig{ + Enabled: true, + CircuitBreaker: types.CircuitBreakerConfig{ + Enabled: false, + Threshold: 1, + Timeout: time.Hour, + SuccessThreshold: 1, + }, + }, false) + if err != nil { + t.Fatalf("NewRegistryFromConfig: %v", err) + } + + mock := newMockRemediator("test", true) + r.Register(RemediatorInfo{ + Type: "test", + Factory: func() (types.Remediator, error) { return mock, nil }, + }) + problem := createTestProblem("test-type", "test-resource") + + for i := 0; i < 5; i++ { + mock.ClearCooldown(problem) + mock.ResetAttempts(problem) + err := r.Remediate(context.Background(), "test", problem) + if err == nil { + t.Fatalf("attempt %d: expected the failing remediator's error", i) + } + if strings.Contains(err.Error(), "circuit breaker") { + t.Fatalf("attempt %d: disabled circuit breaker must never block, got %v", i, err) + } + } + if mock.getCallCount() != 5 { + t.Errorf("remediator called %d times, want 5", mock.getCallCount()) + } + if r.GetCircuitState() != CircuitClosed { + t.Errorf("circuit state = %v, want Closed while disabled", r.GetCircuitState()) + } +} + +func TestApplyConfigDisablingCircuitBreakerClosesOpenCircuit(t *testing.T) { + r := NewRegistry(100, 100) + if err := r.SetCircuitBreakerConfig(CircuitBreakerConfig{Threshold: 1, Timeout: time.Hour, SuccessThreshold: 1}); err != nil { + t.Fatal(err) + } + r.recordCircuitBreakerFailure() + if r.GetCircuitState() != CircuitOpen { + t.Fatalf("test setup: circuit should be Open, got %v", r.GetCircuitState()) + } + + err := r.ApplyConfig(&types.RemediationConfig{ + Enabled: true, + CircuitBreaker: types.CircuitBreakerConfig{ + Enabled: false, + Threshold: 1, + Timeout: time.Hour, + SuccessThreshold: 1, + }, + }, false) + if err != nil { + t.Fatalf("ApplyConfig: %v", err) + } + if r.GetCircuitState() != CircuitClosed { + t.Errorf("disabling the breaker must report Closed, got %v", r.GetCircuitState()) + } +} diff --git a/pkg/remediators/registry_test.go b/pkg/remediators/registry_test.go index 216a080..a1990a6 100644 --- a/pkg/remediators/registry_test.go +++ b/pkg/remediators/registry_test.go @@ -1849,3 +1849,46 @@ func TestRemediateWithStrategies(t *testing.T) { } }) } + +func runRemediationsWhileToggling(t *testing.T, registry *RemediatorRegistry, toggle func(i int)) { + t.Helper() + mock := newMockRemediator("test", false) + registry.Register(RemediatorInfo{ + Type: "test", + Factory: func() (types.Remediator, error) { return mock, nil }, + }) + problem := createTestProblem("test-type", "test-resource") + + stop := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; ; i++ { + select { + case <-stop: + return + default: + toggle(i) + } + } + }() + + for i := 0; i < 200; i++ { + mock.ClearCooldown(problem) + mock.ResetAttempts(problem) + _ = registry.Remediate(context.Background(), "test", problem) + } + close(stop) + wg.Wait() +} + +func TestSetDryRunConcurrentWithRemediate(t *testing.T) { + registry := NewRegistry(0, 100) + runRemediationsWhileToggling(t, registry, func(i int) { registry.SetDryRun(i%2 == 0) }) +} + +func TestSetMaxRemediationsPerHourConcurrentWithRemediate(t *testing.T) { + registry := NewRegistry(0, 100) + runRemediationsWhileToggling(t, registry, func(i int) { registry.SetMaxRemediationsPerHour((i % 2) * 1000) }) +} diff --git a/pkg/types/config.go b/pkg/types/config.go index 483d62e..46b97ef 100644 --- a/pkg/types/config.go +++ b/pkg/types/config.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "regexp" + "sort" "strings" "time" ) @@ -89,6 +90,21 @@ var ( MinCooldownPeriod = 10 * time.Second // Minimum cooldown between remediation attempts ) +// IsValidRemediationStrategy reports whether strategy names a known remediation strategy. +func IsValidRemediationStrategy(strategy string) bool { + return validRemediationStrategies[strategy] +} + +// ValidRemediationStrategies returns the known remediation strategy names, sorted. +func ValidRemediationStrategies() []string { + names := make([]string, 0, len(validRemediationStrategies)) + for name := range validRemediationStrategies { + names = append(names, name) + } + sort.Strings(names) + return names +} + // MonitorRegistryValidator provides an interface for validating monitor types // without creating an import cycle between config and monitors packages. // This interface is implemented by monitors.Registry. @@ -1032,21 +1048,19 @@ func (r *RemediationConfig) ApplyDefaults() error { return fmt.Errorf("invalid cooldownPeriod %q: %w", r.CooldownPeriodString, err) } - // Circuit breaker defaults - if r.CircuitBreaker.Enabled { - if r.CircuitBreaker.Threshold == 0 { - r.CircuitBreaker.Threshold = DefaultCircuitBreakerThreshold - } - if r.CircuitBreaker.TimeoutString == "" { - r.CircuitBreaker.TimeoutString = DefaultCircuitBreakerTimeout - } - r.CircuitBreaker.Timeout, err = time.ParseDuration(r.CircuitBreaker.TimeoutString) - if err != nil { - return fmt.Errorf("invalid circuit breaker timeout %q: %w", r.CircuitBreaker.TimeoutString, err) - } - if r.CircuitBreaker.SuccessThreshold == 0 { - r.CircuitBreaker.SuccessThreshold = 2 - } + // Circuit breaker defaults apply even when disabled so a later enable or reload has usable values + if r.CircuitBreaker.Threshold == 0 { + r.CircuitBreaker.Threshold = DefaultCircuitBreakerThreshold + } + if r.CircuitBreaker.TimeoutString == "" { + r.CircuitBreaker.TimeoutString = DefaultCircuitBreakerTimeout + } + r.CircuitBreaker.Timeout, err = time.ParseDuration(r.CircuitBreaker.TimeoutString) + if err != nil { + return fmt.Errorf("invalid circuit breaker timeout %q: %w", r.CircuitBreaker.TimeoutString, err) + } + if r.CircuitBreaker.SuccessThreshold == 0 { + r.CircuitBreaker.SuccessThreshold = 2 } // Apply defaults to overrides @@ -1301,8 +1315,8 @@ func (r *MonitorRemediationConfig) Validate() error { if r.Strategy == "" { return fmt.Errorf("strategy is required when remediation is enabled") } - if !validRemediationStrategies[r.Strategy] { - return fmt.Errorf("invalid strategy %q, must be one of: systemd-restart, custom-script, node-reboot, pod-delete, flush-dns, restart-interface, reset-routing, flush-ipv6-route", r.Strategy) + if !IsValidRemediationStrategy(r.Strategy) { + return fmt.Errorf("invalid strategy %q, must be one of: %s", r.Strategy, strings.Join(ValidRemediationStrategies(), ", ")) } // Strategy-specific validation diff --git a/pkg/types/config_test.go b/pkg/types/config_test.go index 7e2ebd8..eb73293 100644 --- a/pkg/types/config_test.go +++ b/pkg/types/config_test.go @@ -3220,3 +3220,38 @@ func TestPrometheusExporterConfigApplyDefaults(t *testing.T) { } }) } + +func TestRemediationStrategyHelpers(t *testing.T) { + for _, name := range ValidRemediationStrategies() { + if !IsValidRemediationStrategy(name) { + t.Errorf("%q listed as valid but rejected", name) + } + } + if IsValidRemediationStrategy("webhook") { + t.Error("webhook must not be a valid strategy") + } + + err := (&MonitorRemediationConfig{Enabled: true, Strategy: "webhook"}).Validate() + if err == nil { + t.Fatal("expected an error for an unknown strategy") + } + for _, name := range ValidRemediationStrategies() { + if !strings.Contains(err.Error(), name) { + t.Errorf("error %q should list %q", err.Error(), name) + } + } +} + +func TestRemediationConfigApplyDefaults_DisabledCircuitBreakerStillGetsDefaults(t *testing.T) { + r := RemediationConfig{CircuitBreaker: CircuitBreakerConfig{Enabled: false}} + if err := r.ApplyDefaults(); err != nil { + t.Fatal(err) + } + cb := r.CircuitBreaker + if cb.Enabled { + t.Error("ApplyDefaults must not flip Enabled") + } + if cb.Threshold != DefaultCircuitBreakerThreshold || cb.Timeout != 30*time.Minute || cb.SuccessThreshold != 2 { + t.Errorf("disabled circuit breaker must still get usable defaults, got %+v", cb) + } +} From 1d662b615beafd508c0d0e96fe873a2e87ab4615 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Wed, 7 Oct 2026 09:39:10 -0500 Subject: [PATCH 6/6] fix(remediation): default an omitted circuitBreaker block to enabled A plain bool could not tell an absent circuitBreaker block from an explicit enabled: false, so omitting the block disabled the breaker. CircuitBreakerConfig.Enabled is now a *bool like ReloadConfig.Enabled: nil becomes true in ApplyDefaults and only an explicit false disables. Co-Authored-By: Claude Fable 5.1 --- pkg/reload/validator_test.go | 4 ++- pkg/remediators/registry.go | 2 +- pkg/remediators/registry_applyconfig_test.go | 27 ++++++++++++++-- pkg/types/config.go | 21 +++++++++--- pkg/types/config_test.go | 34 +++++++++++++++----- 5 files changed, 71 insertions(+), 17 deletions(-) diff --git a/pkg/reload/validator_test.go b/pkg/reload/validator_test.go index d51df43..9900be9 100644 --- a/pkg/reload/validator_test.go +++ b/pkg/reload/validator_test.go @@ -817,7 +817,7 @@ func createValidConfig() *types.NodeDoctorConfig { MaxAttemptsGlobal: 3, HistorySize: 100, CircuitBreaker: types.CircuitBreakerConfig{ - Enabled: true, + Enabled: boolPtr(true), Threshold: 5, TimeoutString: "30m", Timeout: 30 * time.Minute, @@ -1890,3 +1890,5 @@ func TestValidate_MonitorRemediationSystemdRestartPasses(t *testing.T) { t.Fatalf("reload validation must accept systemd-restart, got errors: %v", result.Errors) } } + +func boolPtr(b bool) *bool { return &b } diff --git a/pkg/remediators/registry.go b/pkg/remediators/registry.go index dad5294..8c9fad5 100644 --- a/pkg/remediators/registry.go +++ b/pkg/remediators/registry.go @@ -485,7 +485,7 @@ func (r *RemediatorRegistry) ApplyConfig(cfg *types.RemediationConfig, dryRunMod // clobber the running configuration with invalid values. func (r *RemediatorRegistry) applyCircuitBreakerConfig(cfg types.CircuitBreakerConfig) error { cb := CircuitBreakerConfig{ - Disabled: !cfg.Enabled, + Disabled: !cfg.IsEnabled(), Threshold: cfg.Threshold, Timeout: cfg.Timeout, SuccessThreshold: cfg.SuccessThreshold, diff --git a/pkg/remediators/registry_applyconfig_test.go b/pkg/remediators/registry_applyconfig_test.go index 663cd05..2543ccd 100644 --- a/pkg/remediators/registry_applyconfig_test.go +++ b/pkg/remediators/registry_applyconfig_test.go @@ -138,7 +138,7 @@ func TestNewRegistryFromConfigAppliesCircuitBreakerAtStartup(t *testing.T) { r, err := NewRegistryFromConfig(&types.RemediationConfig{ Enabled: true, CircuitBreaker: types.CircuitBreakerConfig{ - Enabled: true, + Enabled: boolPtr(true), Threshold: 5, Timeout: 7 * time.Minute, SuccessThreshold: 2, @@ -188,7 +188,7 @@ func TestCircuitBreakerDisabledNeverBlocks(t *testing.T) { r, err := NewRegistryFromConfig(&types.RemediationConfig{ Enabled: true, CircuitBreaker: types.CircuitBreakerConfig{ - Enabled: false, + Enabled: boolPtr(false), Threshold: 1, Timeout: time.Hour, SuccessThreshold: 1, @@ -237,7 +237,7 @@ func TestApplyConfigDisablingCircuitBreakerClosesOpenCircuit(t *testing.T) { err := r.ApplyConfig(&types.RemediationConfig{ Enabled: true, CircuitBreaker: types.CircuitBreakerConfig{ - Enabled: false, + Enabled: boolPtr(false), Threshold: 1, Timeout: time.Hour, SuccessThreshold: 1, @@ -250,3 +250,24 @@ func TestApplyConfigDisablingCircuitBreakerClosesOpenCircuit(t *testing.T) { t.Errorf("disabling the breaker must report Closed, got %v", r.GetCircuitState()) } } + +func boolPtr(b bool) *bool { return &b } + +func TestNewRegistryFromConfigAbsentCircuitBreakerBlockEnablesBreaker(t *testing.T) { + cfg := types.RemediationConfig{Enabled: true} + if err := cfg.ApplyDefaults(); err != nil { + t.Fatal(err) + } + + r, err := NewRegistryFromConfig(&cfg, false) + if err != nil { + t.Fatalf("NewRegistryFromConfig: %v", err) + } + got := r.GetCircuitBreakerConfig() + if got.Disabled { + t.Fatal("omitting the circuitBreaker block must leave the breaker enabled") + } + if got.Threshold != types.DefaultCircuitBreakerThreshold || got.Timeout != 30*time.Minute || got.SuccessThreshold != 2 { + t.Errorf("breaker must run with the config defaults, got %+v", got) + } +} diff --git a/pkg/types/config.go b/pkg/types/config.go index 46b97ef..62db5e1 100644 --- a/pkg/types/config.go +++ b/pkg/types/config.go @@ -504,13 +504,22 @@ type RemediationCoordinationConfig struct { // CircuitBreakerConfig configures circuit breaker behavior. type CircuitBreakerConfig struct { - Enabled bool `json:"enabled" yaml:"enabled"` + // Enabled is a *bool so an absent key defaults to true while an explicit false disables + Enabled *bool `json:"enabled,omitempty" yaml:"enabled,omitempty"` Threshold int `json:"threshold,omitempty" yaml:"threshold,omitempty"` TimeoutString string `json:"timeout,omitempty" yaml:"timeout,omitempty"` Timeout time.Duration `json:"-" yaml:"-"` SuccessThreshold int `json:"successThreshold,omitempty" yaml:"successThreshold,omitempty"` } +// IsEnabled reports whether the circuit breaker is on; unset means enabled. +func (c *CircuitBreakerConfig) IsEnabled() bool { + if c == nil || c.Enabled == nil { + return true + } + return *c.Enabled +} + // RemediationOverride allows problem-specific remediation overrides. type RemediationOverride struct { Problem string `json:"problem" yaml:"problem"` @@ -1048,7 +1057,11 @@ func (r *RemediationConfig) ApplyDefaults() error { return fmt.Errorf("invalid cooldownPeriod %q: %w", r.CooldownPeriodString, err) } - // Circuit breaker defaults apply even when disabled so a later enable or reload has usable values + if r.CircuitBreaker.Enabled == nil { + enabled := true + r.CircuitBreaker.Enabled = &enabled + } + // Threshold defaults apply even when disabled so a later enable or reload has usable values if r.CircuitBreaker.Threshold == 0 { r.CircuitBreaker.Threshold = DefaultCircuitBreakerThreshold } @@ -1649,8 +1662,8 @@ func (r *RemediationConfig) Validate() error { return fmt.Errorf("historySize must be positive, got %d", r.HistorySize) } - // Validate circuit breaker - if r.CircuitBreaker.Enabled { + // Validate circuit breaker only when explicitly enabled; a nil Enabled means defaults have not run yet + if r.CircuitBreaker.Enabled != nil && *r.CircuitBreaker.Enabled { if r.CircuitBreaker.Threshold <= 0 { return fmt.Errorf("circuitBreaker.threshold must be positive, got %d", r.CircuitBreaker.Threshold) } diff --git a/pkg/types/config_test.go b/pkg/types/config_test.go index eb73293..bb427ec 100644 --- a/pkg/types/config_test.go +++ b/pkg/types/config_test.go @@ -209,7 +209,7 @@ func TestRemediationConfigApplyDefaults(t *testing.T) { name: "circuit breaker enabled gets defaults", input: RemediationConfig{ CircuitBreaker: CircuitBreakerConfig{ - Enabled: true, + Enabled: boolPtr(true), }, }, expected: RemediationConfig{ @@ -220,7 +220,7 @@ func TestRemediationConfigApplyDefaults(t *testing.T) { MaxAttemptsGlobal: DefaultMaxAttemptsGlobal, HistorySize: DefaultHistorySize, CircuitBreaker: CircuitBreakerConfig{ - Enabled: true, + Enabled: boolPtr(true), Threshold: DefaultCircuitBreakerThreshold, TimeoutString: DefaultCircuitBreakerTimeout, Timeout: 30 * time.Minute, @@ -240,7 +240,7 @@ func TestRemediationConfigApplyDefaults(t *testing.T) { name: "invalid circuit breaker timeout", input: RemediationConfig{ CircuitBreaker: CircuitBreakerConfig{ - Enabled: true, + Enabled: boolPtr(true), TimeoutString: "invalid", }, }, @@ -1476,7 +1476,7 @@ func TestRemediationConfigValidation(t *testing.T) { MaxAttemptsGlobal: 3, HistorySize: 100, CircuitBreaker: CircuitBreakerConfig{ - Enabled: true, + Enabled: boolPtr(true), Threshold: 0, Timeout: 30 * time.Minute, }, @@ -1863,7 +1863,7 @@ func TestComplexConfigurationValidation(t *testing.T) { MaxAttemptsGlobal: 5, HistorySize: 200, CircuitBreaker: CircuitBreakerConfig{ - Enabled: true, + Enabled: boolPtr(true), Threshold: 10, Timeout: 45 * time.Minute, SuccessThreshold: 3, @@ -3242,14 +3242,32 @@ func TestRemediationStrategyHelpers(t *testing.T) { } } +func boolPtr(b bool) *bool { return &b } + +func TestRemediationConfigApplyDefaults_AbsentCircuitBreakerIsEnabled(t *testing.T) { + r := RemediationConfig{} + if err := r.ApplyDefaults(); err != nil { + t.Fatal(err) + } + if r.CircuitBreaker.Enabled == nil || !*r.CircuitBreaker.Enabled { + t.Error("an absent circuitBreaker block must default to enabled") + } + if !r.CircuitBreaker.IsEnabled() { + t.Error("IsEnabled must report true after defaults") + } + if !(&CircuitBreakerConfig{}).IsEnabled() { + t.Error("IsEnabled on a zero-value block must be true") + } +} + func TestRemediationConfigApplyDefaults_DisabledCircuitBreakerStillGetsDefaults(t *testing.T) { - r := RemediationConfig{CircuitBreaker: CircuitBreakerConfig{Enabled: false}} + r := RemediationConfig{CircuitBreaker: CircuitBreakerConfig{Enabled: boolPtr(false)}} if err := r.ApplyDefaults(); err != nil { t.Fatal(err) } cb := r.CircuitBreaker - if cb.Enabled { - t.Error("ApplyDefaults must not flip Enabled") + if cb.IsEnabled() { + t.Error("ApplyDefaults must not flip an explicit false") } if cb.Threshold != DefaultCircuitBreakerThreshold || cb.Timeout != 30*time.Minute || cb.SuccessThreshold != 2 { t.Errorf("disabled circuit breaker must still get usable defaults, got %+v", cb)