diff --git a/.githooks/README.md b/.githooks/README.md deleted file mode 100755 index a732dff..0000000 --- a/.githooks/README.md +++ /dev/null @@ -1,220 +0,0 @@ -# Git Hooks for Nexmonyx - -This directory contains Git hooks that help maintain code quality by running validations before certain Git operations. - -## Available Hooks - -### pre-push - -Runs comprehensive validation before pushing to remote: -- Code formatting (gofmt) -- Static analysis (go vet) -- Advanced linting (staticcheck) -- Security scanning (gosec) -- Unit tests with race detection - -This prevents CI/CD failures by catching issues locally before they reach the pipeline. - -## Setup - -### Quick Setup (Recommended) - -Run the setup script: - -```bash -./.githooks/setup.sh -``` - -This will: -1. Configure Git to use `.githooks` directory -2. Enable all available hooks -3. Show current configuration - -### Manual Setup - -Configure Git to use this hooks directory: - -```bash -git config core.hooksPath .githooks -``` - -Verify configuration: - -```bash -git config --get core.hooksPath -# Should output: .githooks -``` - -## Usage - -Once enabled, hooks run automatically: - -```bash -# Pre-push hook runs automatically -git push origin main -``` - -### Bypassing Hooks - -**Warning:** Only bypass hooks when absolutely necessary (e.g., emergency hotfix). - -```bash -# Bypass pre-push validation -git push --no-verify - -# Or bypass all hooks for one commit -git commit --no-verify -m "Emergency fix" -``` - -## Disabling Hooks - -Temporarily disable: - -```bash -git config --unset core.hooksPath -``` - -Re-enable: - -```bash -git config core.hooksPath .githooks -``` - -## Hook Details - -### pre-push Hook - -**What it validates:** -- All 6 components: api, org-management-controller, probe-controller, ingestor, customer-probe-agent, pkg -- 5 checks per component: format, vet, staticcheck, gosec, tests - -**When it runs:** -- Before every `git push` -- After `git push` command, before sending to remote - -**What happens on failure:** -- Push is blocked -- Detailed error output shown -- Instructions provided to fix or bypass - -**Performance:** -- Quick mode available: Only runs format, vet, staticcheck (skip tests) -- Full validation: ~2-5 minutes depending on changes -- Parallel execution: All components validated simultaneously - -## Customization - -### Run Quick Validation Only - -Edit `.githooks/pre-push` and change: - -```bash -./scripts/validate-pipeline-local.sh -``` - -To: - -```bash -./scripts/validate-pipeline-local.sh --quick -``` - -### Validate Specific Components - -Edit `.githooks/pre-push` to add component filtering: - -```bash -# Only validate changed components -CHANGED_COMPONENTS=$(git diff --name-only HEAD @{u} | grep -E '^(api|pkg)/' | cut -d'/' -f1 | sort -u) -for component in $CHANGED_COMPONENTS; do - ./scripts/validate-pipeline-local.sh --component=$component -done -``` - -## Troubleshooting - -### Hook Not Running - -Check configuration: - -```bash -git config --get core.hooksPath -``` - -Verify hook is executable: - -```bash -ls -la .githooks/pre-push -# Should show: -rwxr-xr-x -``` - -Make executable if needed: - -```bash -chmod +x .githooks/pre-push -``` - -### Hook Failing Unexpectedly - -Test hook manually: - -```bash -./.githooks/pre-push -``` - -Run validation script directly: - -```bash -./scripts/validate-pipeline-local.sh --quick -``` - -### Performance Issues - -Use quick mode or component-specific validation: - -```bash -# Quick validation (no tests) -./scripts/validate-pipeline-local.sh --quick - -# Specific component -./scripts/validate-pipeline-local.sh --component=api -``` - -## Best Practices - -1. **Enable hooks for all developers** - - Prevents CI/CD failures - - Faster feedback loop - - Consistent code quality - -2. **Run quick validation frequently** - ```bash - make validate-quick - ``` - -3. **Run full validation before push** - ```bash - make validate-pre-push - ``` - -4. **Only bypass when necessary** - - Emergency hotfixes - - Infrastructure-only changes - - Non-code changes - -5. **Keep hooks updated** - - Pull latest `.githooks` directory - - Re-run setup if hooks change - -## Related Documentation - -- [Local CI/CD Validation Guide](../docs/development/local-cicd-validation-guide.md) -- [Validation Quick Reference](../docs/development/validation-quick-reference.md) *(coming soon)* -- [Task Execution Workflow](../docs/development/task-execution-workflow.md) - -## Support - -For issues or questions: -- Check troubleshooting section above -- Review validation guide documentation -- Test validation script manually -- Check CI/CD pipeline for comparison diff --git a/.githooks/pre-push b/.githooks/pre-push deleted file mode 100755 index 69ffaeb..0000000 --- a/.githooks/pre-push +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -# Pre-push hook that validates all components before pushing -# This prevents CI/CD failures by catching issues locally - -# TEMPORARILY DISABLED - blocking large data transfers -exit 0 - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -echo "" -echo -e "${BLUE}🔍 Running pre-push validation...${NC}" -echo "" - -# Run full pipeline validation -./scripts/validate-pipeline-local.sh - -RESULT=$? - -echo "" - -if [ $RESULT -ne 0 ]; then - echo -e "${RED}╔════════════════════════════════════════════════════════════════╗${NC}" - echo -e "${RED}║ ❌ Pre-push validation FAILED ║${NC}" - echo -e "${RED}╚════════════════════════════════════════════════════════════════╝${NC}" - echo "" - echo -e "${YELLOW}Fix the issues above before pushing.${NC}" - echo "" - echo -e "${YELLOW}To bypass this check (NOT RECOMMENDED):${NC}" - echo -e "${YELLOW} git push --no-verify${NC}" - echo "" - exit 1 -fi - -echo -e "${GREEN}╔════════════════════════════════════════════════════════════════╗${NC}" -echo -e "${GREEN}║ ✅ Pre-push validation passed - proceeding with push ║${NC}" -echo -e "${GREEN}╚════════════════════════════════════════════════════════════════╝${NC}" -echo "" - -exit 0 diff --git a/.githooks/setup.sh b/.githooks/setup.sh deleted file mode 100755 index 10ecfc9..0000000 --- a/.githooks/setup.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -# Setup script to enable git hooks for validation - -# Colors for output -GREEN='\033[0;32m' -BLUE='\033[0;34m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -echo "" -echo -e "${BLUE}╔════════════════════════════════════════════════════════════════╗${NC}" -echo -e "${BLUE}║ Git Hooks Setup for Nexmonyx ║${NC}" -echo -e "${BLUE}╚════════════════════════════════════════════════════════════════╝${NC}" -echo "" - -# Check if already configured -CURRENT_HOOKS_PATH=$(git config --get core.hooksPath) - -if [ "$CURRENT_HOOKS_PATH" = ".githooks" ]; then - echo -e "${GREEN}✓ Git hooks are already configured!${NC}" - echo "" - echo "Current configuration:" - echo " core.hooksPath = .githooks" - echo "" - exit 0 -fi - -echo -e "${BLUE}This will configure Git to use local validation hooks:${NC}" -echo "" -echo " â€ĸ Pre-push hook: Validates all components before pushing" -echo " â€ĸ Prevents CI/CD failures by catching issues locally" -echo " â€ĸ Can be bypassed with: git push --no-verify" -echo "" - -read -p "Enable git hooks? (y/n): " -n 1 -r -echo "" - -if [[ ! $REPLY =~ ^[Yy]$ ]]; then - echo -e "${YELLOW}Setup cancelled.${NC}" - echo "" - exit 0 -fi - -# Configure git to use .githooks directory -git config core.hooksPath .githooks - -echo "" -echo -e "${GREEN}╔════════════════════════════════════════════════════════════════╗${NC}" -echo -e "${GREEN}║ ✓ Git hooks configured successfully! ║${NC}" -echo -e "${GREEN}╚════════════════════════════════════════════════════════════════╝${NC}" -echo "" -echo "Configuration:" -echo " core.hooksPath = .githooks" -echo "" -echo "Active hooks:" -echo " â€ĸ pre-push: Full validation before push" -echo "" -echo -e "${BLUE}To disable hooks:${NC}" -echo " git config --unset core.hooksPath" -echo "" -echo -e "${BLUE}To bypass on specific push:${NC}" -echo " git push --no-verify" -echo "" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08aafe9..dc58c08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,29 +107,9 @@ jobs: go tool cover -func=coverage-integration.out | grep total | awk '{print "Integration test coverage: " $3}' fi + # Threshold lives in the Makefile (COVERAGE_THRESHOLD) so local and CI gates agree. - name: Check coverage threshold - run: | - if [ ! -f coverage.out ]; then - echo "::error::Coverage file not found" - exit 1 - fi - - COVERAGE=$(go tool cover -func=coverage.out | grep total | awk '{print $3}' | sed 's/%//') - - if [ -z "$COVERAGE" ]; then - echo "::error::Failed to extract coverage percentage" - exit 1 - fi - - THRESHOLD=70 - echo "Current coverage: ${COVERAGE}%" - echo "Minimum threshold: ${THRESHOLD}%" - - if (( $(echo "$COVERAGE < $THRESHOLD" | bc -l) )); then - echo "::error::Coverage ${COVERAGE}% is below minimum threshold of ${THRESHOLD}%" - exit 1 - fi - echo "Coverage check passed!" + run: make coverage-threshold COVERAGE_PROFILE=coverage.out - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 @@ -219,6 +199,12 @@ jobs: cd cmd/overlay-test-server go build -o ../../bin/overlay-test-server + - name: Build node-doctor-controller binary + run: | + mkdir -p bin + cd cmd/node-doctor-controller + go build -o ../../bin/node-doctor-controller + - name: Test binary run: ./bin/node-doctor --version @@ -293,6 +279,21 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max + - name: Build controller Docker image (no push) + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile.controller + platforms: linux/amd64 + push: false + tags: ${{ env.REGISTRY }}/node-doctor-controller:ci-${{ github.sha }} + build-args: | + VERSION=${{ steps.meta.outputs.VERSION }} + GIT_COMMIT=${{ steps.meta.outputs.GIT_COMMIT }} + BUILD_TIME=${{ steps.meta.outputs.BUILD_TIME }} + cache-from: type=gha,scope=controller + cache-to: type=gha,mode=max,scope=controller + # Summary job - provides overall status # # New gating jobs are wired in here rather than added as new required status checks: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7692003..d79e59f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -171,11 +171,72 @@ jobs: cache-from: type=gha,scope=overlay-test cache-to: type=gha,mode=max,scope=overlay-test + # Build, push and sign the controller Docker image (multi-arch) + docker-controller: + name: Build & Push Controller Image + runs-on: self-hosted-linux + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/supporttools/node-doctor-controller + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=latest,enable=${{ !contains(github.ref_name, 'rc') && !contains(github.ref_name, 'beta') && !contains(github.ref_name, 'alpha') }} + + - name: Build and push controller image + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile.controller + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + VERSION=${{ github.ref_name }} + GIT_COMMIT=${{ github.sha }} + BUILD_TIME=${{ github.event.head_commit.timestamp }} + cache-from: type=gha,scope=controller + cache-to: type=gha,mode=max,scope=controller + + - name: Verify pre-baked cosign + run: | + set -euo pipefail + command -v cosign >/dev/null 2>&1 || { echo "::error::cosign is missing from this self-hosted runner. It must be pre-baked into the runner image - there is deliberately no download fallback. Fix the runner image."; exit 1; } + cosign version + + - name: Sign controller image + env: + COSIGN_EXPERIMENTAL: 1 + IMAGE_REF: ${{ env.REGISTRY }}/supporttools/node-doctor-controller:${{ steps.meta.outputs.version }} + run: | + echo "Signing controller image with cosign (keyless)..." + echo "Tag: ${IMAGE_REF}" + cosign sign --yes "${IMAGE_REF}" + # Create GitHub Release github-release: name: Create GitHub Release runs-on: self-hosted-linux - needs: [docker, docker-overlay-test] + needs: [docker, docker-overlay-test, docker-controller] steps: - name: Checkout code uses: actions/checkout@v5 @@ -242,13 +303,13 @@ jobs: # Helm Chart Publishing # - # Depends on docker-overlay-test as well as docker: the chart's default values reference - # BOTH images, and the post-package guard below asserts those references actually resolve - # in the registry. That assertion is only meaningful once both images have been pushed. + # Depends on every image job: the chart's default values reference all three images, and + # the post-package guard below asserts those references actually resolve in the registry. + # That assertion is only meaningful once all of them have been pushed. helm-publish: name: Publish Helm Chart runs-on: self-hosted-linux - needs: [docker, docker-overlay-test] + needs: [docker, docker-overlay-test, docker-controller] outputs: chart-version: ${{ steps.get-chart-version.outputs.version }} @@ -289,7 +350,7 @@ jobs: # docker/metadata-action in the `docker` job publishes # `type=semver,pattern={{version}}`, which strips the leading v: the image lands on # Docker Hub as `1.8.7`, never `v1.8.7`. values.yaml.template feeds ${IMAGE_TAG} into - # BOTH image.tag and overlayTest.image.tag, so when IMAGE_TAG carried the v, every + # image.tag, overlayTest.image.tag and controller.image.tag, so when IMAGE_TAG carried the v, every # published chart defaulted to an image tag that was never pushed — # `helm install supporttools/node-doctor` with default values gave ImagePullBackOff # on both DaemonSets. node-doctor-overlay-test has never had a v-prefixed tag for any @@ -354,7 +415,7 @@ jobs: with open('/tmp/chart-verify/node-doctor/values.yaml') as fh: v = yaml.safe_load(fh) refs = [] - for path in (('image',), ('overlayTest', 'image')): + for path in (('image',), ('overlayTest', 'image'), ('controller', 'image')): node = v for key in path: node = (node or {}).get(key) @@ -509,7 +570,7 @@ jobs: release-complete: name: Release Complete runs-on: self-hosted-linux - needs: [docker, docker-overlay-test, github-release, helm-publish] + needs: [docker, docker-overlay-test, docker-controller, github-release, helm-publish] if: always() steps: - name: Check release status @@ -525,6 +586,10 @@ jobs: echo "❌ Overlay test Docker build job failed" exit 1 fi + if [[ "${{ needs.docker-controller.result }}" != "success" ]]; then + echo "❌ Controller Docker build job failed" + exit 1 + fi if [[ "${{ needs.github-release.result }}" != "success" ]]; then echo "❌ GitHub release job failed" exit 1 @@ -556,7 +621,7 @@ jobs: notify-failure: name: Notify Release Failure runs-on: self-hosted-linux - needs: [docker, docker-overlay-test, github-release, helm-publish, grype-scan, release-complete] + needs: [docker, docker-overlay-test, docker-controller, github-release, helm-publish, grype-scan, release-complete] if: failure() steps: # A tag name may legally contain characters that would break out of the JSON @@ -580,6 +645,7 @@ jobs: echo "| --- | --- |" echo "| Docker images | ${{ needs.docker.result }} |" echo "| Overlay test image | ${{ needs.docker-overlay-test.result }} |" + echo "| Controller image | ${{ needs.docker-controller.result }} |" echo "| GitHub release | ${{ needs.github-release.result }} |" echo "| Helm chart publish | ${{ needs.helm-publish.result }} |" echo "| Grype scan | ${{ needs.grype-scan.result }} |" @@ -600,7 +666,7 @@ jobs: "type": "section", "text": { "type": "mrkdwn", - "text": "❌ *node-doctor* release *${{ steps.ctx.outputs.ref }}* *FAILED*\n\n*Docker images:* `${{ needs.docker.result }}`\n*Overlay test image:* `${{ needs.docker-overlay-test.result }}`\n*GitHub release:* `${{ needs.github-release.result }}`\n*Helm chart publish:* `${{ needs.helm-publish.result }}`\n\nA failed chart publish means charts.support.tools still serves the previous version.\n\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Workflow Run>" + "text": "❌ *node-doctor* release *${{ steps.ctx.outputs.ref }}* *FAILED*\n\n*Docker images:* `${{ needs.docker.result }}`\n*Overlay test image:* `${{ needs.docker-overlay-test.result }}`\n*Controller image:* `${{ needs.docker-controller.result }}`\n*GitHub release:* `${{ needs.github-release.result }}`\n*Helm chart publish:* `${{ needs.helm-publish.result }}`\n\nA failed chart publish means charts.support.tools still serves the previous version.\n\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Workflow Run>" } } ] diff --git a/.goreleaser.yml b/.goreleaser.yml index 282ff70..7b676b6 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -30,8 +30,6 @@ builds: flags: - -trimpath mod_timestamp: '{{.CommitTimestamp}}' - # Note: Darwin builds temporarily disabled due to cross-compilation type issues - # TODO: Fix pkg/monitors/system/disk.go:58 uint32->int64 conversion for darwin # Archive configuration archives: diff --git a/.validation.json b/.validation.json deleted file mode 100644 index e7cfc1e..0000000 --- a/.validation.json +++ /dev/null @@ -1,149 +0,0 @@ -{ - "_comment": "Validation configuration for node-doctor local CI/CD validation", - "_info": "Adapted from Nexmonyx repo-template for single-binary Kubernetes DaemonSet project", - "validation": { - "enabled": true, - "parallel_jobs": 2, - "fail_fast": false, - "quick_mode": { - "enabled": true, - "skip_tests": true, - "skip_gosec": true, - "skip_coverage": true - } - }, - "components": { - "_comment": "node-doctor is a single binary Kubernetes monitoring tool", - "node-doctor": { - "enabled": true, - "path": "cmd/node-doctor", - "test_timeout": "10m", - "coverage_threshold": 50, - "coverage_exclude": [ - "cmd/node-doctor/main.go" - ] - } - }, - "tools": { - "gofmt": { - "enabled": true, - "check_only": true, - "fix": false - }, - "goimports": { - "enabled": false, - "local": "github.com/supporttools/node-doctor" - }, - "go_vet": { - "enabled": true, - "flags": [ - "-all" - ] - }, - "staticcheck": { - "enabled": true, - "checks": [ - "all", - "-ST1000", - "-ST1003" - ], - "fail_on": [ - "error", - "warning" - ] - }, - "gosec": { - "enabled": true, - "severity": "medium", - "confidence": "medium", - "exclude_rules": [ - "G104" - ], - "scan_tests": false - }, - "golangci_lint": { - "enabled": false, - "config": ".golangci.yml", - "timeout": "5m" - }, - "go_test": { - "enabled": true, - "flags": [ - "-v", - "-race", - "-cover" - ], - "timeout": "10m", - "coverage_mode": "atomic", - "coverage_profile": "coverage.txt" - } - }, - "repository": { - "go_version": "1.21", - "go_mod_tidy_check": true, - "cleanliness_check": true, - "allowed_root_files": [ - ".gitignore", - ".gitattributes", - ".validation.json", - "README.md", - "LICENSE", - "CLAUDE.md", - "CONTRIBUTING.md", - "PROJECT_SETUP_CHECKLIST.md", - "makefile", - "Makefile", - "Dockerfile", - "go.mod", - "go.sum" - ], - "forbidden_patterns": [ - "*.swp", - "*.swo", - "*~", - ".DS_Store", - "*.key", - "*.pem", - "credentials.json", - "secrets.yaml", - "kubeconfig" - ] - }, - "ci_cd": { - "mirror_github_actions": true, - "cache_enabled": true, - "cache_paths": [ - "~/.cache/go-build", - "~/go/pkg/mod" - ] - }, - "reporting": { - "verbose": true, - "show_passed": true, - "show_skipped": true, - "color_output": true, - "summary": true - }, - "pre_commit": { - "enabled": true, - "auto_fix": false, - "checks": [ - "gofmt", - "go_vet" - ] - }, - "pre_push": { - "enabled": true, - "run_full_validation": true, - "allow_skip": false, - "checks": [ - "go_version", - "cleanliness", - "gofmt", - "go_vet", - "staticcheck", - "gosec", - "tests" - ] - } -} diff --git a/.version-rc b/.version-rc deleted file mode 100644 index 1df767c..0000000 --- a/.version-rc +++ /dev/null @@ -1 +0,0 @@ -v1.5.5-rc3 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c8d713b..137846e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -337,7 +337,7 @@ func (r *ServiceRemediator) GetCooldown() time.Duration { ### Test Coverage Expectations -- **Minimum Coverage**: 80% overall code coverage +- **Minimum Coverage**: 70% unit test coverage (`make coverage-check`; CI enforces the same threshold) - **Target Coverage**: 85%+ for production code - **New Code**: Must include tests for all new functionality @@ -677,8 +677,7 @@ When contributing to Node Doctor: - **[Architecture Guide](docs/architecture.md)** - System design and component overview - **[Monitors Guide](docs/monitors.md)** - Monitor implementation details - **[Remediation Guide](docs/remediation.md)** - Remediator patterns and safety mechanisms -- **[Task Execution Workflow](docs/development/task-execution-workflow.md)** - Development process -- **[Validation Quick Reference](docs/development/validation-quick-reference.md)** - Testing and validation +- **[Testing Guide](docs/testing-guide.md)** - Testing and validation ### Build Targets diff --git a/Dockerfile.controller b/Dockerfile.controller new file mode 100644 index 0000000..202c74f --- /dev/null +++ b/Dockerfile.controller @@ -0,0 +1,46 @@ +# Dockerfile for node-doctor-controller - cluster-wide coordination API for node-doctor agents +# Pure-Go SQLite (modernc.org/sqlite), so the binary is fully static and runs from scratch. + +FROM golang:1.25-alpine AS builder + +RUN apk add --no-cache git + +WORKDIR /build + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +ARG VERSION=dev +ARG GIT_COMMIT=unknown +ARG BUILD_TIME=unknown +ARG TARGETARCH + +RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build \ + -ldflags="-X main.Version=${VERSION} -X main.GitCommit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME} -w -s" \ + -trimpath \ + -o node-doctor-controller \ + ./cmd/node-doctor-controller + +# SQLite needs a writable data dir; in-cluster this is replaced by the PVC mount. +RUN mkdir -p /data && chown 65534:65534 /data + +FROM scratch + +COPY --from=builder /build/node-doctor-controller /usr/local/bin/node-doctor-controller +COPY --from=builder --chown=65534:65534 /data /data + +USER 65534:65534 + +# 8080: HTTP API, health and metrics endpoint +EXPOSE 8080 + +ENTRYPOINT ["/usr/local/bin/node-doctor-controller"] + +LABEL org.opencontainers.image.title="Node Doctor Controller" \ + org.opencontainers.image.description="Cluster-wide coordination controller for node-doctor agents" \ + org.opencontainers.image.vendor="SupportTools" \ + org.opencontainers.image.url="https://github.com/supporttools/node-doctor" \ + org.opencontainers.image.source="https://github.com/supporttools/node-doctor" \ + org.opencontainers.image.licenses="Apache-2.0" diff --git a/Makefile b/Makefile index b3f2a8e..f38e132 100644 --- a/Makefile +++ b/Makefile @@ -1,32 +1,17 @@ -# ================================================================================================ -# Nexmonyx Repository Template - Makefile -# ================================================================================================ -# -# This makefile provides a complete build, test, and deployment framework for Go/Kubernetes projects. -# -# CUSTOMIZATION REQUIRED: -# 1. Replace {{PROJECT_NAME}} with your project name -# 2. Replace {{REGISTRY}} with your container registry (e.g., ghcr.io/myorg, harbor.mycompany.com/myproject) -# 3. Replace {{HELM_REPO}} with your Helm chart repository URL -# 4. Update COMPONENTS list with your actual components -# 5. Customize KUBECONFIG paths for your environments -# 6. Adjust build targets for your project structure -# -# ================================================================================================ +# node-doctor build, test, and local validation targets. +# Images and charts are published only by .github/workflows/release.yml. -.PHONY: help all test build deploy \ - build-local test-local validate-local validate-pipeline-local validate-quick \ - deploy-dev deploy-stg deploy-prd \ - bump bump-with-monitoring \ - qa-check devils-advocate workflow-status \ +.PHONY: help all test build \ + build-local test-local validate-local validate-pipeline-local validate-quick validate-component \ + workflow-status workflow-help gh-help \ gh-status gh-watch gh-logs gh-builds \ - check-prerequisites check-docker check-kubectl \ + check-prerequisites check-docker check-kubectl check-go-version require-version \ build test test-integration test-e2e test-all test-ci \ test-net-icmp-integration \ lint fmt clean install-deps \ docker-build docker-push \ - helm-lint helm-package helm-publish helm-generate helm-verify-generated \ - coverage-check + helm-lint helm-package helm-generate helm-verify-generated \ + coverage-check coverage-threshold # ================================================================================================ # Project Configuration @@ -44,38 +29,11 @@ GIT_COMMIT := $(shell git rev-parse HEAD) BUILD_TIME := $(shell date -u +"%Y-%m-%dT%H:%M:%SZ") GO_VERSION := 1.25 -# RC Version - for release candidate builds -RC_VERSION_FILE := .version-rc -RC_VERSION := $(shell [ -f $(RC_VERSION_FILE) ] && cat $(RC_VERSION_FILE) || echo "v0.1.0-rc.1") - export VERSION export GIT_COMMIT export BUILD_TIME export GO_VERSION -# ================================================================================================ -# Environment Configuration -# ================================================================================================ - -# CUSTOMIZE: Update these paths for your kubeconfig locations -KUBECONFIG_DEV := $(HOME)/.kube/dev-cluster -KUBECONFIG_STG := $(HOME)/.kube/staging-cluster -KUBECONFIG_PRD := $(HOME)/.kube/config - -# Kubernetes context for production deployment -KUBECTL_CONTEXT_PRD := a1-ops-prd - -export KUBECONFIG_DEV -export KUBECONFIG_STG -export KUBECONFIG_PRD - -# Kubernetes namespaces (node-doctor runs in kube-system typically) -NAMESPACE_DEV := kube-system -NAMESPACE_STG := kube-system -NAMESPACE_PRD := node-doctor - -# Helm Chart settings -HELM_REPO_URL := https://charts.support.tools CHART_VERSION := v$(shell git rev-list --count HEAD) # ================================================================================================ @@ -83,11 +41,12 @@ CHART_VERSION := v$(shell git rev-list --count HEAD) # ================================================================================================ # Node Doctor binaries -COMPONENTS := node-doctor overlay-test-server +COMPONENTS := node-doctor overlay-test-server node-doctor-controller # Docker images DOCKER_IMAGE_node-doctor := $(REGISTRY)/$(PROJECT_NAME) DOCKER_IMAGE_overlay-test-server := $(REGISTRY)/$(PROJECT_NAME)-overlay-test +DOCKER_IMAGE_node-doctor-controller := $(REGISTRY)/$(PROJECT_NAME)-controller # ================================================================================================ # Color Output Functions @@ -143,16 +102,11 @@ check-go-version: @./scripts/validate-go-version.sh @$(call print_success,"Go version check passed") -check-kubeconfig-%: - @$(call print_status,"Checking kubeconfig for $*...") - @ENV_UPPER=$$(echo $* | tr '[:lower:]' '[:upper:]') && \ - KUBECONFIG_VAR=KUBECONFIG_$$ENV_UPPER && \ - eval KUBECONFIG_PATH=\$$$$KUBECONFIG_VAR && \ - if [ ! -f "$$KUBECONFIG_PATH" ]; then \ - $(call print_error,"Kubeconfig not found for $*: $$KUBECONFIG_PATH"); \ - exit 1; \ - fi - @$(call print_success,"Kubeconfig validated for $*") +# Pushing images needs a real tag; the epoch default would litter the registry. +require-version: +ifeq ($(origin VERSION),file) + @$(call print_error,"Set VERSION explicitly: make docker-push VERSION=1.2.3"); exit 1 +endif # ================================================================================================ # Build Targets @@ -181,6 +135,12 @@ build-overlay-test-server-local: @cd cmd/overlay-test-server && go build -o ../../bin/overlay-test-server @$(call print_success,"overlay-test-server built: bin/overlay-test-server") +build-node-doctor-controller-local: + @$(call print_status,"Building node-doctor-controller locally...") + @mkdir -p bin + @cd cmd/node-doctor-controller && go build -ldflags="-X main.Version=$(VERSION) -X main.GitCommit=$(GIT_COMMIT) -X main.BuildTime=$(BUILD_TIME)" -o ../../bin/node-doctor-controller + @$(call print_success,"node-doctor-controller built: bin/node-doctor-controller") + build-local: build-all-local # ================================================================================================ @@ -239,14 +199,12 @@ test-ci: fi @$(call print_success,"CI test gate passed") -# End-to-end tests +# E2E tests create a kind cluster; they need docker, kind, and kubectl. +# Env: E2E_KEEP_CLUSTER=1, E2E_EXPORT_LOGS=1, E2E_DEBUG=1 (see test/e2e/README.md). test-e2e: - @$(call print_status,"Running E2E tests...") - @if [ -d "test/e2e" ]; then \ - go test ./test/e2e/... -v -timeout 10m; \ - else \ - $(call print_warning,"E2E tests not yet implemented (test/e2e/ does not exist)"); \ - fi + @$(call print_status,"Running E2E tests (kind cluster)...") + @command -v kind >/dev/null 2>&1 || ($(call print_error,"kind not found") && exit 1) + @go test -tags=e2e ./test/e2e/... -v -timeout 30m @$(call print_success,"E2E tests completed") # Run the real ICMP pinger integration test under privilege. @@ -272,13 +230,21 @@ test-all: @go tool cover -func=coverage/coverage.out | grep total | awk '{print "Total coverage: " $$3}' @$(call print_success,"All tests passed - coverage report: coverage/coverage.html") -# Check coverage meets minimum threshold (80%) -COVERAGE_THRESHOLD := 80 +# Single source for the coverage gate; ci.yml calls coverage-threshold with its own profile. +COVERAGE_THRESHOLD := 70 +COVERAGE_PROFILE ?= coverage/coverage.out + coverage-check: - @$(call print_status,"Checking coverage threshold (minimum $(COVERAGE_THRESHOLD)%)...") - @mkdir -p coverage - @go test ./... -covermode=atomic -coverprofile=coverage/coverage.out -short > /dev/null 2>&1 - @COVERAGE=$$(go tool cover -func=coverage/coverage.out | grep total | awk '{print $$3}' | sed 's/%//'); \ + @$(call print_status,"Generating unit test coverage profile...") + @mkdir -p $(dir $(COVERAGE_PROFILE)) + @go test ./pkg/... ./cmd/... -short -covermode=atomic -coverprofile=$(COVERAGE_PROFILE) > /dev/null + @$(MAKE) coverage-threshold COVERAGE_PROFILE=$(COVERAGE_PROFILE) + +coverage-threshold: + @$(call print_status,"Checking $(COVERAGE_PROFILE) against minimum $(COVERAGE_THRESHOLD)%...") + @[ -f $(COVERAGE_PROFILE) ] || { $(call print_error,"$(COVERAGE_PROFILE) not found"); exit 1; } + @COVERAGE=$$(go tool cover -func=$(COVERAGE_PROFILE) | grep total | awk '{print $$3}' | sed 's/%//'); \ + [ -n "$$COVERAGE" ] || { $(call print_error,"Failed to extract coverage percentage"); exit 1; }; \ echo "Current coverage: $${COVERAGE}%"; \ if [ $$(echo "$${COVERAGE} < $(COVERAGE_THRESHOLD)" | bc -l) -eq 1 ]; then \ $(call print_error,"Coverage $${COVERAGE}% is below minimum threshold of $(COVERAGE_THRESHOLD)%"); \ @@ -327,8 +293,8 @@ install-deps: # Docker build shorthand docker-build: build-all-images -# Docker push shorthand -docker-push: push-all-images +# Docker push shorthand; never tags :latest, release.yml owns that +docker-push: require-version push-all-images # ================================================================================================ # Validation Targets (mirrors CI/CD pipeline) @@ -346,7 +312,7 @@ validate-quick: check-prerequisites check-go-version validate-component: @$(call print_status,"Validating component: $(COMPONENT)...") - @./scripts/validate-pipeline-local.sh --component $(COMPONENT) + @./scripts/validate-pipeline-local.sh --component=$(COMPONENT) @$(call print_success,"Component validation passed") validate-local: validate-pipeline-local @@ -359,35 +325,45 @@ validate-local: validate-pipeline-local build-node-doctor-image: check-docker @$(call print_status,"Building node-doctor Docker image...") @docker build -f Dockerfile -t $(DOCKER_IMAGE_node-doctor):$(VERSION) . - @docker tag $(DOCKER_IMAGE_node-doctor):$(VERSION) $(DOCKER_IMAGE_node-doctor):latest @$(call print_success,"node-doctor image built: $(DOCKER_IMAGE_node-doctor):$(VERSION)") # Build Docker image for overlay-test-server build-overlay-test-server-image: check-docker @$(call print_status,"Building overlay-test-server Docker image...") @docker build -f Dockerfile.overlay-test -t $(DOCKER_IMAGE_overlay-test-server):$(VERSION) . - @docker tag $(DOCKER_IMAGE_overlay-test-server):$(VERSION) $(DOCKER_IMAGE_overlay-test-server):latest @$(call print_success,"overlay-test-server image built: $(DOCKER_IMAGE_overlay-test-server):$(VERSION)") +build-node-doctor-controller-image: check-docker + @$(call print_status,"Building node-doctor-controller Docker image...") + @docker build -f Dockerfile.controller \ + --build-arg VERSION=$(VERSION) --build-arg GIT_COMMIT=$(GIT_COMMIT) --build-arg BUILD_TIME=$(BUILD_TIME) \ + -t $(DOCKER_IMAGE_node-doctor-controller):$(VERSION) . + @docker tag $(DOCKER_IMAGE_node-doctor-controller):$(VERSION) $(DOCKER_IMAGE_node-doctor-controller):latest + @$(call print_success,"node-doctor-controller image built: $(DOCKER_IMAGE_node-doctor-controller):$(VERSION)") + # Build all images -build-all-images: build-node-doctor-image build-overlay-test-server-image +build-all-images: build-node-doctor-image build-overlay-test-server-image build-node-doctor-controller-image @$(call print_success,"Docker images built successfully") # Push node-doctor image to registry -push-node-doctor-image: +push-node-doctor-image: require-version @$(call print_status,"Pushing node-doctor image to registry...") @docker push $(DOCKER_IMAGE_node-doctor):$(VERSION) - @docker push $(DOCKER_IMAGE_node-doctor):latest @$(call print_success,"node-doctor image pushed") # Push overlay-test-server image to registry -push-overlay-test-server-image: +push-overlay-test-server-image: require-version @$(call print_status,"Pushing overlay-test-server image to registry...") @docker push $(DOCKER_IMAGE_overlay-test-server):$(VERSION) - @docker push $(DOCKER_IMAGE_overlay-test-server):latest @$(call print_success,"overlay-test-server image pushed") -push-all-images: push-node-doctor-image push-overlay-test-server-image +push-node-doctor-controller-image: + @$(call print_status,"Pushing node-doctor-controller image to registry...") + @docker push $(DOCKER_IMAGE_node-doctor-controller):$(VERSION) + @docker push $(DOCKER_IMAGE_node-doctor-controller):latest + @$(call print_success,"node-doctor-controller image pushed") + +push-all-images: push-node-doctor-image push-overlay-test-server-image push-node-doctor-controller-image @$(call print_success,"Images pushed to registry") # ================================================================================================ @@ -446,143 +422,6 @@ helm-package: @helm package ./helm/$(PROJECT_NAME) --version $(CHART_VERSION) @$(call print_success,"Helm chart packaged") -helm-publish: helm-package - @$(call print_status,"Publishing Helm chart to repository...") - @# CUSTOMIZE: Add your Helm chart publish command - @$(call print_success,"Helm chart published") - -# ================================================================================================ -# Deployment Targets -# ================================================================================================ - -deploy-dev: check-kubeconfig-dev - @$(call print_status,"Deploying to development environment...") - @KUBECONFIG=$(KUBECONFIG_DEV) helm upgrade --install $(PROJECT_NAME) ./helm/$(PROJECT_NAME) \ - --namespace $(NAMESPACE_DEV) \ - --set image.tag=$(VERSION) \ - --set environment=dev \ - --wait - @$(call print_success,"Deployed to development") - -deploy-stg: check-kubeconfig-stg - @$(call print_status,"Deploying to staging environment...") - @KUBECONFIG=$(KUBECONFIG_STG) helm upgrade --install $(PROJECT_NAME) ./helm/$(PROJECT_NAME) \ - --namespace $(NAMESPACE_STG) \ - --set image.tag=$(VERSION) \ - --set environment=staging \ - --wait - @$(call print_success,"Deployed to staging") - -deploy-prd: check-kubeconfig-prd - @echo "âš ī¸ CRITICAL DECISION: Deploy to Production" - @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - @echo "What: Deploy $(PROJECT_NAME) version $(VERSION) to production" - @echo "Why: Approved for production deployment" - @echo "Risk: May impact production users" - @echo "" - @read -p "Do you approve? (y/n): " approve; \ - if [ "$$approve" != "y" ] && [ "$$approve" != "Y" ]; then \ - $(call print_error,"Production deployment cancelled"); \ - exit 1; \ - fi - @$(call print_status,"Deploying to production environment...") - @KUBECONFIG=$(KUBECONFIG_PRD) helm upgrade --install $(PROJECT_NAME) ./helm/$(PROJECT_NAME) \ - --namespace $(NAMESPACE_PRD) \ - --set image.tag=$(VERSION) \ - --set environment=production \ - --wait - @$(call print_success,"Deployed to production") - -# ================================================================================================ -# Version Bump and Deployment Workflow -# ================================================================================================ - -bump: validate-pipeline-local - @$(call print_status,"Creating new version bump...") - @$(call print_status,"Current VERSION: $(VERSION)") - @git add . && git commit -m "bump: automated version bump to $(VERSION) [skip ci]" || true - @git push origin $$(git rev-parse --abbrev-ref HEAD) - @$(call print_success,"Version bumped and pushed - CI/CD will handle deployment") - -bump-with-monitoring: bump - @$(call print_status,"Monitoring GitHub Actions workflow...") - @$(MAKE) gh-watch - -# Increment RC version (e.g., v0.1.0-rc.1 -> v0.1.0-rc.2) -increment-rc-version: - @$(call print_status,"Incrementing RC version...") - @CURRENT_RC=$$(cat $(RC_VERSION_FILE)); \ - BASE_VERSION=$$(echo $$CURRENT_RC | sed 's/-rc\.[0-9]*$$//'); \ - RC_NUM=$$(echo $$CURRENT_RC | sed 's/.*-rc\.//'); \ - NEW_RC_NUM=$$((RC_NUM + 1)); \ - NEW_RC_VERSION="$$BASE_VERSION-rc.$$NEW_RC_NUM"; \ - echo $$NEW_RC_VERSION > $(RC_VERSION_FILE); \ - echo "$(GREEN)✅ RC version updated: $$CURRENT_RC -> $$NEW_RC_VERSION$(NC)" - -# Deploy to production cluster using kubectl -deploy-prd-kubectl: check-kubectl - @$(call print_status,Deploying to production cluster $(KUBECTL_CONTEXT_PRD)...) - @$(call print_status,Checking namespace $(NAMESPACE_PRD)...) - @KUBECONFIG=$(KUBECONFIG_PRD) kubectl config use-context $(KUBECTL_CONTEXT_PRD) - @KUBECONFIG=$(KUBECONFIG_PRD) kubectl get namespace $(NAMESPACE_PRD) > /dev/null 2>&1 || \ - (echo "Creating namespace $(NAMESPACE_PRD)..." && \ - KUBECONFIG=$(KUBECONFIG_PRD) kubectl create namespace $(NAMESPACE_PRD)) - @$(call print_status,Applying RBAC resources...) - @KUBECONFIG=$(KUBECONFIG_PRD) kubectl apply -f deployment/rbac.yaml -n $(NAMESPACE_PRD) - @$(call print_status,Applying DaemonSet...) - @KUBECONFIG=$(KUBECONFIG_PRD) kubectl apply -f deployment/daemonset.yaml -n $(NAMESPACE_PRD) - @$(call print_status,Waiting for rollout to complete...) - @KUBECONFIG=$(KUBECONFIG_PRD) kubectl rollout status daemonset/node-doctor -n $(NAMESPACE_PRD) --timeout=5m - @$(call print_success,Deployed to production cluster $(KUBECTL_CONTEXT_PRD)) - -# Build and push RC release to $(REGISTRY) and deploy to a1-ops-prd cluster. -# -# This target deliberately does NOT tag :latest. It used to, which meant a workstation -# build of a *release candidate* could overwrite the :latest that release.yml publishes -# only for non-rc tags. Only .github/workflows/release.yml moves :latest. -# -# The git steps below are also not `|| true`: a failed commit/tag/push used to be -# swallowed, leaving an image pushed and a cluster deployed from a revision that was -# never tagged or pushed anywhere. -bump-rc: validate-pipeline-local increment-rc-version - @$(call print_status,Building RC release: $(RC_VERSION)) - @$(call print_status,Registry: $(REGISTRY)/$(PROJECT_NAME)) - @$(call print_status,Cluster: $(KUBECTL_CONTEXT_PRD) Namespace: $(NAMESPACE_PRD)) - @echo "" - @$(call print_status,Building multi-arch Docker image for amd64 and arm64...) - @docker buildx build \ - --platform linux/amd64,linux/arm64 \ - --build-arg VERSION=$(RC_VERSION) \ - --build-arg GIT_COMMIT=$(GIT_COMMIT) \ - --build-arg BUILD_TIME=$(BUILD_TIME) \ - -t $(REGISTRY)/$(PROJECT_NAME):$(RC_VERSION) \ - -f Dockerfile \ - --push \ - . - @$(call print_success,Multi-arch image built and pushed: $(REGISTRY)/$(PROJECT_NAME):$(RC_VERSION)) - @echo "" - @$(call print_status,Updating DaemonSet image tag...) - @sed -i.bak 's|image: .*node-doctor:.*|image: $(REGISTRY)/$(PROJECT_NAME):$(RC_VERSION)|' deployment/daemonset.yaml - @rm -f deployment/daemonset.yaml.bak - @echo "" - @$(MAKE) deploy-prd-kubectl - @echo "" - @$(call print_status,Committing RC release...) - @git add $(RC_VERSION_FILE) deployment/daemonset.yaml - @git commit -m "bump-rc: Release candidate $(RC_VERSION) deployed to $(KUBECTL_CONTEXT_PRD)" - @git tag -a $(RC_VERSION) -m "Release candidate $(RC_VERSION)" - @git push origin main --tags - @echo "" - @echo "🎉 RC Release $(RC_VERSION) complete!" - @echo " - Built and pushed to $(REGISTRY)/$(PROJECT_NAME):$(RC_VERSION)" - @echo " - Deployed to $(KUBECTL_CONTEXT_PRD) cluster" - @echo " - Tagged as $(RC_VERSION)" - @echo "" - @echo "Next steps:" - @echo " - Verify deployment: kubectl --context=$(KUBECTL_CONTEXT_PRD) -n $(NAMESPACE_PRD) get pods -l app=node-doctor" - @echo " - Check logs: kubectl --context=$(KUBECTL_CONTEXT_PRD) -n $(NAMESPACE_PRD) logs -l app=node-doctor" - @echo " - Monitor health: kubectl --context=$(KUBECTL_CONTEXT_PRD) -n $(NAMESPACE_PRD) get pods -l app=node-doctor -w" - # ================================================================================================ # Quality Workflow Targets # ================================================================================================ @@ -591,21 +430,6 @@ workflow-status: @$(call print_status,"Checking workflow status...") @echo "TODO: Check TaskForge or project management system" -qa-check: - @$(call print_status,"Running QA validation...") - @echo "QA Check: Running validation suite" - @$(MAKE) validate-pipeline-local - @$(MAKE) test-local - @$(call print_success,"QA validation passed") - -devils-advocate: - @$(call print_status,"Running Devils Advocate validation...") - @echo "Devils Advocate: Challenging implementation..." - @echo "- Testing edge cases" - @echo "- Verifying error handling" - @echo "- Checking security implications" - @$(call print_success,"Devils Advocate review complete") - # ================================================================================================ # GitHub Actions Monitoring # ================================================================================================ @@ -624,14 +448,14 @@ gh-logs: gh-builds: @$(call print_status,"Showing recent builds...") - @gh run list --workflow=pipeline-v2.yml --limit 10 + @gh run list --workflow=ci.yml --limit 10 # ================================================================================================ # Help System # ================================================================================================ help: - @echo "$(PROJECT_NAME) Makefile - Repository Template" + @echo "$(PROJECT_NAME) Makefile" @echo "" @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" @echo "QUICK START" @@ -652,7 +476,7 @@ help: @echo " make build Compile binary (shorthand for build-local)" @echo " make build-local Build node-doctor binary" @echo " make docker-build Build Docker image" - @echo " make docker-push Push Docker image to registry" + @echo " make docker-push VERSION=x.y.z Push Docker image (VERSION required, never :latest)" @echo " make clean Remove build artifacts and caches" @echo "" @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" @@ -661,9 +485,9 @@ help: @echo "" @echo " make test Run unit tests (fast)" @echo " make test-integration Run integration tests" - @echo " make test-e2e Run end-to-end tests" + @echo " make test-e2e Run end-to-end tests (creates a kind cluster)" @echo " make test-all Run all tests with coverage report" - @echo " make coverage-check Verify coverage >= 80% threshold" + @echo " make coverage-check Verify unit coverage >= $(COVERAGE_THRESHOLD)% threshold" @echo " make lint Run golangci-lint" @echo " make fmt Format code with gofmt and goimports" @echo "" @@ -672,36 +496,17 @@ help: @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" @echo "" @echo " make install-deps Install golangci-lint, goimports, etc." - @echo " make validate-quick Quick validation (format, vet, staticcheck)" - @echo " make validate-pipeline-local Full validation (mirrors CI/CD)" @echo "" @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" @echo "VALIDATION COMMANDS (Pre-push)" @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" @echo "" - @echo " make validate-pipeline-local Full validation (mirrors CI/CD exactly)" - @echo " make validate-quick Quick validation (format, vet, staticcheck)" - @echo " make validate-component COMPONENT=api Validate specific component" + @echo " make validate-pipeline-local gofmt, vet, golangci-lint, gosec, tests, helm (mirrors CI)" + @echo " make validate-quick Same without tests" + @echo " make validate-component COMPONENT=node-doctor Same, explicit component" + @echo " SKIP_LINT=1 Allow a missing golangci-lint" @echo "" - @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - @echo "DEPLOYMENT COMMANDS" - @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - @echo "" - @echo " make bump Bump version and trigger CI/CD" - @echo " make bump-rc Build RC, push to $(REGISTRY), deploy to a1-ops-prd" - @echo " make bump-with-monitoring Bump and watch deployment" - @echo " make deploy-dev Deploy to development" - @echo " make deploy-stg Deploy to staging" - @echo " make deploy-prd Deploy to production (requires approval)" - @echo " make deploy-prd-kubectl Deploy to a1-ops-prd using kubectl" - @echo "" - @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - @echo "QUALITY WORKFLOW COMMANDS" - @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - @echo "" - @echo " make qa-check Run QA validation" - @echo " make devils-advocate Run adversarial testing" - @echo " make workflow-status Check workflow status" + @echo " Deployment is done by .github/workflows/release.yml on a v* tag; see docs/release-process.md" @echo "" @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" @echo "GITHUB ACTIONS MONITORING" @@ -716,9 +521,10 @@ help: @echo "HELM COMMANDS" @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" @echo "" + @echo " make helm-generate Render Chart.yaml/values.yaml from their templates" + @echo " make helm-verify-generated Fail if Chart.yaml/values.yaml drifted from templates" @echo " make helm-lint Lint Helm chart" @echo " make helm-package Package Helm chart" - @echo " make helm-publish Publish Helm chart to repository" @echo "" @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" @echo "UTILITY COMMANDS" @@ -726,20 +532,15 @@ help: @echo "" @echo " make check-prerequisites Check all required tools installed" @echo " make check-go-version Verify Go version matches requirements" - @echo " make check-kubeconfig-dev Verify development kubeconfig" + @echo " make workflow-status Check workflow status" @echo " make help Show this help message" @echo "" - @echo "For more information, see docs/development/" + @echo "For more information, see CONTRIBUTING.md" workflow-help: - @echo "Quality Workflow Commands" + @echo "Workflow Commands" @echo "" @echo " make workflow-status - Show current workflow status" - @echo " make qa-check - Run QA validation gate" - @echo " make devils-advocate - Run adversarial testing gate" - @echo " make workflow-validate - Full workflow validation" - @echo "" - @echo "See docs/development/task-execution-workflow.md for complete workflow documentation" gh-help: @echo "GitHub Actions Monitoring Commands" diff --git a/README.md b/README.md index 29f5369..321bd3c 100644 --- a/README.md +++ b/README.md @@ -513,7 +513,7 @@ node-doctor/ - [Monitors](docs/monitors.md) - Health check monitor implementations - [Remediation](docs/remediation.md) - Auto-remediation system - [Configuration](docs/configuration.md) - Configuration reference -- [Deployment](docs/deployment.md) - Deployment guide for production clusters +- [Deployment](deployment/README.md) - Deployment guide for production clusters - [Release Process](docs/release-process.md) - Release management and versioning - [Troubleshooting](docs/troubleshooting.md) - Common issues and debugging guide diff --git a/cmd/node-doctor/bind_ordering_test.go b/cmd/node-doctor/bind_ordering_test.go index c9f8d40..4e4aae1 100644 --- a/cmd/node-doctor/bind_ordering_test.go +++ b/cmd/node-doctor/bind_ordering_test.go @@ -5,7 +5,6 @@ import ( "go/ast" "go/parser" "go/token" - "path/filepath" "strings" "testing" "time" @@ -29,7 +28,7 @@ import ( // anyone reorders createExporters so networked init runs first, this test hangs // on an unservable socket and fails. func TestHealthEndpointServesBeforeNetworkedExporters(t *testing.T) { - socket := filepath.Join(t.TempDir(), "health.sock") + socket := testSocketPath(t) // Phase 2 blocks until we release it — standing in for a wedged exporter // Start() on a degraded node. diff --git a/cmd/node-doctor/healthcheck_test.go b/cmd/node-doctor/healthcheck_test.go index 6f91b64..2c10da8 100644 --- a/cmd/node-doctor/healthcheck_test.go +++ b/cmd/node-doctor/healthcheck_test.go @@ -2,17 +2,29 @@ package main import ( "context" + "os" "path/filepath" "testing" "github.com/supporttools/node-doctor/pkg/health" ) +// t.TempDir() paths can exceed the unix socket path limit (104 bytes on macOS). +func testSocketPath(t *testing.T) string { + t.Helper() + dir, err := os.MkdirTemp("", "nd") + if err != nil { + t.Fatalf("mkdir temp: %v", err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + return filepath.Join(dir, "health.sock") +} + // TestRunHealthCheck exercises the exec-probe health check against a real health // server unix socket. This is the code path the Kubernetes exec probes invoke, and // the reason a1pinode01-class hostPort-8080 conflicts no longer crashloop the pod. func TestRunHealthCheck(t *testing.T) { - sockPath := filepath.Join(t.TempDir(), "health.sock") + sockPath := testSocketPath(t) srv, err := health.NewServer(&health.Config{ Enabled: true, Port: 0, // ephemeral TCP, no conflict diff --git a/cmd/node-doctor/main.go b/cmd/node-doctor/main.go index 6b39afc..42ee3db 100644 --- a/cmd/node-doctor/main.go +++ b/cmd/node-doctor/main.go @@ -281,21 +281,14 @@ func main() { // /remediation/history immediately on start (no race window). var remediatorRegistry *remediators.RemediatorRegistry if config.Remediation.Enabled { - maxPerHour := config.Remediation.MaxRemediationsPerHour - if maxPerHour == 0 { - maxPerHour = 10 // sensible default - } - historySize := config.Remediation.HistorySize - if historySize == 0 { - historySize = 100 + var err error + remediatorRegistry, err = remediators.NewRegistryFromConfig(&config.Remediation, config.Settings.DryRunMode) + if err != nil { + log.Fatalf("Failed to initialize remediator registry: %v", err) } - remediatorRegistry = remediators.NewRegistry(maxPerHour, historySize) - remediatorRegistry.SetDryRun(config.Remediation.DryRun || config.Settings.DryRunMode) - // Wire the per-minute token-bucket burst limit. A value of 0 leaves the - // per-minute check disabled (only the per-hour window applies). - remediatorRegistry.SetMaxRemediationsPerMinute(config.Remediation.MaxRemediationsPerMinute) - log.Printf("[INFO] Remediator registry initialized (dry-run=%v, maxPerHour=%d, maxPerMinute=%d)", - remediatorRegistry.IsDryRun(), maxPerHour, config.Remediation.MaxRemediationsPerMinute) + stats := remediatorRegistry.GetStats() + log.Printf("[INFO] Remediator registry initialized (dry-run=%v, maxPerHour=%d, maxPerMinute=%d, circuitBreaker=%+v)", + stats.DryRun, stats.MaxPerHour, config.Remediation.MaxRemediationsPerMinute, remediatorRegistry.GetCircuitBreakerConfig()) // Register the built-in remediator strategies so the detector's dispatch // (which addresses a remediator by its strategy type) can find one. diff --git a/deploy/controller/deployment.yaml b/deploy/controller/deployment.yaml index a546dd5..b9df478 100644 --- a/deploy/controller/deployment.yaml +++ b/deploy/controller/deployment.yaml @@ -34,7 +34,7 @@ spec: fsGroup: 65534 containers: - name: controller - image: supporttools/node-doctor-controller:latest + image: supporttools/node-doctor-controller:1.9.0 imagePullPolicy: IfNotPresent args: - --config=/etc/node-doctor/controller.yaml diff --git a/docs/README.md b/docs/README.md index 01f98f8..9f5e647 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,216 +1,36 @@ -# Documentation Hub - -Central navigation for all project documentation. - -## 🚀 Quick Start - -New to the project? Start here: - -1. **[Quick Start Guide](quick-start.md)** - Deploy Node Doctor, import dashboards, configure monitors -2. **[Task Execution Workflow](development/task-execution-workflow.md)** - Mandatory development process -3. **[Template README](../README.md)** - Template usage guide - -## 📚 Core Documentation - -### Development Standards -**Location**: `docs/development/` - -Essential reading for all developers: - -- **[Task Execution Workflow](development/task-execution-workflow.md)** ⭐ MANDATORY - 8-step process -- **[Master Task Management](development/master-task-management.md)** ⭐ Priority system (P1-P5) -- **[TaskForge Usage Guide](development/taskforge-usage-guide.md)** - Project management system -- **[API Handler Architecture](development/api-handler-architecture-standards.md)** - Handler patterns -- **[Helm Chart Standards](development/helm-chart-standards.md)** - Kubernetes deployment -- **[Local CI/CD Validation](development/local-cicd-validation-guide.md)** - Pre-push validation -- **[Database Patterns Guide](development/database-patterns-guide.md)** - Multi-tenant vs single-tenant -- **[Error Response Standard](development/error-response-standard.md)** - API error handling -- **[Web UI-API Synchronization](development/web-ui-api-synchronization.md)** - Frontend sync -- **[Go SDK Synchronization](development/go-sdk-synchronization-guide.md)** - SDK updates - -### Workflows -**Location**: `docs/workflows/` - -Standard process workflows: - -- **[Feature Development](workflows/feature-development.md)** ⭐ Complete feature lifecycle -- **[Project Initialization](workflows/project-initialization.md)** - Project setup -- **[PRD Creation](workflows/prd-creation.md)** - Requirements documents -- **[Sprint Management](workflows/sprint-management.md)** - Agile processes -- **[Deployment](workflows/deployment.md)** - Release procedures - -## đŸŽ¯ By Role - -### For Developers -1. [Task Execution Workflow](development/task-execution-workflow.md) -2. [API Handler Architecture](development/api-handler-architecture-standards.md) -3. [Local CI/CD Validation](development/local-cicd-validation-guide.md) -4. [Feature Development Workflow](workflows/feature-development.md) - -### For DevOps/SRE -1. [Helm Chart Standards](development/helm-chart-standards.md) -2. [Deployment Workflow](workflows/deployment.md) -3. [Local CI/CD Validation](development/local-cicd-validation-guide.md) - -### For Architects -1. [Database Patterns Guide](development/database-patterns-guide.md) -2. [Architecture Review Workflow](workflows/architecture-review.md) -3. [Design Documentation](design/) - -### For Product/QA -1. [PRD Creation Workflow](workflows/prd-creation.md) -2. [Testing Strategy](workflows/testing-strategy.md) -3. [TaskForge Usage Guide](development/taskforge-usage-guide.md) - -### For Project Managers -1. [Master Task Management](development/master-task-management.md) -2. [TaskForge Usage Guide](development/taskforge-usage-guide.md) -3. [Sprint Management Workflow](workflows/sprint-management.md) - -## 📂 Directory Structure - -``` -docs/ -├── README.md # This file - navigation hub -├── development/ # Core standards and guides (30 files) -│ ├── task-execution-workflow.md -│ ├── master-task-management.md -│ ├── taskforge-usage-guide.md -│ ├── api-handler-architecture-standards.md -│ ├── helm-chart-standards.md -│ └── ... (see development/ for full list) -├── workflows/ # Process workflows -│ ├── feature-development.md -│ ├── project-initialization.md -│ ├── prd-creation.md -│ └── ... -├── design/ # Architecture and design specs -├── operations/ # Deployment and operations guides -├── testing/ # Testing strategies and guides -├── security/ # Security policies and procedures -├── examples/ # Configuration examples -└── archive/ # Historical documentation -``` - -## 🔍 Finding Documentation - -### By Topic - -- **Task Management**: `development/master-task-management.md`, `development/taskforge-usage-guide.md` -- **Quality Assurance**: `development/task-execution-workflow.md` (quality gates and QA process) -- **API Development**: `development/api-handler-architecture-standards.md`, `development/error-response-standard.md` -- **Database**: `development/database-patterns-guide.md` -- **Deployment**: `development/helm-chart-standards.md`, `workflows/deployment.md` -- **Testing**: `development/local-cicd-validation-guide.md`, `workflows/testing-strategy.md` -- **Frontend**: `development/web-ui-api-synchronization.md` -- **Operations**: `troubleshooting.md` - Common issues, debugging, performance tuning - -### By Question - -- **"How do I deploy Node Doctor?"** → [Quick Start Guide](quick-start.md) -- **"How do I import Grafana dashboards?"** → [Quick Start Guide](quick-start.md#import-grafana-dashboards) -- **"How do I start a new feature?"** → [Feature Development Workflow](workflows/feature-development.md) -- **"What's the development process?"** → [Task Execution Workflow](development/task-execution-workflow.md) -- **"How do I validate before pushing?"** → [Local CI/CD Validation](development/local-cicd-validation-guide.md) -- **"How do I deploy?"** → [Deployment Workflow](workflows/deployment.md) -- **"What are the API standards?"** → [API Handler Architecture](development/api-handler-architecture-standards.md) -- **"How do I write Helm charts?"** → [Helm Chart Standards](development/helm-chart-standards.md) -- **"How do tasks work?"** → [Master Task Management](development/master-task-management.md) -- **"Why isn't my monitor running?"** → [Troubleshooting Guide](troubleshooting.md) -- **"How do I debug issues?"** → [Troubleshooting Guide](troubleshooting.md) - -## 🎓 Learning Path - -### Week 1: Foundation -1. Read Task Execution Workflow -2. Read Master Task Management -3. Try a small task following the workflow - -### Week 2: Standards -1. Read API Handler Architecture Standards -2. Read Helm Chart Standards -3. Read Local CI/CD Validation Guide -4. Practice validation locally - -### Week 3: Workflows -1. Read Feature Development Workflow -2. Read Deployment Workflow -3. Complete a feature following workflows -4. Get QA and Devils Advocate review - -### Week 4: Mastery -1. Review all skeptic agents -2. Understand quality gates deeply -3. Mentor others on the process -4. Contribute documentation improvements - -## 📝 Documentation Standards - -### Creating Documentation - -1. **Use Markdown**: GitHub-flavored markdown -2. **Clear Structure**: Headings, lists, code blocks -3. **Cross-Reference**: Link to related docs -4. **Examples**: Include practical examples -5. **Update Date**: Add "Last Updated" footer - -### Organizing Documentation - -- **development/**: Standards, guides, and patterns -- **workflows/**: Step-by-step process documentation -- **design/**: Architecture and design decisions -- **operations/**: Deployment and operational guides -- **examples/**: Configuration examples -- **archive/**: Historical documentation (not current) - -### Maintaining Documentation - -- Review quarterly for accuracy -- Update when processes change -- Archive obsolete documentation -- Keep examples current -- Test documented procedures - -## 🔄 Documentation Updates - -This documentation is living and should evolve with the project: - -- **Add**: New standards, workflows, or guides -- **Update**: When processes change -- **Archive**: When practices become obsolete -- **Link**: Cross-reference related documentation -- **Example**: Add real-world examples - -## 💡 Contributing - -When adding documentation: - -1. Determine correct directory -2. Use clear, descriptive filename -3. Follow markdown standards -4. Add cross-references -5. Update this README if needed -6. Include examples -7. Test instructions work - -## ❓ Getting Help - -- **Can't find documentation?** Check this README or search repository -- **Documentation unclear?** Open an issue or ask the team -- **Want to contribute?** Follow contribution guidelines - -## đŸŽ¯ Remember - -**Documentation is code**. It should be: -- Version controlled -- Peer reviewed -- Tested for accuracy -- Updated regularly -- Easy to find and navigate - -Good documentation saves hours of confusion and prevents mistakes. - ---- - -**Last Updated**: 2025-10-29 -**Maintained By**: Development Team +# Node Doctor Documentation + +Index of the docs in this repository. Pick the section for your role. + +## Operators + +Deploying, configuring, and running Node Doctor. + +- [Quick Start](quick-start.md) - Install Node Doctor and import the dashboards. +- [Helm Chart](../helm/node-doctor/README.md) - Chart values and install options. +- [DaemonSet Deployment](../deployment/README.md) - Deploy with plain manifests. +- [Configuration Reference](configuration.md) - Every config option and its default. +- [Monitors](monitors.md) - The health check monitors and their settings. +- [DNS Monitor: Advanced Features](monitors/dns-advanced.md) - Extra DNS monitor options. +- [DNS Monitor: Troubleshooting](monitors/dns-troubleshooting.md) - Diagnose DNS monitor problems. +- [Remediation](remediation.md) - Auto-remediation and its safety controls. +- [Controller Deployment](controller-deployment.md) - Deploy the Node Doctor controller. +- [Troubleshooting](troubleshooting.md) - Common problems and how to debug them. +- [Grafana Dashboards](../dashboards/README.md) - Dashboard files and how to import them. +- [Example Configurations](../config/examples/README.md) - Sample config files. + +## Developers + +Building, testing, and releasing Node Doctor. + +- [Architecture](architecture.md) - Components and how they fit together. +- [Developer Testing Guide](testing-guide.md) - Test patterns and how to run each suite. +- [Testing](testing.md) - Test organization and commands. +- [Release Process](release-process.md) - Versioning, tagging, and publishing a release. +- [Contributing](../CONTRIBUTING.md) - Workflow, coding standards, and PR process. +- [Security Policy](../SECURITY.md) - How to report a vulnerability. +- [E2E Tests](../test/e2e/README.md) - Run the end-to-end suite against a cluster. +- [Integration Tests](../test/integration/README.md) - Run the integration suite. +- [GitHub Actions Workflows](../.github/workflows/README.md) - What each CI workflow does. + +Run `scripts/check-doc-links.sh` to check relative links in every markdown file. diff --git a/docs/deployment.md b/docs/deployment.md deleted file mode 100644 index 06977ec..0000000 --- a/docs/deployment.md +++ /dev/null @@ -1,432 +0,0 @@ -# Node Doctor - Real Cluster Deployment Guide - -This guide walks through deploying Node Doctor to a real Kubernetes cluster. - -## Prerequisites - -- Kubernetes cluster (1.19+) with kubectl access -- Docker installed for building images (for manual deployment) -- Access to Docker Hub registry (for manual deployment) -- Cluster admin privileges (for RBAC and DaemonSet deployment) - -## Deployment Options - -| Method | Recommended For | Description | -|--------|----------------|-------------| -| **Helm Chart** | Production | Easiest installation with configurable values | -| **Automated RC** | Development/Testing | One-command build and deploy for RC releases | -| **Manual** | Custom/Advanced | Full control over each deployment step | - ---- - -## Option 1: Helm Chart Deployment (Recommended) - -The Helm chart is the recommended method for deploying Node Doctor to production clusters. - -### Add the Helm Repository - -```bash -helm repo add supporttools https://charts.support.tools -helm repo update -``` - -### Basic Installation - -```bash -helm install node-doctor supporttools/node-doctor \ - --namespace node-doctor \ - --create-namespace -``` - -### Installation with Custom Values - -```bash -# Create custom values file -cat > custom-values.yaml << 'EOF' -settings: - logLevel: info - logFormat: json - updateInterval: 30s - enableRemediation: true - dryRunMode: false - -resources: - requests: - cpu: 50m - memory: 128Mi - limits: - cpu: 500m - memory: 256Mi - -serviceMonitor: - enabled: true - interval: 30s -EOF - -# Install with custom values -helm install node-doctor supporttools/node-doctor \ - --namespace node-doctor \ - --create-namespace \ - -f custom-values.yaml -``` - -### Verify Helm Deployment - -```bash -# Check DaemonSet status -kubectl get daemonset -n node-doctor node-doctor - -# View pods -kubectl get pods -n node-doctor -l app.kubernetes.io/name=node-doctor - -# Check logs -kubectl logs -n node-doctor -l app.kubernetes.io/name=node-doctor --tail=50 - -# View node conditions -kubectl get nodes -o custom-columns='NAME:.metadata.name,HEALTHY:.status.conditions[?(@.type=="NodeDoctorHealthy")].status' -``` - -### Helm Upgrade and Rollback - -```bash -# Upgrade to latest version -helm repo update -helm upgrade node-doctor supporttools/node-doctor -n node-doctor - -# Upgrade with new values -helm upgrade node-doctor supporttools/node-doctor \ - -n node-doctor \ - --set settings.logLevel=debug - -# Rollback to previous release -helm rollback node-doctor -n node-doctor -``` - -### Uninstall via Helm - -```bash -helm uninstall node-doctor -n node-doctor -kubectl delete namespace node-doctor -``` - -For complete Helm chart configuration options, see [../helm/node-doctor/README.md](../helm/node-doctor/README.md). - ---- - -## Option 2: Automated RC Deployment (Development/Testing) - -For rapid deployment of release candidates to the `a1-ops-prd` cluster: - -```bash -# One command to build, push, and deploy -make bump-rc -``` - -This single command will: -1. ✅ Validate the pipeline (run all tests) -2. ✅ Increment the RC version (e.g., v0.1.0-rc.1 → v0.1.0-rc.2) -3. ✅ Build Docker image with RC tag -4. ✅ Push to Docker Hub registry: `supporttools/node-doctor` -5. ✅ Deploy to `a1-ops-prd` cluster in `node-doctor` namespace -6. ✅ Commit and tag the release - -**Configuration:** -- Cluster: `a1-ops-prd` (from `~/.kube/config`) -- Namespace: `node-doctor` (auto-created if needed) -- Registry: `docker.io/supporttools/node-doctor` -- Version tracking: `.version-rc` file - -**Verify Deployment:** -```bash -# Check pod status -kubectl --context=a1-ops-prd -n node-doctor get pods -l app=node-doctor - -# View logs -kubectl --context=a1-ops-prd -n node-doctor logs -l app=node-doctor - -# Monitor health -kubectl --context=a1-ops-prd -n node-doctor get pods -l app=node-doctor -w -``` - ---- - -## Option 3: Manual Deployment (Advanced) - -If you prefer manual control over each step, follow the detailed instructions below. - -### Step 1: Build and Push Container Image - -```bash -# Build the Docker image with version tag -docker build -t supporttools/node-doctor:v0.1.0 \ - --build-arg VERSION=v0.1.0 \ - --build-arg GIT_COMMIT=$(git rev-parse --short HEAD) \ - --build-arg BUILD_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ) \ - . - -# Tag as latest -docker tag supporttools/node-doctor:v0.1.0 \ - supporttools/node-doctor:latest - -# Login to Docker Hub (if not already logged in) -docker login - -# Push both tags -docker push supporttools/node-doctor:v0.1.0 -docker push supporttools/node-doctor:latest -``` - -**Image Details:** -- Size: ~85 MB (optimized multi-stage Alpine build) -- Base: Alpine Linux 3.19 -- Go: 1.24.4 -- Binary location: `/usr/local/bin/node-doctor` - -## Step 2: Deploy to Kubernetes Cluster - -### 2.1 Deploy RBAC Resources (MUST BE FIRST) - -```bash -# Apply RBAC: ServiceAccount, ClusterRole, ClusterRoleBinding -kubectl apply -f deployment/rbac.yaml - -# Verify RBAC resources created -kubectl get serviceaccount -n kube-system node-doctor -kubectl get clusterrole node-doctor -kubectl get clusterrolebinding node-doctor -``` - -### 2.2 Deploy Node Doctor DaemonSet - -```bash -# Apply DaemonSet (includes ConfigMap and Service) -kubectl apply -f deployment/daemonset.yaml - -# Watch rollout -kubectl rollout status daemonset/node-doctor -n kube-system - -# Verify pods running on all nodes -kubectl get pods -n kube-system -l app=node-doctor -o wide -``` - -### 2.3 Quick Deployment (Both Files) - -```bash -# Apply both in correct order -kubectl apply -f deployment/rbac.yaml -f deployment/daemonset.yaml -``` - -## Step 3: Verify Deployment - -### 3.1 Check Pod Status - -```bash -# View all Node Doctor pods -kubectl get pods -n kube-system -l app=node-doctor - -# Check pod logs -POD_NAME=$(kubectl get pods -n kube-system -l app=node-doctor -o jsonpath='{.items[0].metadata.name}') -kubectl logs -n kube-system $POD_NAME --tail=50 -f -``` - -### 3.2 Test Health Endpoints - -```bash -# Test health endpoint -kubectl exec -n kube-system $POD_NAME -- curl -s localhost:8080/healthz - -# Test readiness -kubectl exec -n kube-system $POD_NAME -- curl -s localhost:8080/ready - -# Test detailed status -kubectl exec -n kube-system $POD_NAME -- curl -s localhost:8080/status | jq . -``` - -### 3.3 Check Metrics - -```bash -# View Prometheus metrics -kubectl exec -n kube-system $POD_NAME -- curl -s localhost:9100/metrics | head -20 - -# Check for Node Doctor specific metrics -kubectl exec -n kube-system $POD_NAME -- curl -s localhost:9100/metrics | grep node_doctor -``` - -### 3.4 Verify Node Conditions - -```bash -# Check if NodeDoctorHealthy condition is set -kubectl get nodes -o json | jq '.items[].status.conditions[] | select(.type == "NodeDoctorHealthy")' - -# View full node conditions -kubectl describe nodes | grep -A 10 "Conditions:" -``` - -### 3.5 Check Kubernetes Events - -```bash -# View Node Doctor events -kubectl get events -n kube-system --field-selector involvedObject.name=node-doctor --sort-by='.lastTimestamp' -``` - -## Step 4: Run Validation Script - -```bash -# Run comprehensive validation -cd /home/mmattox/go/src/github.com/supporttools/node-doctor -bash deployment/validate.sh -``` - -**Expected Output:** -- ✅ kubectl available -- ✅ Cluster connection OK -- ✅ RBAC manifest exists -- ✅ DaemonSet manifest exists -- ✅ DaemonSet deployed -- ✅ All pods running -- ✅ ServiceAccount exists -- ✅ ClusterRole exists -- ✅ ClusterRoleBinding exists -- ✅ ConfigMap exists -- ✅ Service exists -- ✅ Health endpoints responding -- ✅ Metrics endpoint working - -## What's Monitored - -### CPU Monitor (system-cpu) -- Load average thresholds: 80% warning, 95% critical -- Thermal throttling detection -- CPU usage monitoring - -### Memory Monitor (system-memory) -- Memory usage: 85% warning, 95% critical -- Swap usage: 50% warning, 80% critical -- OOM kill detection via /dev/kmsg - -### Disk Monitor (system-disk) -- Monitored paths: /, /var/lib/kubelet, /var/lib/docker, /var/lib/containerd -- Space thresholds: 85% warning, 95% critical -- Inode thresholds: 85% warning, 95% critical -- Readonly filesystem detection -- I/O health monitoring - -## Monitoring Configuration - -Configuration is embedded in the DaemonSet ConfigMap (lines 498-569 of daemonset.yaml). - -To update configuration: -```bash -# Edit ConfigMap -kubectl edit configmap -n kube-system node-doctor-config - -# Or apply updated manifest -kubectl apply -f deployment/daemonset.yaml - -# Restart pods to pick up changes -kubectl rollout restart daemonset/node-doctor -n kube-system -``` - -## Troubleshooting - -### Pods Not Starting - -```bash -# Check pod events -kubectl describe pod -n kube-system $POD_NAME - -# Common issues: -# - ImagePullBackOff: Check Harbor registry access -# - CrashLoopBackOff: Check logs for errors -# - Pending: Check node selectors and tolerations -``` - -### Permission Errors - -```bash -# Verify RBAC is applied -kubectl get serviceaccount -n kube-system node-doctor -kubectl get clusterrole node-doctor -kubectl get clusterrolebinding node-doctor - -# Check pod service account -kubectl get pods -n kube-system -l app=node-doctor -o jsonpath='{.items[0].spec.serviceAccountName}' -``` - -### Monitors Not Working - -```bash -# Check pod has host access -kubectl exec -n kube-system $POD_NAME -- ls -la /host -kubectl exec -n kube-system $POD_NAME -- ls -la /dev/kmsg -kubectl exec -n kube-system $POD_NAME -- cat /proc/meminfo | head -5 - -# Verify security context -kubectl get pods -n kube-system -l app=node-doctor -o jsonpath='{.items[0].spec.containers[0].securityContext}' -``` - -### No Metrics Appearing - -```bash -# Check Prometheus endpoint is accessible -kubectl exec -n kube-system $POD_NAME -- curl -s localhost:9100/metrics - -# Verify Service is created -kubectl get service -n kube-system node-doctor-metrics - -# Check if Prometheus can scrape -kubectl get pods -n kube-system -l app=node-doctor -o jsonpath='{.items[0].metadata.annotations}' -``` - -## Uninstalling - -```bash -# Remove DaemonSet (includes ConfigMap and Service) -kubectl delete -f deployment/daemonset.yaml - -# Remove RBAC resources -kubectl delete -f deployment/rbac.yaml - -# Verify cleanup -kubectl get all -n kube-system -l app=node-doctor -``` - -## Production Considerations - -1. **Resource Limits**: Adjust CPU/memory limits based on cluster size -2. **Image Pull Secrets**: Add imagePullSecrets if Harbor requires authentication -3. **Node Selectors**: Add nodeSelector to limit deployment to specific nodes -4. **Monitoring Integration**: Configure Prometheus ServiceMonitor or PodMonitor -5. **Alerting**: Set up alerts for NodeDoctorHealthy condition changes -6. **Backup**: Include ConfigMap in cluster backup procedures -7. **Updates**: Use rolling updates with maxUnavailable: 25% - a percentage, not 1, so a single crashlooping pod cannot consume the whole budget and stall the rollout at 0/N - -## Support - -- GitHub: https://github.com/supporttools/node-doctor -- Documentation: See `/docs` directory -- Issues: Report at GitHub issues - -## Summary of Components - -**Completed Tasks:** -- ✅ Task #3129: DaemonSet manifest -- ✅ Task #3130: RBAC manifests -- ✅ Task #3131: ConfigMap manifest (embedded in daemonset.yaml) -- ✅ Task #3132: Service manifest (embedded in daemonset.yaml) -- ✅ Dockerfile for container image - -**What's Working:** -- CPU Monitor: 93.1% test coverage -- Memory Monitor: 95.7% test coverage -- Disk Monitor: 93.0% test coverage -- All tests passing -- Binary builds successfully -- Docker image builds (84.8 MB) -- Ready for production deployment - -**Image Registry:** -- Docker Hub: `supporttools/node-doctor:v0.1.0` -- Image: `node-doctor:v0.1.0` (84.8 MB Alpine-based) - -You're ready to deploy to a real cluster! 🚀 diff --git a/docs/development/local-cicd-validation-guide.md b/docs/development/local-cicd-validation-guide.md deleted file mode 100644 index d274d5f..0000000 --- a/docs/development/local-cicd-validation-guide.md +++ /dev/null @@ -1,734 +0,0 @@ -# Local CI/CD Validation Guide - -## Table of Contents -1. [Problem Statement](#problem-statement) -2. [CI/CD Pipeline Architecture](#cicd-pipeline-architecture) -3. [Local Validation Script](#local-validation-script) -4. [Known Issues & Fixes](#known-issues--fixes) -5. [Usage Guide](#usage-guide) -6. [Integration with Git Hooks](#integration-with-git-hooks) -7. [Troubleshooting](#troubleshooting) - ---- - -## Problem Statement - -### The Gap - -Our current local validation script (`scripts/validate-all.sh`) was designed for the original repository structure and checks these components: -- `api/` -- `agent/` -- `monitoring-agent/` -- `ui/` - -However, our **CI/CD pipeline** (`.github/workflows/pipeline-v2.yml`) now tests a completely different set of 6 components: -- `api` (via `cd api && ...`) -- `org-management-controller` (via `./pkg/controller/org-management/...`) -- `probe-controller` (via `./pkg/probe/...`) -- `ingestor` (via `./pkg/ingestor/...`) -- `customer-probe-agent` (via `./cmd/customer-probe-agent/...`) -- `pkg` (via `./pkg/...`) - -### The Cost - -This mismatch means: -- ❌ Issues that will fail CI/CD **cannot be detected locally** -- ❌ Developers push code thinking it's validated -- ❌ CI/CD fails 15-30 minutes later -- ❌ Developer context-switches back to fix the issue -- ❌ Another push, another 15-30 minute wait -- ❌ **Total time wasted per issue: 30-60 minutes** - -### The Solution - -Create a new validation script (`validate-pipeline-local.sh`) that: -- ✅ **Mirrors the exact CI/CD pipeline steps** -- ✅ Tests all 6 components the CI/CD tests -- ✅ Runs the same commands in the same order -- ✅ Provides immediate feedback (5-10 minutes vs 15-30 minutes) -- ✅ Detects 100% of CI/CD issues before pushing - ---- - -## CI/CD Pipeline Architecture - -### Pipeline File -**Location**: `.github/workflows/pipeline-v2.yml` - -### Test Matrix - -The CI/CD pipeline uses GitHub Actions matrix strategy to test 6 components in parallel: - -```yaml -strategy: - matrix: - component: - - api - - org-management-controller - - probe-controller - - ingestor - - customer-probe-agent - - pkg - fail-fast: false - max-parallel: 6 -``` - -### Test Steps for Each Component - -Every component goes through these 5 validation steps: - -#### 1. Go Format Check (gofmt) -Ensures all Go code is properly formatted using `gofmt`. - -**Per-component commands:** -```bash -# api -test -z "$(find api -name '*.go' -not -path '*/vendor/*' -exec gofmt -l {} +)" - -# pkg -test -z "$(find pkg -name '*.go' -not -path '*/vendor/*' -exec gofmt -l {} +)" - -# org-management-controller -test -z "$(find pkg/controller/org-management -name '*.go' -not -path '*/vendor/*' -exec gofmt -l {} +)" - -# probe-controller -test -z "$(find pkg/probe -name '*.go' -not -path '*/vendor/*' -exec gofmt -l {} +)" - -# ingestor -test -z "$(find pkg/ingestor -name '*.go' -not -path '*/vendor/*' -exec gofmt -l {} +)" - -# customer-probe-agent -test -z "$(find cmd/customer-probe-agent -name '*.go' -not -path '*/vendor/*' -exec gofmt -l {} +)" -``` - -**Exit code**: `0` = pass, `1` = files need formatting - -#### 2. Go Vet (static analysis) -Examines Go source code and reports suspicious constructs. - -**Per-component commands:** -```bash -# api -cd api && go vet ./... - -# pkg -go vet ./pkg/... - -# org-management-controller -go vet ./pkg/controller/org-management/... - -# probe-controller -go vet ./pkg/probe/... - -# ingestor -go vet ./pkg/ingestor/... - -# customer-probe-agent (HAS BUG - see Known Issues) -go vet ./cmd/customer-probe-agent/... # ❌ BROKEN - should cd first -``` - -#### 3. Staticcheck (advanced linting) -Staticcheck is a state-of-the-art linter for Go. It uses static analysis to find bugs and performance issues. - -**Installation** (done in CI/CD): -```bash -go install honnef.co/go/tools/cmd/staticcheck@latest -``` - -**Per-component commands:** -```bash -# api -cd api && staticcheck ./... - -# pkg -staticcheck ./pkg/... - -# org-management-controller -staticcheck ./pkg/controller/org-management/... - -# probe-controller -staticcheck ./pkg/probe/... - -# ingestor -staticcheck ./pkg/ingestor/... - -# customer-probe-agent (HAS BUG - see Known Issues) -staticcheck ./cmd/customer-probe-agent/... # ❌ BROKEN - should cd first -``` - -**Suppression syntax**: -```go -//lint:ignore SA1019 Reason for ignoring this specific check -return deprecatedFunction() - -//lint:file-ignore SA1019 AWS SDK deprecations - planned migration -package services -``` - -#### 4. Gosec (security scanning) -Inspects source code for security problems. - -**Installation** (done in CI/CD): -```bash -go install github.com/securego/gosec/v2/cmd/gosec@latest -``` - -**Per-component commands:** -```bash -# api -cd api && gosec -quiet ./... - -# pkg -gosec -quiet ./pkg/... - -# org-management-controller -gosec -quiet ./pkg/controller/org-management/... - -# probe-controller -gosec -quiet ./pkg/probe/... - -# ingestor -gosec -quiet ./pkg/ingestor/... - -# customer-probe-agent (HAS BUG - see Known Issues) -gosec -quiet ./cmd/customer-probe-agent/... # ❌ BROKEN - should cd first -``` - -**Suppression syntax**: -```go -// #nosec G115 - queueSize is always positive, validated at creation -queueCapacity := uint64(pq.queueSize) -``` - -#### 5. Unit Tests with Race Detection -Runs all unit tests with race detector and generates coverage reports. - -**Per-component commands:** -```bash -# api -cd api && go test -v -race -coverprofile=coverage.out -covermode=atomic ./... - -# org-management-controller -go test -v -race -coverprofile=coverage.out -covermode=atomic ./pkg/controller/org-management/... - -# probe-controller -go test -v -race -coverprofile=coverage.out -covermode=atomic ./pkg/probe/... - -# ingestor -go test -v -race -coverprofile=coverage.out -covermode=atomic ./pkg/ingestor/... - -# customer-probe-agent -cd cmd/customer-probe-agent && go test -v -race -coverprofile=coverage.out -covermode=atomic ./... - -# pkg -go test -v -race -coverprofile=coverage.out -covermode=atomic ./pkg/... -``` - -**Flags explained**: -- `-v`: Verbose output -- `-race`: Enable race detector (catches data races) -- `-coverprofile=coverage.out`: Generate coverage report -- `-covermode=atomic`: Coverage mode for race detector compatibility - ---- - -## Local Validation Script - -### Script Location -**File**: `scripts/validate-pipeline-local.sh` - -### Design Principles - -1. **Exact Parity with CI/CD** - - Uses identical commands - - Same execution order - - Same exit codes - -2. **Clear Output** - - Component-by-component progress - - ✓/✗ indicators for each step - - Color-coded results - -3. **Detailed Logging** - - Saves full output to `.validation-logs/` - - Timestamped log files - - Easy debugging - -4. **Fast Failure** - - Stops on first error per component (default) - - `--continue` flag to validate all components - -5. **Component Isolation** - - Each component tested independently - - Can validate single component with `--component` - -### Script Structure - -```bash -#!/bin/bash -# Validates all 6 CI/CD components locally before pushing - -set -e - -# Color codes -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' - -# Components matching CI/CD pipeline -COMPONENTS=( - "api" - "org-management-controller" - "probe-controller" - "ingestor" - "customer-probe-agent" - "pkg" -) - -# Validation functions -validate_go_fmt() { ... } -validate_go_vet() { ... } -validate_staticcheck() { ... } -validate_gosec() { ... } -validate_tests() { ... } - -# Main validation loop -for component in "${COMPONENTS[@]}"; do - echo "Validating $component..." - validate_go_fmt "$component" - validate_go_vet "$component" - validate_staticcheck "$component" - validate_gosec "$component" - validate_tests "$component" -done -``` - -### Output Format - -``` -🔍 Local CI/CD Validation -═══════════════════════════════════════════════════════════════ - -[1/6] Validating api... - ✓ go fmt check - ✓ go vet - ✓ staticcheck - ✓ gosec - ✓ tests (47 passed) - -[2/6] Validating org-management-controller... - ✓ go fmt check - ✓ go vet - ✓ staticcheck - ✓ gosec - ✓ tests (23 passed) - -... - -═══════════════════════════════════════════════════════════════ -✅ All 6 components passed validation! - -Logs saved to: .validation-logs/20251011_123045/ -``` - ---- - -## Known Issues & Fixes - -### Issue 1: customer-probe-agent Module Confusion - -**Problem**: `customer-probe-agent` has its own `go.mod` file (separate Go module), but the CI/CD workflow runs validation commands from the repository root. - -**Symptom**: -```bash -$ go vet ./cmd/customer-probe-agent/... -pattern ./cmd/customer-probe-agent/...: directory prefix cmd/customer-probe-agent -does not contain main module or its selected dependencies -``` - -**Root Cause**: When a directory has its own `go.mod`, it becomes a separate module. Commands like `go vet ./cmd/customer-probe-agent/...` from the root try to vet it as part of the root module, which fails. - -**Fix in Workflow** (`.github/workflows/pipeline-v2.yml`): - -```yaml -# BEFORE (BROKEN): -- name: Run go vet - run: | - case "${{ matrix.component }}" in - customer-probe-agent) - go vet ./cmd/customer-probe-agent/... # ❌ Fails - ;; - esac - -# AFTER (FIXED): -- name: Run go vet - run: | - case "${{ matrix.component }}" in - customer-probe-agent) - cd cmd/customer-probe-agent && go vet ./... # ✅ Works - ;; - esac -``` - -**Apply same fix to**: -- `staticcheck` step (lines 148-150) -- `gosec` step (lines 172-174) - -**Tests are already correct**: They already `cd` into the directory before running. - -### Issue 2: Type Safety - uint vs uint64 - -**Problem**: Some fields use `uint` for organization IDs but NATS functions expect `uint64`. - -**Symptom**: -```go -// pkg/ingestor/server/server.go:331 -if uint64(conn.OrganizationID) != organizationID -``` - -The fact that we need `uint64()` cast suggests a type mismatch. - -**Investigation**: -```bash -# Find all OrganizationID field definitions -grep -r "OrganizationID.*uint" pkg/ingestor/server/ - -# Check if it should be uint64 everywhere -grep -r "func.*organizationID uint64" pkg/ -``` - -**Fix**: Ensure OrganizationID is consistently `uint64` throughout ingestor package. - -### Issue 3: Staticcheck False Positives - -**Problem**: Staticcheck sometimes reports false positives for format strings. - -**Symptom**: -``` -pkg/notification/analytics/reports.go:392:21: couldn't parse format string (SA5009) -``` - -**Fix**: Use `//lint:ignore SA5009` to suppress: -```go -//lint:ignore SA5009 staticcheck false positive - format string is valid with %% for literal % -return fmt.Sprintf(html, ...) -``` - -**File-level suppression**: -```go -//lint:file-ignore SA1019 AWS SDK deprecations - planned migration - -package services -``` - ---- - -## Usage Guide - -### Basic Usage - -#### Validate All Components -```bash -./scripts/validate-pipeline-local.sh -``` - -This is the **recommended command** before every push. It validates all 6 components exactly as CI/CD will. - -**Time**: ~5-10 minutes (depends on test count) - -#### Validate Single Component -```bash -./scripts/validate-pipeline-local.sh --component ingestor -``` - -Use this when you've only changed code in one component and want quick feedback. - -**Time**: ~1-2 minutes per component - -#### Quick Mode (Skip Heavy Checks) -```bash -./scripts/validate-pipeline-local.sh --quick -``` - -Skips expensive operations like full test suites. Only runs fmt, vet, staticcheck, gosec. - -**Time**: ~2-3 minutes - -**When to use**: Pre-commit checks, quick verification - -#### Continue on Failure -```bash -./scripts/validate-pipeline-local.sh --continue -``` - -Validates all components even if some fail. Useful for getting complete picture of issues. - -**Default behavior**: Stops at first failure per component - -### Makefile Integration - -#### Add to Makefile - -```makefile -# ============================================================================= -# Local CI/CD Validation -# ============================================================================= - -.PHONY: validate-pipeline-local validate-component - -# Validate all components (mirrors CI/CD pipeline) -validate-pipeline-local: - @echo "Running full pipeline validation locally..." - @./scripts/validate-pipeline-local.sh - -# Validate single component -validate-component: - @echo "Validating component: $(COMPONENT)..." - @./scripts/validate-pipeline-local.sh --component $(COMPONENT) - -# Quick validation (skip expensive tests) -validate-quick: - @echo "Running quick validation..." - @./scripts/validate-pipeline-local.sh --quick -``` - -#### Usage via Make - -```bash -# Full validation -make validate-pipeline-local - -# Single component -make validate-component COMPONENT=ingestor - -# Quick mode -make validate-quick -``` - -### Reading Validation Logs - -Logs are saved to `.validation-logs/TIMESTAMP/` with this structure: - -``` -.validation-logs/ -└── 20251011_123045/ - ├── api_fmt.log - ├── api_vet.log - ├── api_staticcheck.log - ├── api_gosec.log - ├── api_tests.log - ├── ingestor_fmt.log - ├── ingestor_vet.log - └── ... -``` - -**To view a specific failure**: -```bash -# List recent validation runs -ls -lt .validation-logs/ - -# View specific log -cat .validation-logs/20251011_123045/ingestor_staticcheck.log - -# Search for errors -grep -r "error:" .validation-logs/20251011_123045/ -``` - -**Log cleanup**: Script automatically keeps last 10 runs, deletes older logs. - ---- - -## Integration with Git Hooks - -### Pre-Push Hook - -**File**: `.githooks/pre-push` - -```bash -#!/bin/bash -# Validate code before pushing to remote - -echo "🔍 Running local CI/CD validation before push..." -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - -# Run quick validation (skip expensive full test suites) -./scripts/validate-pipeline-local.sh --quick - -# Check exit code -if [ $? -ne 0 ]; then - echo "" - echo "❌ Validation failed! Push aborted." - echo "" - echo "To fix:" - echo " 1. Review errors above" - echo " 2. Fix issues and commit" - echo " 3. Push again" - echo "" - echo "To skip validation (NOT recommended):" - echo " git push --no-verify" - exit 1 -fi - -echo "" -echo "✅ Validation passed! Proceeding with push..." -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -exit 0 -``` - -### Enable Git Hooks - -```bash -# Configure git to use .githooks/ directory -git config core.hooksPath .githooks - -# Make hook executable -chmod +x .githooks/pre-push -``` - -### Override Hook (Emergency) - -If you **absolutely must** push without validation (production emergency, etc.): - -```bash -git push --no-verify -``` - -âš ī¸ **Warning**: Use sparingly! This bypasses all validation and CI/CD will likely fail. - ---- - -## Troubleshooting - -### Issue: "staticcheck: command not found" - -**Cause**: staticcheck not installed - -**Fix**: -```bash -go install honnef.co/go/tools/cmd/staticcheck@latest -``` - -### Issue: "gosec: command not found" - -**Cause**: gosec not installed - -**Fix**: -```bash -go install github.com/securego/gosec/v2/cmd/gosec@latest -``` - -### Issue: Tests timeout - -**Cause**: Some tests take longer than default timeout - -**Fix**: Increase timeout in test command: -```bash -go test -v -race -timeout=10m -coverprofile=coverage.out ./... -``` - -### Issue: "permission denied" when running script - -**Cause**: Script not executable - -**Fix**: -```bash -chmod +x scripts/validate-pipeline-local.sh -``` - -### Issue: customer-probe-agent validation fails - -**Cause**: Workflow bug - see [Known Issues](#known-issues--fixes) - -**Temporary Workaround**: Skip customer-probe-agent until workflow fixed: -```bash -./scripts/validate-pipeline-local.sh --skip customer-probe-agent -``` - -### Issue: Log directory fills up disk space - -**Cause**: Many validation runs, logs not cleaned up - -**Fix**: -```bash -# Manual cleanup (keeps last 10 runs) -ls -t .validation-logs | tail -n +11 | xargs -I {} rm -rf .validation-logs/{} - -# Or delete all logs -rm -rf .validation-logs -``` - -### Issue: Different results locally vs CI/CD - -**Possible causes**: -1. **Go version mismatch**: CI/CD uses Go 1.24 - check yours: - ```bash - go version # Should be 1.24.x - ``` - -2. **Stale dependencies**: Update go.mod: - ```bash - go mod tidy - go mod download - ``` - -3. **Uncommitted changes**: CI/CD tests committed code: - ```bash - git status # Check for uncommitted changes - ``` - -4. **Cache issues**: Clear Go cache: - ```bash - go clean -cache -testcache -modcache - ``` - -### Issue: Validation takes too long - -**Solutions**: - -1. **Use quick mode** for pre-commit: - ```bash - ./scripts/validate-pipeline-local.sh --quick - ``` - -2. **Validate only changed components**: - ```bash - # If you only changed ingestor - ./scripts/validate-pipeline-local.sh --component ingestor - ``` - -3. **Run in parallel** (future enhancement): - ```bash - ./scripts/validate-pipeline-local.sh --parallel - ``` - ---- - -## Appendix: Component Coverage Matrix - -| Component | fmt | vet | staticcheck | gosec | tests | Path | -|-----------|-----|-----|-------------|-------|-------|------| -| api | ✓ | ✓ | ✓ | ✓ | ✓ | `api/` | -| org-management-controller | ✓ | ✓ | ✓ | ✓ | ✓ | `pkg/controller/org-management/` | -| probe-controller | ✓ | ✓ | ✓ | ✓ | ✓ | `pkg/probe/` | -| ingestor | ✓ | ✓ | ✓ | ✓ | ✓ | `pkg/ingestor/` | -| customer-probe-agent | ✓ | âš ī¸ | âš ī¸ | âš ī¸ | ✓ | `cmd/customer-probe-agent/` | -| pkg | ✓ | ✓ | ✓ | ✓ | ✓ | `pkg/` | - -âš ī¸ = Has workflow bug, needs `cd` before command - ---- - -## Related Documentation - -- [Release Process](../release-process.md) - What the tag-triggered pipeline actually publishes -- [GitHub Actions Workflows](../../.github/workflows/README.md) - Job-by-job CI reference -- [Task Execution Workflow](./task-execution-workflow.md) -- `make help` - Authoritative list of Makefile targets for this repo - ---- - -**Last Updated**: 2025-10-11 -**Maintained By**: Platform Team -**Questions**: See #engineering-platform on Slack diff --git a/docs/development/master-task-management.md b/docs/development/master-task-management.md deleted file mode 100644 index 461897e..0000000 --- a/docs/development/master-task-management.md +++ /dev/null @@ -1,275 +0,0 @@ -# Master Task Management System - -The {{PROJECT_NAME}} project follows a prioritized task management system based on the master design documents. ALL development work must follow these rules for controlled, systematic progress. - -## Task Reference Documents - -- **Master Design Document**: `/master-design-doc.md` - System architecture and service dependencies -- **Task Management System**: TaskForge project management system with 546 tasks organized in hierarchical structure - - Tasks organized: Project → Features → Tasks (11 features, 546 total tasks) - - Accessible via MCP tools: `mcp__taskforge__getTasks`, `mcp__taskforge__createTask`, `mcp__taskforge__updateTask` - - See [TaskForge Usage Guide](taskforge-usage-guide.md) for complete documentation - - Migration from todo-mcp documented in `taskforge-migration-guide.md` - -## Priority System (MUST FOLLOW) - -### Priority 1 - Foundation Services (IMPLEMENT FIRST) - -- org-management-controller: Core orchestration system -- notification-service: Required by all alert systems -- Database consolidation: Multi-tenant and single-tenant patterns -- **BLOCKER**: No P2+ work until P1 foundation is complete - -### Priority 2 - Core Monitoring (BUSINESS CRITICAL) - -- alert-controller: Internal monitoring alerts -- health-controller: Platform health monitoring -- linux-agent: Enhanced data collection -- **DEPENDENCY**: Requires P1 foundation services - -### Priority 3 - Advanced Features (VALUE-ADD) - -- probe-controller, tag-controller, job-controller, scheduler-controller -- **DEPENDENCY**: Requires P1 + P2 core services - -### Priority 4 - Business Operations (REVENUE) - -- billing-controller, reporting-controller -- **DEPENDENCY**: Requires monitoring infrastructure from P1-P3 - -### Priority 5 - Platform Polish (UX) - -- Web-UI critical fixes (P5A) then enhancements (P5B) -- **PARALLEL**: Can work on P5A critical fixes alongside P1-P2 - -## Task Execution Rules - -### Before Starting ANY Task - -Use TaskForge to select the next task: - -```javascript -// 1. List tasks for current priority using TaskForge -mcp__taskforge__getTasks({ - projectId: 1, - featureId: 1, // P1-Foundation-Services - status: "todo", - priority: "high" -}); - -// 2. Check for blocked tasks -mcp__taskforge__getBlockedTasks({ - projectId: 1 -}); - -// 3. Use TodoWrite tool to track session progress -``` - -**Task Selection Rules:** - -1. **NEVER skip priorities** - Complete P1 before P2, P2 before P3, etc. -2. **RESPECT dependencies** - Check [DEPENDS ON: service] annotations -3. **ONE service at a time** - Complete service implementation before starting next -4. **CRITICAL first** - Within priority, do CRITICAL before HIGH before MEDIUM - -### Task Status Tracking (MANDATORY) - -Using TaskForge (PostgreSQL-backed project management system): - -```javascript -// 1. Start a task - set status to "in_progress" -mcp__taskforge__updateTask({ - taskId: 123, - status: "in_progress" -}); - -// 2. Complete a task -mcp__taskforge__updateTask({ - taskId: 123, - status: "done", - actualHours: 6.5 -}); - -// 3. Block a task (for dependencies or issues) -mcp__taskforge__updateTask({ - taskId: 123, - status: "blocked" -}); -// Or use mcp__taskforge__setDependency to track blocking tasks - -// 4. Delete irrelevant tasks (soft delete) -mcp__taskforge__deleteTask({ - taskId: 123 -}); - -// 5. Use TodoWrite tool for session-level progress tracking -``` - -## Implementation Standards - -### For Backend Services (Go) - -```bash -# ALWAYS follow this sequence: -# 1. Create service directory structure -mkdir -p controllers/[service-name]/{cmd,pkg/{handlers,models,services}} - -# 2. Implement core functionality with proper patterns -# - Single-function-per-file handlers -# - Comprehensive logging (Trace/Debug) -# - Complete Swagger documentation -# - Database models in BOTH locations (ConnectDB.go AND migrator.go) - -# 3. Test locally before committing -make build-api-local -make test-api-local -go test -v ./controllers/[service-name]/... - -# 4. Update TaskForge with completion -``` - -### For Multi-Service Integration - -```bash -# ALWAYS verify service dependencies: -# 1. notification-service running before alert-controller -# 2. org-management-controller before single-tenant services -# 3. Database schema supports both single/multi-tenant patterns - -# Test integration points: -# - API endpoints responding -# - Database connections working -# - Service-to-service communication functional -``` - -### Deployment Integration - -```bash -# Each service MUST include: -# 1. Dockerfile in service directory -# 2. Helm templates in helm/server/templates// -# 3. Helm chart integration with parent chart (values.yaml) -# 4. CI/CD pipeline integration - -# Test deployment locally: -# Build → Test → Deploy to dev → Validate -``` - -## Controlled Development Workflow - -### Phase 1: Foundation (P1 Tasks Only) - -1. org-management-controller core API -2. notification-service with multi-tenant support -3. Database schema consolidation -4. **GATE**: Foundation integration testing before P2 - -### Phase 2: Core Monitoring (P2 Tasks) - -1. alert-controller with notification integration -2. health-controller with org-management integration -3. linux-agent enhancements -4. **GATE**: End-to-end monitoring workflow before P3 - -### Phase 3: Advanced Features (P3 Tasks) - -1. probe-controller, tag-controller implementation -2. job-controller and scheduler-controller -3. **GATE**: Full platform feature testing before P4 - -### Phase 4: Business Operations (P4 Tasks) - -1. billing-controller with notification integration -2. reporting-controller with data aggregation -3. **GATE**: Business workflow testing before P5B - -### Phase 5: Platform Polish - -1. P5A: Critical web-UI fixes (parallel with P1-P2) -2. P5B: Web-UI enhancements (after P1-P4 complete) - -## Quality Gates (MANDATORY) - -### Before Moving to Next Priority - -```bash -# 1. All tasks in current priority marked [COMPLETED] -# 2. Integration tests passing for completed services -# 3. Local deployment successful -# 4. No critical bugs or blockers -# 5. Documentation updated - -# Validation commands: -make build-all-local -make test-local -./scripts/validate-all.sh -``` - -### Service Completion Checklist - -- [ ] Core functionality implemented and tested -- [ ] Database models added to both locations -- [ ] API endpoints with Swagger docs -- [ ] Service integration tests passing -- [ ] Helm chart and deployment configs -- [ ] TaskForge updated with completion -- [ ] No critical or high-priority bugs -- [ ] Ready for next service dependencies - -## Emergency Procedures - -### If Blocked on Current Priority - -1. Document blocker in TaskForge -2. Create specific blocker resolution tasks -3. **NEVER jump to next priority** - resolve blocker first -4. Consider parallel work on P5A critical fixes only - -### If Service Integration Fails - -1. Roll back to last working state -2. Create integration debugging tasks -3. Fix integration before continuing with service -4. Update master design doc if architecture changes needed - -## Task Development Workflow - -**ALL development tasks MUST follow the comprehensive workflow documented in `task-execution-workflow.md`.** - -This mandatory 8-step workflow includes: - -1. **Task Selection & Research** - Mandatory research phase before any coding -2. **Design & Planning** - Present comprehensive implementation plan -3. **Approval Gate** - Wait for explicit approval before coding -4. **Implementation** - Code with strict adherence to standards -5. **Quality Assurance** - Build verification, QA review, and testing -6. **Deployment** - Mandatory deployment with `make bump` -7. **Devils Advocate Verification** - Final validation of task completion -8. **Task Completion** - Update TaskForge and commit changes - -**Key Requirements:** -- ONE task at a time - never work on multiple tasks simultaneously -- RESEARCH before coding - no exceptions -- BUILD verification required before QA review -- QA validation required before deployment -- DEPLOYMENT mandatory before completion -- DEVILS ADVOCATE approval required before marking complete -- Multi-repository updates must be completed together - -**Multi-Repository Considerations:** - -When tasks affect multiple repositories ({{PROJECT_NAME}}, go-sdk, linux-agent, web-ui, cli): -- Update repositories in dependency order: API → SDK → Agent/UI/CLI -- Build and test ALL modified repositories -- Commit and tag ALL repositories appropriately -- NEVER leave repositories in inconsistent state - -See [Task Execution Workflow](task-execution-workflow.md) for complete details, examples, and enforcement rules. - -## See Also - -- [TaskForge Usage Guide](taskforge-usage-guide.md) - Complete TaskForge MCP tools documentation -- [Task Execution Workflow](task-execution-workflow.md) - Mandatory 8-step development workflow -- [go-sdk Synchronization Guide](go-sdk-synchronization-guide.md) - SDK update requirements -- [Web UI-API Synchronization](web-ui-api-synchronization.md) - Frontend sync procedures -- [Helm Chart Standards](helm-chart-standards.md) - Deployment configuration rules diff --git a/docs/development/task-execution-workflow.md b/docs/development/task-execution-workflow.md deleted file mode 100644 index 6f92549..0000000 --- a/docs/development/task-execution-workflow.md +++ /dev/null @@ -1,1018 +0,0 @@ -# Task Execution Workflow - -**⭐ Core Framework Document** - This workflow defines the mandatory process for executing all development tasks in the Node Doctor project. - -## 📝 About This Document - -This workflow is adapted from production-tested practices and customized for Node Doctor - a Kubernetes DaemonSet monitoring tool with auto-remediation capabilities. - -**Node Doctor Specifics:** -- Single repository, single binary architecture -- Go 1.21+ required -- Kubernetes DaemonSet deployment model -- Monitor/Remediator plugin system -- Direct node access with elevated privileges -- Safety-first remediation approach - -## Available Resources - -### Task Management System - -The project uses PostgreSQL-backed TaskForge for persistent task management: -- **Tasks organized by project** (P1-P5 priority levels, 14 features) -- **Priority-based**: Urgent (đŸ”Ĩ), High (🔴), Medium (🟡), Low (đŸŸĸ) -- **Persistent across sessions**: All Claude Code instances share the same task state -- **MCP Tools**: `mcp__taskforge__getTasks`, `mcp__taskforge__updateTask`, `mcp__taskforge__getBlockedTasks` -- See [TaskForge Usage Guide](taskforge-usage-guide.md) and [Master Task Management](master-task-management.md) - -### Development Environment - -- Local Kubernetes cluster access available via configured kubeconfig (see Makefile KUBECONFIG_* variables) -- DaemonSet testing environment for node-level operations -- Use this environment to: - - Test monitor implementations with real node metrics - - Validate remediator safety mechanisms - - Run integration tests with Kubernetes API - - Verify DaemonSet deployment and RBAC configurations - - Debug monitor/remediator interactions - - Test exporter functionality (Prometheus metrics, Kubernetes events) -- Always verify environment context before running operations requiring elevated privileges -- Use kube-system namespace for DaemonSet deployments - -### Single Repository Structure - -Node Doctor uses a single repository with the following structure: - -``` -node-doctor/ -├── cmd/node-doctor/ # Main entry point -├── pkg/ -│ ├── monitors/ # Monitor implementations -│ ├── remediators/ # Auto-remediation implementations -│ ├── exporters/ # Prometheus/Kubernetes exporters -│ ├── problemdetector/ # Core orchestration -│ ├── config/ # Configuration types -│ └── utils/ # Shared utilities -├── deploy/ # Kubernetes manifests -│ ├── daemonset.yaml # DaemonSet configuration -│ ├── rbac.yaml # ServiceAccount/ClusterRole -│ └── configmap.yaml # Monitor/remediator configs -├── docs/ # Documentation -└── tests/ # Integration/E2E tests -``` - ---- - -## Workflow Steps - -### 1. Receive and Initiate Task - -- I will provide you with a specific task from TaskForge or by name/description -- Retrieve tasks using `mcp__taskforge__getTasks` with appropriate filters (featureId, priority, status) -- Verify the task priority level and ensure no higher priority tasks are pending -- Check for task dependencies (blockedByTaskId field) -- If task has unmet dependencies or is already completed, report this and await further instruction -- Update task status to "in_progress" using `mcp__taskforge__updateTask` when starting work - -**TodoWrite Integration (MANDATORY for session tracking)**: -``` -Use TodoWrite tool to create initial task breakdown: -- Research and design phase -- Implementation phase -- Build verification -- QA review -- Testing -- Deployment -- Devils advocate verification -- Task completion - -Mark first item as "in_progress" immediately. - -NOTE: TodoWrite tracks workflow steps WITHIN the current session. -TaskForge tracks project-level task completion ACROSS all sessions. -Both are mandatory - use TodoWrite for workflow progress, TaskForge for task completion. -``` - -### 2. Research & Design Phase (MANDATORY - NO CODING WITHOUT THIS) - -**Thoroughly research the task requirements:** -- Review architecture.md for system design specifications -- Review monitors.md for monitor specifications and patterns -- Review remediation.md for safety mechanisms and remediator patterns -- Identify integration points with Problem Detector orchestrator -- Review existing monitors/remediators for similar patterns -- Understand safety requirements and failure modes - -**Present a comprehensive implementation plan that includes:** - -#### Task Overview -Clear summary aligned with Node Doctor architecture (monitors, remediators, exporters) - -#### Design Approach -Technical strategy following Node Doctor's established patterns: -- **Monitor Pattern**: Implement Monitor interface with Start()/Stop() methods -- **Channel-Based Communication**: Use Go channels for status updates to Problem Detector -- **Remediator Pattern**: Implement Remediator interface with safety mechanisms -- **Safety First**: Cooldown periods, attempt limiting, circuit breakers, rate limiting -- **Configuration-Driven**: YAML/JSON config with validation -- **Kubernetes Integration**: Use client-go for API interactions, respect RBAC boundaries - -#### Files Affected -Complete list with brief explanation: -- **Monitor implementations**: `pkg/monitors/[monitor-name].go`, `pkg/monitors/[monitor-name]_test.go` -- **Remediator implementations**: `pkg/remediators/[remediator-name].go`, `pkg/remediators/[remediator-name]_test.go` -- **Exporter updates**: `pkg/exporters/prometheus.go`, `pkg/exporters/kubernetes.go` -- **Configuration types**: `pkg/config/types.go`, `pkg/config/validation.go` -- **Problem Detector integration**: `pkg/problemdetector/orchestrator.go` -- **Deployment manifests**: `deploy/daemonset.yaml`, `deploy/rbac.yaml`, `deploy/configmap.yaml` -- **Documentation**: `docs/monitors.md`, `docs/remediation.md` - -#### Dependencies -Prerequisites and blockers: -- Node access requirements (hostPath, hostNetwork, hostPID) -- Kubernetes permissions (RBAC for ServiceAccount) -- System dependencies (systemd, network tools, disk utilities) -- Go packages (client-go, prometheus/client_golang, etc.) -- Safety mechanism implementations (circuit breaker, rate limiter) - -#### Success Criteria -Specific, measurable conditions: -- Project builds successfully: `make build-node-doctor-local` -- All tests pass: `make test-node-doctor-local` -- Validation passes: `make validate-pipeline-local` -- QA agent verification passes -- DaemonSet deploys successfully: `kubectl apply -f deploy/` -- Monitor collects data and reports problems correctly -- Remediator respects safety limits and cooldowns -- Exporters publish metrics/events correctly -- Devils advocate verification passes - -#### Risks & Mitigations -Potential issues and fallbacks: -- **Node access risks**: Privileged operations, validate inputs rigorously -- **Remediation risks**: Implement circuit breakers, cooldown periods, attempt limits -- **Performance risks**: Resource limits, efficient polling, async operations -- **Failure modes**: Graceful degradation, proper error handling, monitoring fallbacks - -**TodoWrite Update**: Mark "Research and design phase" as completed, mark "Implementation phase" as in_progress. - -### 3. Wait for Approval (MANDATORY GATE) - -**You must pause here and await explicit instruction.** Do not proceed to coding without approval. - -Based on feedback: -- `✅ Approve`: Proceed immediately to Step 4 (Implementation) -- `🔄 Request changes`: Revise plan according to feedback and re-present -- `â­ī¸ Skip`: Mark task as skipped and await new assignment - -**TodoWrite Update**: When approval received, mark "Implementation phase" as in_progress. - -### 4. Implementation Phase - -Implement the approved design with strict adherence to: - -#### Go Version -- **ALWAYS use Go 1.21+** - verify go.mod specifies "go 1.21" or higher - -#### Monitor Implementation Pattern -```go -package monitors - -import ( - "context" - "time" - "github.com/supporttools/node-doctor/pkg/problemdetector" -) - -// CPUMonitor implements the Monitor interface -type CPUMonitor struct { - config *CPUMonitorConfig - stopCh chan struct{} - statusCh chan *problemdetector.Status -} - -// NewCPUMonitor creates a new CPU monitor instance -func NewCPUMonitor(config *CPUMonitorConfig) (*CPUMonitor, error) { - // Validate config - if err := config.Validate(); err != nil { - return nil, fmt.Errorf("invalid config: %w", err) - } - - return &CPUMonitor{ - config: config, - stopCh: make(chan struct{}), - statusCh: make(chan *problemdetector.Status, 10), - }, nil -} - -// Start begins monitoring (runs in goroutine) -func (m *CPUMonitor) Start() (<-chan *problemdetector.Status, error) { - go m.monitorLoop() - return m.statusCh, nil -} - -// Stop halts monitoring -func (m *CPUMonitor) Stop() { - close(m.stopCh) -} - -func (m *CPUMonitor) monitorLoop() { - ticker := time.NewTicker(m.config.PollInterval) - defer ticker.Stop() - defer close(m.statusCh) - - for { - select { - case <-m.stopCh: - return - case <-ticker.C: - status := m.checkCPU() - if status != nil { - m.statusCh <- status - } - } - } -} -``` - -#### Remediator Implementation Pattern -```go -package remediators - -import ( - "context" - "time" - "github.com/supporttools/node-doctor/pkg/problemdetector" -) - -// SystemdRemediator handles systemd service issues -type SystemdRemediator struct { - config *SystemdRemediatorConfig - circuitBreaker *CircuitBreaker - rateLimiter *RateLimiter - lastRemediation map[string]time.Time -} - -// CanRemediate checks if this remediator handles the problem -func (r *SystemdRemediator) CanRemediate(problem problemdetector.Problem) bool { - return problem.Type == "systemd-service-failed" -} - -// Remediate attempts to fix the problem with safety checks -func (r *SystemdRemediator) Remediate(ctx context.Context, problem problemdetector.Problem) error { - // Check circuit breaker - if r.circuitBreaker.IsOpen() { - return fmt.Errorf("circuit breaker open, too many failures") - } - - // Check rate limiter - if !r.rateLimiter.Allow() { - return fmt.Errorf("rate limit exceeded") - } - - // Check cooldown period - if lastTime, ok := r.lastRemediation[problem.Resource]; ok { - if time.Since(lastTime) < r.GetCooldown() { - return fmt.Errorf("cooldown period not elapsed") - } - } - - // Perform remediation - err := r.restartService(ctx, problem.Resource) - - // Update circuit breaker and tracking - if err != nil { - r.circuitBreaker.RecordFailure() - return err - } - - r.circuitBreaker.RecordSuccess() - r.lastRemediation[problem.Resource] = time.Now() - return nil -} - -// GetCooldown returns the cooldown period for this remediator -func (r *SystemdRemediator) GetCooldown() time.Duration { - return r.config.CooldownPeriod -} -``` - -#### Configuration Pattern -```go -package config - -// NodeDoctorConfig is the root configuration -type NodeDoctorConfig struct { - Monitors []MonitorConfig `json:"monitors" yaml:"monitors"` - Remediators []RemediatorConfig `json:"remediators" yaml:"remediators"` - Exporters ExporterConfig `json:"exporters" yaml:"exporters"` - Global GlobalConfig `json:"global" yaml:"global"` -} - -// Validate performs comprehensive config validation -func (c *NodeDoctorConfig) Validate() error { - if len(c.Monitors) == 0 { - return fmt.Errorf("at least one monitor required") - } - - for i, monitor := range c.Monitors { - if err := monitor.Validate(); err != nil { - return fmt.Errorf("monitor[%d]: %w", i, err) - } - } - - // Validate remediators, exporters, global config... - return nil -} -``` - -#### DaemonSet Deployment -```yaml -# deploy/daemonset.yaml -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: node-doctor - namespace: kube-system -spec: - selector: - matchLabels: - app: node-doctor - template: - metadata: - labels: - app: node-doctor - spec: - serviceAccountName: node-doctor - hostNetwork: true # Access host networking - hostPID: true # Access host processes - containers: - - name: node-doctor - image: supporttools/node-doctor:latest - securityContext: - privileged: true # Required for remediation - volumeMounts: - - name: config - mountPath: /etc/node-doctor - - name: host-root - mountPath: /host - readOnly: true - volumes: - - name: config - configMap: - name: node-doctor-config - - name: host-root - hostPath: - path: / -``` - -#### RBAC Configuration -```yaml -# deploy/rbac.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: node-doctor - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: node-doctor -rules: -- apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: ["events"] - verbs: ["create", "patch"] -- apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: node-doctor -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: node-doctor -subjects: -- kind: ServiceAccount - name: node-doctor - namespace: kube-system -``` - -#### Repository Cleanliness -- Keep repository root clean (only essential config/docs) -- Proper file placement in pkg/, cmd/, deploy/ directories -- No generated files in repository (binaries, coverage files) -- Run `make validate-pipeline-local` to verify standards - -#### Write Tests as You Code (MANDATORY) - -**Critical Rule: NEVER write code without corresponding tests. Tests are not optional or an afterthought.** - -**Unit Tests (Go):** -- For EVERY monitor/remediator, write corresponding tests in `*_test.go` files -- Write tests BEFORE or ALONGSIDE implementation - not after -- Follow Go testing conventions with table-driven tests -- Minimum coverage requirements: - - **Happy path**: Valid inputs with expected behavior - - **Edge cases**: Boundary conditions, empty values, maximum limits - - **Error cases**: Error handling, validation failures, recovery - - **Safety mechanisms**: Circuit breaker, rate limiter, cooldown enforcement - - **Concurrent operations**: Race conditions and thread safety -- Run tests incrementally: `go test -v ./pkg/monitors/...` -- Maintain >80% code coverage -- Use mocks for system calls (systemd, disk ops, network) - -**Integration Tests:** -- Test monitor integration with Problem Detector -- Test remediator safety mechanisms under load -- Test exporter publishing to Prometheus/Kubernetes -- Verify channel communication patterns -- Place in `*_integration_test.go` files -- Run with build tags: `go test -tags=integration -v ./...` - -**End-to-End (E2E) Tests:** -- Required for complete monitoring workflows -- Test scenarios: - - Monitor detects problem → Problem Detector receives → Exporter publishes - - Problem detected → Remediator attempts fix → Safety limits enforced - - DaemonSet deployment → Configuration loading → Monitoring active -- Use realistic test scenarios with Docker/kind clusters -- Place in `tests/e2e/` directory -- Document test setup and requirements - -**Testing Workflow Integration:** -```bash -# 1. Write test first (TDD) or alongside code -# 2. Implement feature -# 3. Run unit tests -go test -v ./pkg/monitors/... -go test -v ./pkg/remediators/... - -# 4. Run integration tests -go test -tags=integration -v ./... - -# 5. Run E2E tests -go test -tags=e2e -v ./tests/e2e/... - -# 6. Check coverage -go test -cover -coverprofile=coverage.out ./... -go tool cover -html=coverage.out -``` - -**Quality Gates for Testing:** -- [ ] Every new monitor/remediator has unit tests -- [ ] All edge cases and error paths covered -- [ ] Integration tests pass for Problem Detector interaction -- [ ] E2E tests pass for complete workflows -- [ ] Code coverage >80% (target: 85%+) -- [ ] No flaky tests - all deterministic -- [ ] Tests run in reasonable time (<2min for unit, <10min for full suite) - -**If implementation reveals issues with approved plan, stop and report immediately** - -**TodoWrite Update**: Mark "Implementation phase" as completed, mark "Build verification" as in_progress. - -### 5. Quality Assurance Cycle (MANDATORY) - -#### Step 5.1: Build Verification (BLOCKER) - -```bash -# Build node-doctor binary -make build-node-doctor-local - -# Verify binary exists and is executable -./bin/node-doctor --version -``` - -- **MUST PASS**: Zero compilation errors, zero linting errors -- If build fails, fix immediately before proceeding -- Build must complete successfully before moving to QA - -**TodoWrite Update**: Mark "Build verification" as completed, mark "QA review" as in_progress. - -#### Step 5.2: QA Agent Review (BLOCKER) - -- Invoke QA agent using Task tool with qa-engineer subagent -- Provide QA agent with: - - Changed files and context - - Original task requirements - - Success criteria - - Architecture document references (architecture.md, monitors.md, remediation.md) - -- QA agent will verify: - - Code quality and Go standards compliance - - Monitor interface implementation correct - - Remediator safety mechanisms implemented - - Configuration validation comprehensive - - Channel-based communication patterns correct - - Proper error handling and logging - - DaemonSet/RBAC configurations secure - - Integration with Problem Detector correct - - Security considerations (privileged access, input validation) - - Test coverage adequate (>80%) - -- **BLOCKER**: Address ALL QA findings before proceeding - -**TodoWrite Update**: Mark "QA review" as completed, mark "Testing" as in_progress. - -#### Step 5.3: Run Full Test Suite - -```bash -# Component tests -make test-node-doctor-local - -# Full validation (mirrors CI/CD pipeline exactly) -make validate-pipeline-local - -# Quick validation (format, vet, lint only - no tests) -make validate-quick - -# Race detector -go test -race -v ./... -``` - -**Validation checks:** -- Code formatting (gofmt) -- Static analysis (go vet) -- Advanced linting (staticcheck) -- Security scanning (gosec) -- Tests with race detection -- Coverage analysis - -- Report test coverage and results -- **If tests fail**: Fix issues, re-run QA review if needed, re-test -- Test results must include: files tested, pass/fail counts, coverage % - -See [Validation Quick Reference](validation-quick-reference.md) for troubleshooting - -**TodoWrite Update**: Mark "Testing" as completed, mark "Deployment" as in_progress. - -### 6. Deployment (MANDATORY) - -For Node Doctor, deployment means applying Kubernetes manifests to the cluster: - -```bash -# Build and push container image -make build-image-node-doctor VERSION=v$(date +%s) -make push-image-node-doctor VERSION=v$(date +%s) - -# Update DaemonSet manifest with new image version -# (Or use kustomize/helm for version management) - -# Apply Kubernetes resources -kubectl apply -f deploy/rbac.yaml -kubectl apply -f deploy/configmap.yaml -kubectl apply -f deploy/daemonset.yaml - -# Verify DaemonSet rollout -kubectl rollout status daemonset/node-doctor -n kube-system - -# Check pod status on all nodes -kubectl get pods -n kube-system -l app=node-doctor -o wide - -# Verify monitoring is active (check logs) -kubectl logs -n kube-system -l app=node-doctor --tail=50 -``` - -#### Deployment Success Criteria -- DaemonSet manifest applies without errors -- Pods created on all nodes (or selected nodes if nodeSelector used) -- All pods reach Running state -- Health checks pass (if implemented) -- Monitors start and begin reporting status -- Exporters publish metrics to Prometheus -- Kubernetes events created successfully -- No CrashLoopBackOff or ImagePullBackOff errors - -**If deployment fails, enter fix cycle (fix → build → QA → deploy)** - -**TodoWrite Update**: Mark "Deployment" as completed, mark "Devils advocate verification" as in_progress. - -### 7. Devils Advocate Verification (MANDATORY) - -- Use Task tool with devils-advocate-qa subagent -- Provide complete context: - - Original task specification - - Implementation details - - Test results (coverage, pass/fail) - - Deployment status - - Monitor/remediator behavior observations - -- Devils advocate will verify: - - Original request was fully fulfilled - - Business requirements met - - No edge cases missed - - Implementation matches task specification - - All success criteria satisfied - - Safety mechanisms working correctly - - No security issues introduced - - DaemonSet deployment stable - -#### If issues found: Enter Quality Cycle -- Fix identified issues -- Build verification (`make build-node-doctor-local`) -- QA agent re-review -- Re-test (`make test-node-doctor-local`) -- Re-deploy (kubectl apply) -- Devils advocate re-verification -- **STAY IN CYCLE** until fully resolved - -**TodoWrite Update**: When devils advocate approves, mark "Devils advocate verification" as completed, mark "Task completion" as in_progress. - -### 8. Task Completion (ONLY AFTER DEVILS ADVOCATE APPROVAL) - -#### Provide completion summary: -- **Files changed/created/deleted**: - - Monitors: `pkg/monitors/[name].go` and tests - - Remediators: `pkg/remediators/[name].go` and tests - - Configuration: `pkg/config/types.go` updates - - Deployment: `deploy/*.yaml` manifests - - Documentation: `docs/*.md` updates - -- **Tests written/updated**: - - Unit tests with coverage metrics (target: >80%) - - Integration tests results - - E2E test scenarios - -- **Test results**: All passing with coverage report - -- **Deployment confirmation**: - - DaemonSet deployed successfully - - Running on X nodes - - Monitors active and reporting - - Exporters publishing metrics - -- **Safety verification** (for remediator tasks): - - Cooldown periods enforced - - Circuit breaker tested - - Rate limiting active - - Attempt limits respected - -- Any warnings or notes about behavior or limitations - -#### Update TaskForge: -```javascript -// Mark task as completed -mcp__taskforge__updateTask({ - taskId: , - status: "done", - actualHours: -}); - -// This automatically records completion timestamp in database -// Task is now marked complete across all sessions -``` - -#### Commit with proper format: -```bash -cd /home/mmattox/go/src/github.com/supporttools/node-doctor - -git add . -git commit -m "feat(monitors): add CPU throttling monitor - -- Implement CPUMonitor with throttling detection -- Add systemd remediator with safety mechanisms -- Update DaemonSet with host access permissions -- Add comprehensive unit tests (87% coverage) -- Add integration tests for Problem Detector -- Task completion: [TaskForge task description]" - -git push origin main -``` - -**TodoWrite Update**: Mark "Task completion" as completed. Clean up todo list - all workflow steps done. - -**Await next task assignment** - ---- - -## Node Doctor-Specific Requirements - -### ABSOLUTE RULES - -1. **ONE TASK AT A TIME** - Never work on multiple tasks simultaneously -2. **RESEARCH BEFORE CODING** - Mandatory research phase, no exceptions -3. **DESIGN COMPLIANCE** - Must follow architecture.md specifications -4. **GO 1.21+ ONLY** - Verify version in go.mod file -5. **BUILD VERIFICATION** - Must pass before QA review -6. **QA VALIDATION** - Must pass before deployment -7. **DEPLOYMENT MANDATORY** - Must deploy DaemonSet to cluster -8. **DEVILS ADVOCATE** - Must pass before completion -9. **QUALITY CYCLE** - Stay in cycle until fully resolved -10. **PRIORITY ENFORCEMENT** - Never skip priority levels (P1 → P2 → P3 → P4 → P5) -11. **SAFETY FIRST** - All remediators must implement safety mechanisms - -### FORBIDDEN ACTIONS - -- Starting new task before current fully completed -- Skipping workflow steps -- Proceeding with failed builds -- Ignoring QA findings -- Bypassing deployment -- Marking complete without devils advocate approval -- Creating files in repository root (except allowed config/docs) -- Using Go version < 1.21 -- Implementing remediators without safety mechanisms -- Skipping cooldown periods in remediators -- Disabling circuit breakers or rate limiters -- Running privileged operations without input validation -- Ignoring RBAC boundaries in Kubernetes operations - -### SAFETY-CRITICAL RULES (Remediators) - -**Every remediator MUST implement:** -1. **Cooldown Period**: Minimum time between remediation attempts (default: 5 minutes) -2. **Attempt Limiting**: Maximum attempts per time window (default: 3 per hour) -3. **Circuit Breaker**: Stop attempting after consecutive failures (default: 5 failures) -4. **Rate Limiting**: Global rate limit across all problems (default: 10 per minute) -5. **Input Validation**: Validate ALL inputs before system calls -6. **Audit Logging**: Log ALL remediation attempts with outcomes -7. **Dry-Run Mode**: Support dry-run testing without actual changes - -**Never:** -- Execute system commands without validation -- Disable safety mechanisms "temporarily" -- Skip cooldown periods for "urgent" fixes -- Ignore circuit breaker state -- Bypass rate limiting - -### KUBERNETES DEPLOYMENT RULES - -1. **DaemonSet Only**: Node Doctor must run as DaemonSet in kube-system -2. **RBAC Minimal**: Grant only necessary permissions, follow least privilege -3. **Resource Limits**: Always specify CPU/memory limits -4. **Node Selection**: Support nodeSelector/tolerations for selective deployment -5. **Config Management**: Use ConfigMaps for configuration, not environment variables -6. **Secrets Management**: Never hardcode credentials, use Kubernetes Secrets -7. **Health Checks**: Implement liveness and readiness probes - -### EMERGENCY PROCEDURES - -If workflow cannot be completed due to infrastructure issues: - -1. Document blocker in task description -2. Use TaskForge to track blocker: - ```javascript - mcp__taskforge__createTask({ - projectId: 41, - featureId: 250, // Core Framework or appropriate feature - name: "Blocker: [description]", - priority: "urgent", - status: "todo" - }); - ``` -3. Use `mcp__taskforge__setDependency` to link blocked task -4. Do not start new tasks until blocker resolved -5. Escalate for assistance if needed - -If safety mechanisms trigger (circuit breaker, rate limits): -1. **DO NOT disable safety mechanisms** -2. Investigate root cause of failures -3. Fix underlying issue, not the safety mechanism -4. Reset circuit breaker only after verifying fix -5. Document incident and lessons learned - ---- - -## Common Node Doctor Scenarios - -### Scenario 1: Adding New Monitor - -**Files affected**: `pkg/monitors/`, tests, config, docs - -**Workflow**: -1. Create monitor implementation (`pkg/monitors/[name].go`) -2. Implement Monitor interface (Start, Stop methods) -3. Use channels for status communication -4. Write comprehensive unit tests -5. Add configuration type and validation -6. Register monitor in orchestrator -7. Update ConfigMap with monitor config -8. Test in local cluster -9. Update docs/monitors.md - -### Scenario 2: Adding New Remediator - -**Files affected**: `pkg/remediators/`, tests, config, docs - -**Workflow**: -1. Create remediator implementation (`pkg/remediators/[name].go`) -2. Implement Remediator interface (CanRemediate, Remediate, GetCooldown) -3. **MANDATORY**: Implement ALL safety mechanisms - - Circuit breaker - - Rate limiter - - Cooldown tracker - - Attempt counter -4. Write tests for safety mechanisms -5. Write tests for remediation logic -6. Add configuration type and validation -7. Register remediator in orchestrator -8. Test in isolated environment first -9. Update deploy/rbac.yaml if new permissions needed -10. Update docs/remediation.md - -### Scenario 3: Adding New Exporter - -**Files affected**: `pkg/exporters/`, tests, config, deployment - -**Workflow**: -1. Create exporter implementation -2. Implement Exporter interface -3. Add metrics definitions (Prometheus) or event types (Kubernetes) -4. Write tests for metric accuracy -5. Update DaemonSet with necessary service ports -6. Create Service resource if needed (for Prometheus scraping) -7. Test metric publishing -8. Update documentation - -### Scenario 4: Updating DaemonSet Configuration - -**Files affected**: `deploy/daemonset.yaml`, `deploy/configmap.yaml` - -**Workflow**: -1. Modify DaemonSet manifest -2. Update resource limits if needed -3. Add volume mounts for new host access -4. Update RBAC if new permissions required -5. Test deployment in dev cluster -6. Verify backward compatibility -7. Document changes in deployment guide -8. Roll out with `kubectl apply` - -### Scenario 5: Performance Optimization - -**Files affected**: Multiple `pkg/` files, tests - -**Workflow**: -1. Profile current performance (CPU, memory, goroutines) -2. Identify bottlenecks -3. Implement optimizations -4. Verify no functionality regression -5. Run benchmark tests -6. Update resource limits in DaemonSet -7. Document performance improvements - ---- - -## Quality Gates Summary - -| Phase | Gate | Blocker | Tool | -|-------|------|---------|------| -| Research | Design approved | Yes | Manual approval | -| Build | Binary compiles | Yes | `make build-node-doctor-local` | -| QA | Agent review passes | Yes | Task tool (qa-engineer) | -| Test | All tests pass | Yes | `make validate-pipeline-local` | -| Deploy | DaemonSet healthy | Yes | `kubectl apply` + rollout status | -| Verify | Devils advocate approves | Yes | Task tool (devils-advocate-qa) | -| Complete | All gates passed | Yes | TaskForge update + commit | - ---- - -## Workflow Diagram - -``` -┌─────────────────────────────────────────────────────────────┐ -│ 1. Receive Task │ -│ - Verify priority & dependencies (TaskForge) │ -│ - Update status to "in_progress" │ -│ - Create TodoWrite breakdown │ -└────────────────────────â”Ŧ────────────────────────────────────┘ - │ - â–ŧ -┌─────────────────────────────────────────────────────────────┐ -│ 2. Research & Design Phase (MANDATORY) │ -│ - Review architecture.md, monitors.md, remediation.md │ -│ - Identify integration points │ -│ - Design safety mechanisms │ -│ - Present implementation plan │ -└────────────────────────â”Ŧ────────────────────────────────────┘ - │ - â–ŧ -┌─────────────────────────────────────────────────────────────┐ -│ 3. Wait for Approval (GATE) │ -│ ✅ Approve → Continue │ -│ 🔄 Revise → Back to Step 2 │ -│ â­ī¸ Skip → Await new task │ -└────────────────────────â”Ŧ────────────────────────────────────┘ - │ (approved) - â–ŧ -┌─────────────────────────────────────────────────────────────┐ -│ 4. Implementation Phase │ -│ - Implement monitors/remediators/exporters │ -│ - Follow coding standards │ -│ - Write tests alongside code (TDD) │ -│ - Implement safety mechanisms │ -└────────────────────────â”Ŧ────────────────────────────────────┘ - │ - â–ŧ -┌─────────────────────────────────────────────────────────────┐ -│ 5.1 Build Verification (BLOCKER) │ -│ make build-node-doctor-local │ -│ ❌ Failed → Fix → Retry │ -└────────────────────────â”Ŧ────────────────────────────────────┘ - │ (passed) - â–ŧ -┌─────────────────────────────────────────────────────────────┐ -│ 5.2 QA Agent Review (BLOCKER) │ -│ Review code quality, safety, security │ -│ ❌ Issues found → Fix → Retry │ -└────────────────────────â”Ŧ────────────────────────────────────┘ - │ (passed) - â–ŧ -┌─────────────────────────────────────────────────────────────┐ -│ 5.3 Run Test Suite (BLOCKER) │ -│ make validate-pipeline-local │ -│ ❌ Tests failed → Fix → Back to 5.2 │ -└────────────────────────â”Ŧ────────────────────────────────────┘ - │ (passed) - â–ŧ -┌─────────────────────────────────────────────────────────────┐ -│ 6. Deployment (MANDATORY) │ -│ kubectl apply -f deploy/ │ -│ kubectl rollout status daemonset/node-doctor │ -│ ❌ Failed → Fix → Back to 5.1 │ -└────────────────────────â”Ŧ────────────────────────────────────┘ - │ (deployed) - â–ŧ -┌─────────────────────────────────────────────────────────────┐ -│ 7. Devils Advocate Verification (MANDATORY) │ -│ Verify task completion and safety │ -│ ❌ Issues found → Back to 5.1 (Quality Cycle) │ -└────────────────────────â”Ŧ────────────────────────────────────┘ - │ (approved) - â–ŧ -┌─────────────────────────────────────────────────────────────┐ -│ 8. Task Completion │ -│ - Provide completion summary │ -│ - Update TaskForge status to "done" │ -│ - Commit with proper message format │ -│ - Clean up TodoWrite │ -│ - Await next task │ -└─────────────────────────────────────────────────────────────┘ -``` - -### Monitor/Remediator Development Flow - -``` -New Feature Request - │ - â–ŧ -┌──────────────────────────────────────────────────────────────┐ -│ Research & Design Phase │ -│ - Review similar monitors/remediators │ -│ - Design safety mechanisms (for remediators) │ -│ - Plan channel communication │ -│ - Design configuration schema │ -└───────────────────────â”Ŧ──────────────────────────────────────┘ - │ - â–ŧ -┌──────────────────────────────────────────────────────────────┐ -│ Implementation │ -│ - pkg/monitors/[name].go OR pkg/remediators/[name].go │ -│ - pkg/monitors/[name]_test.go (unit tests) │ -│ - pkg/config/types.go (config schema) │ -│ - tests/integration/[name]_test.go │ -└───────────────────────â”Ŧ──────────────────────────────────────┘ - │ - â–ŧ -┌──────────────────────────────────────────────────────────────┐ -│ Quality Assurance │ -│ - Build verification │ -│ - QA agent review │ -│ - Full test suite (unit, integration, e2e) │ -│ - Safety mechanism validation (remediators) │ -└───────────────────────â”Ŧ──────────────────────────────────────┘ - │ - â–ŧ -┌──────────────────────────────────────────────────────────────┐ -│ Deployment to Cluster │ -│ - Update deploy/configmap.yaml with new config │ -│ - Update deploy/rbac.yaml if new permissions needed │ -│ - kubectl apply -f deploy/ │ -│ - Verify DaemonSet rollout │ -│ - Monitor logs for new feature activity │ -└───────────────────────â”Ŧ──────────────────────────────────────┘ - │ - â–ŧ -┌──────────────────────────────────────────────────────────────┐ -│ Devils Advocate Verification │ -│ - Verify feature works as designed │ -│ - Verify safety mechanisms active (remediators) │ -│ - Verify no regressions in existing monitors │ -│ - Check resource usage impact │ -└───────────────────────â”Ŧ──────────────────────────────────────┘ - │ - â–ŧ - Feature Complete - Monitor active on all nodes -``` - ---- - -This workflow ensures systematic, quality-driven development aligned with Node Doctor architecture and Kubernetes best practices. All work follows the **Safety First** principle, especially for remediators with privileged access to nodes. diff --git a/docs/development/validation-quick-reference.md b/docs/development/validation-quick-reference.md deleted file mode 100644 index cd60680..0000000 --- a/docs/development/validation-quick-reference.md +++ /dev/null @@ -1,445 +0,0 @@ -# Local CI/CD Validation - Quick Reference - -Quick reference guide for using the local validation system to catch issues before pushing. - -## âš ī¸ CRITICAL: Validation Mode Selection - -**ALWAYS use FULL validation before marking tasks complete or pushing to remote!** - -- **Quick Mode** (`--quick`): Skips gosec security scans and tests - - ✅ Use during development for fast feedback - - ❌ **NEVER** use as final validation before commit/push - -- **Full Mode** (default): Runs ALL checks including gosec and tests - - ✅ **MANDATORY** before pushing to remote - - ✅ **MANDATORY** before marking task complete - - ✅ Mirrors CI/CD pipeline exactly - -**Lesson Learned**: Using quick mode as final validation led to CI/CD failures that could have been caught locally. Always run full validation as the last step! - -## Before You Push - -### Full Validation (Recommended) - -```bash -# Validate everything - mirrors CI/CD completely -make validate-pipeline-local - -# Or directly -./scripts/validate-pipeline-local.sh -``` - -**When to use:** Before pushing to main/master, before creating PR - -**Time:** ~2-5 minutes (depends on changed components) - -### Quick Validation - -```bash -# Skip tests and gosec - fast feedback -make validate-quick - -# Or directly -./scripts/validate-pipeline-local.sh --quick -``` - -**When to use:** Before committing, during development, frequent checks - -**Time:** ~30 seconds - -### Component-Specific - -```bash -# Validate only one component -make validate-component COMPONENT=api - -# Or directly -./scripts/validate-pipeline-local.sh --component=ingestor -``` - -**Valid components:** api, org-management-controller, probe-controller, ingestor, customer-probe-agent, pkg - -**When to use:** Working on specific component, debugging validation failures - -**Time:** ~20 seconds per component - -## Common Commands - -### Makefile Targets - -```bash -make validate-pipeline-local # Full validation -make validate-quick # Quick check (no tests) -make validate-pre-push # Alias for full validation -make validate-component COMPONENT=api # Single component -``` - -### Script Options - -```bash -./scripts/validate-pipeline-local.sh --quick # Skip tests -./scripts/validate-pipeline-local.sh --component=api # One component -./scripts/validate-pipeline-local.sh --sequential # Run one at a time -./scripts/validate-pipeline-local.sh --fail-fast # Stop on first error -./scripts/validate-pipeline-local.sh --help # Show all options -``` - -## Common Issues and Fixes - -### Format Issues - -**Error:** -``` -✗ [component] format FAILED -pkg/foo/bar.go -``` - -**Fix:** -```bash -# Auto-fix formatting -gofmt -w . - -# Or specific file -gofmt -w pkg/foo/bar.go -``` - -### Go Vet Errors - -**Error:** -``` -✗ [component] go-vet FAILED -composite literal uses unkeyed fields -``` - -**Fix:** Review the error, fix the code issue. These are real bugs that need fixing. - -### Staticcheck Warnings - -**Error:** -``` -✗ [component] staticcheck FAILED -pkg/foo/bar.go:42:2: field token is unused (U1000) -``` - -**Fix Option 1 - Suppress (if intentional):** -```go -type Foo struct { - token string //lint:ignore U1000 Reserved for future JWT auth -} -``` - -**Fix Option 2 - Remove unused code:** -```go -// Delete the unused field/function -``` - -### Gosec Security Warnings - -**Error:** -``` -✗ [component] gosec FAILED -[G115] integer overflow conversion (CWE-190) -``` - -**Fix Option 1 - Suppress (if validated):** -```go -//nolint:gosec // G115 - Validated: orgID range checked before conversion -orgID := uint64(unsafeValue) -``` - -**Fix Option 2 - Use safe conversion:** -```go -import "github.com/{{PROJECT_NAME}}/{{PROJECT_NAME}}/pkg/utils" - -orgID := utils.SafeUintToUint64(value) // Handles overflow -``` - -### Test Failures - -**Error:** -``` -✗ [component] tests FAILED -FAIL: TestFoo (0.00s) -``` - -**Fix:** Review test output, fix the failing test or code - -**Run specific test:** -```bash -go test -v ./pkg/path/to/package -run TestName -``` - -### Race Detector Failures - -**Error:** -``` -WARNING: DATA RACE -Write at 0x... by goroutine X -``` - -**Fix:** This is a real concurrency bug - MUST be fixed - -**Never suppress race detector warnings!** - -## Component-Specific Commands - -### API - -```bash -cd /path/to/{{PROJECT_NAME}} -gofmt -l api/ -cd api && go vet ./... -cd api && staticcheck ./... -cd api && gosec -quiet ./... -cd api && go test -v -race ./... -``` - -### pkg - -```bash -gofmt -l pkg/ -go vet ./pkg/... -staticcheck ./pkg/... -gosec -quiet ./pkg/... -go test -v -race ./pkg/... -``` - -### org-management-controller - -```bash -gofmt -l pkg/controller/org-management/ -go vet ./pkg/controller/org-management/... -staticcheck ./pkg/controller/org-management/... -gosec -quiet ./pkg/controller/org-management/... -go test -v -race ./pkg/controller/org-management/... -``` - -### probe-controller - -```bash -gofmt -l pkg/probe/ -go vet ./pkg/probe/... -staticcheck ./pkg/probe/... -gosec -quiet ./pkg/probe/... -go test -v -race ./pkg/probe/... -``` - -### ingestor - -```bash -gofmt -l pkg/ingestor/ -go vet ./pkg/ingestor/... -staticcheck ./pkg/ingestor/... -gosec -quiet ./pkg/ingestor/... -go test -v -race ./pkg/ingestor/... -``` - -### customer-probe-agent - -**Note:** Has separate go.mod - must cd into directory - -```bash -gofmt -l cmd/customer-probe-agent/ -cd cmd/customer-probe-agent && go vet ./... -cd cmd/customer-probe-agent && staticcheck ./... -cd cmd/customer-probe-agent && gosec -quiet ./... -cd cmd/customer-probe-agent && go test -v -race ./... -``` - -## Enable Pre-Push Hook - -### Automatic Validation on Push - -```bash -# Run setup script -./.githooks/setup.sh - -# Or manually configure -git config core.hooksPath .githooks -``` - -### Verify Hook Enabled - -```bash -git config --get core.hooksPath -# Should output: .githooks -``` - -### Bypass Hook (Emergency Only) - -```bash -git push --no-verify -``` - -**Warning:** Only use --no-verify for emergencies! - -## Troubleshooting - -### "directory prefix does not contain main module" - -**Issue:** customer-probe-agent has separate go.mod - -**Fix:** Always cd into directory: -```bash -cd cmd/customer-probe-agent && go vet ./... -``` - -### "couldn't parse format string" - -**Issue:** Staticcheck false positive on `%.2f%%` - -**Fix:** Add suppression: -```go -//lint:ignore SA5009 staticcheck false positive - format string valid -return fmt.Sprintf("%.2f%%", value) -``` - -### Race Detector Takes Too Long - -**Solution:** Use quick mode during development: -```bash -make validate-quick # Skips race detector -``` - -Run full validation before push: -```bash -make validate-pre-push # Includes race detector -``` - -### Validation Hangs - -**Solution:** Run sequential instead of parallel: -```bash -./scripts/validate-pipeline-local.sh --sequential -``` - -### Need to Validate Only Changed Code - -**Solution:** Validate specific component: -```bash -# If you only changed ingestor code -make validate-component COMPONENT=ingestor -``` - -## Best Practices - -### Development Workflow - -1. **During development:** `make validate-quick` (frequent checks) -2. **Before commit:** `make validate-quick` (ensure no format/lint issues) -3. **Before push:** `make validate-pre-push` (full validation with tests - **MANDATORY**) -4. **Before marking task complete:** Full validation must pass - no exceptions! - -**âš ī¸ WARNING**: Quick mode skips gosec security scans and tests. These WILL fail in CI/CD if you don't run full validation locally! - -### Time-Saving Tips - -1. **Use quick mode frequently** - catches 80% of issues in 30 seconds -2. **Validate specific components** - faster feedback when working on one area -3. **Enable pre-push hook** - prevents accidental pushes with issues -4. **Fix format issues first** - fastest to fix, unblocks other checks - -### When to Bypass Validation - -**Only bypass in these situations:** -- Emergency hotfix (non-code changes) -- Infrastructure-only changes (Helm charts, docs) -- You've already validated manually -- CI/CD pipeline will catch the issues anyway (not recommended!) - -**Never bypass for:** -- Code changes -- "It works on my machine" -- "CI/CD is slow" -- Time pressure (fix the issues!) - -## Examples - -### Example 1: Quick Development Check - -```bash -# Make some code changes -vim pkg/api/handlers/foo.go - -# Quick check before committing -make validate-quick - -# If passes, commit -git add . -git commit -m "feat: add new handler" -``` - -### Example 2: Full Pre-Push Validation - -```bash -# Finished feature, ready to push -make validate-pre-push - -# All checks pass -git push origin feature-branch -``` - -### Example 3: Fix Validation Failures - -```bash -# Validation fails on format -make validate-quick -# ✗ [api] format FAILED - -# Fix formatting -gofmt -w api/ - -# Re-validate -make validate-quick -# ✓ All checks pass - -# Push -git push origin main -``` - -### Example 4: Component-Specific Development - -```bash -# Working on ingestor only -vim pkg/ingestor/server/metrics.go - -# Validate just ingestor -make validate-component COMPONENT=ingestor - -# Passes, commit and continue -git commit -am "feat(ingestor): improve metrics collection" -``` - -## Validation Checklist - -Before pushing code, ensure: - -- [ ] Code is properly formatted (`gofmt`) -- [ ] No vet warnings (real bugs) -- [ ] No staticcheck issues (or properly suppressed) -- [ ] No security issues (gosec) -- [ ] All tests passing -- [ ] No race conditions detected -- [ ] Validation script returns exit code 0 - -## Need More Help? - -- **Full documentation:** [local-cicd-validation-guide.md](local-cicd-validation-guide.md) -- **Git hooks:** [.githooks/README.md](../../.githooks/README.md) -- **Task workflow:** [task-execution-workflow.md](task-execution-workflow.md) -- **CI/CD pipeline:** [.github/workflows/pipeline-v2.yml](../../.github/workflows/pipeline-v2.yml) - -## Quick Commands Summary - -```bash -# Most common commands -make validate-quick # Fast check -make validate-pipeline-local # Full check -make validate-component COMPONENT=api # One component -./.githooks/setup.sh # Enable pre-push hook -git push --no-verify # Bypass hook (emergency) -``` - ---- - -**Remember:** Validation saves time by catching issues early. A few minutes of local validation prevents hours of CI/CD debugging! diff --git a/docs/release-process.md b/docs/release-process.md index 8b2834b..a281cfe 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -338,7 +338,7 @@ prerelease marker — is treated as a prerelease. `ci.yml` used to defeat this exclusion entirely: it had no prerelease gate and pushed `:latest` (amd64-only) on **every** `v*` tag, so cutting an RC could leave `latest` pointing at RC content, on one architecture. `ci.yml` no longer pushes anything, so `release.yml`'s exclusion -is now the only thing that decides `:latest`. `make bump-rc` likewise no longer tags `:latest`. +is now the only thing that decides `:latest`. The Makefile never tags or pushes `:latest`. ## Creating a Release @@ -460,9 +460,7 @@ step, no `crazy-max/ghaction-import-gpg`, and no `.asc` output. If you previousl `GPG_PRIVATE_KEY` repository secret, nothing consumes it — it should be removed and the key material revoked, because an unused secret is pure attack surface. -`scripts/setup-gpg-signing-subkey.sh` and `scripts/verify-gpg-setup.sh` are left over from a -signing scheme that was never wired into the workflow. Running them configures nothing. -Likewise `.goreleaser.yml` exists in the repo root but **is never invoked** — no workflow runs +`.goreleaser.yml` exists in the repo root but **is never invoked** — no workflow runs GoReleaser, so it produces no binaries, no archives, and no `checksums.txt`. ### Verifying an image @@ -529,8 +527,7 @@ helm -n node-doctor get manifest node-doctor | grep -A6 'Minimal resources' ### If the cluster was deployed with raw manifests -Deployments made with `kubectl apply -f deployment/daemonset.yaml` (or `make deploy-prd-kubectl`) -are not Helm releases and have no `helm history`. Roll those back with: +Deployments made with `kubectl apply -f deployment/daemonset.yaml` are not Helm releases and have no `helm history`. Roll those back with: ```bash kubectl -n node-doctor rollout undo daemonset/node-doctor @@ -570,53 +567,14 @@ version must be unreachable, delete the tags from Docker Hub and revert the char ### `VERSION` defaults to an epoch timestamp -`Makefile:42` sets `VERSION := $(shell date +%s)`. Every target that interpolates `$(VERSION)` -without an override gets an integer like `1786... `, which is never a published image tag. +`VERSION := $(shell date +%s)` in the Makefile, so local `make build-*-image` tags images with an +integer that is never a published tag. `make docker-push` / `push-*-image` refuse to run unless +`VERSION` is set explicitly (`make docker-push VERSION=1.8.7`), and no Makefile target tags or +pushes `:latest`; only `release.yml` does. -```bash -make deploy-dev # helm upgrade --install ... --set image.tag=1786132891 -make deploy-stg # same -make deploy-prd # same, behind a y/n prompt -``` - -**These three targets are broken as written.** They deploy the local chart with an image tag that -does not exist in any registry, producing `ImagePullBackOff`. If you use them, always override: - -```bash -make deploy-dev VERSION=1.8.7 -``` - -They also pass `--set environment=`, a value the chart templates do not consume. - -`make build-*-image` / `push-*-image` have the same `VERSION` default; locally built images are -tagged with the epoch and with `latest`. Pushing those tags to `docker.io/supporttools` -overwrites the registry `latest` with a local build — avoid `make docker-push` unless that is -explicitly what you want. - -### `make helm-publish` does nothing - -The `helm-publish` target packages the chart and then hits a `# CUSTOMIZE: Add your Helm chart publish -command` placeholder before printing "Helm chart published". It publishes nothing. The only path -to `charts.support.tools` is the `helm-publish` job in `release.yml`, i.e. a tag push. - -### `make bump-rc` is a direct-to-production path - -It increments `.version-rc`, builds and pushes a multi-arch image, rewrites the image line in -`deployment/daemonset.yaml` with `sed`, applies it to the `a1-ops-prd` cluster, then commits, -tags and **pushes to `main` with `--tags`**. Pushing the tag re-triggers the full `release.yml`. -Understand all of that before running it. - -Two things about it were fixed alongside the tag-naming work, worth knowing if you remember the -old behaviour: - -- It used to also tag `:latest`, so a workstation build of a *release candidate* could overwrite - the `:latest` that `release.yml` publishes only for stable tags. It now pushes only the RC tag. -- Its `git commit` / `git tag` / `git push` steps used to end in `|| true`, silently swallowing - failures — leaving an image pushed and a cluster deployed from a revision that was never - tagged or pushed anywhere. They now fail loudly. - -Its console output used to claim it pushes "to Harbor"; it pushes to Docker Hub -(`REGISTRY := docker.io/supporttools`), and the strings now say so. +The former `deploy-dev/stg/prd`, `deploy-prd-kubectl`, `bump`, `bump-rc` and `helm-publish` +targets were removed. Deploy with Helm from `charts.support.tools` (see Rollback Procedures for +the equivalent `helm upgrade` commands); publishing is a `v*` tag push. ### Useful and accurate @@ -754,4 +712,4 @@ Listed explicitly because previous versions of this document claimed otherwise: - [Helm rollback](https://helm.sh/docs/helm/helm_rollback/) - [GitHub Actions Workflows](../.github/workflows/) - [Node Doctor Architecture](./architecture.md) -- [Deployment Guide](./deployment.md) +- [Deployment Guide](../deployment/README.md) diff --git a/helm/node-doctor/templates/controller-deployment.yaml b/helm/node-doctor/templates/controller-deployment.yaml index e4678bb..eeaf92b 100644 --- a/helm/node-doctor/templates/controller-deployment.yaml +++ b/helm/node-doctor/templates/controller-deployment.yaml @@ -34,7 +34,7 @@ spec: {{- end }} containers: - name: controller - image: "{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag | default .Chart.AppVersion }}" + image: "{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag | default .Values.image.tag }}" imagePullPolicy: {{ .Values.controller.image.pullPolicy }} args: - --config=/etc/node-doctor/controller.yaml diff --git a/helm/node-doctor/values.yaml b/helm/node-doctor/values.yaml index 8803a81..47bbe98 100644 --- a/helm/node-doctor/values.yaml +++ b/helm/node-doctor/values.yaml @@ -590,8 +590,8 @@ controller: image: repository: supporttools/node-doctor-controller pullPolicy: IfNotPresent - # Defaults to chart appVersion when empty - tag: "" + # Falls back to image.tag when empty + tag: "1.0.0" # Service account for the controller serviceAccount: diff --git a/helm/node-doctor/values.yaml.template b/helm/node-doctor/values.yaml.template index f4f71f9..23a7e32 100644 --- a/helm/node-doctor/values.yaml.template +++ b/helm/node-doctor/values.yaml.template @@ -590,8 +590,8 @@ controller: image: repository: supporttools/node-doctor-controller pullPolicy: IfNotPresent - # Defaults to chart appVersion when empty - tag: "" + # Falls back to image.tag when empty + tag: "${IMAGE_TAG}" # Service account for the controller serviceAccount: diff --git a/hooks/coding/basic-automation.toml b/hooks/coding/basic-automation.toml deleted file mode 100644 index 24e2d65..0000000 --- a/hooks/coding/basic-automation.toml +++ /dev/null @@ -1,24 +0,0 @@ -# Basic Coding Project Automation Hooks -# Customize based on project needs - -# Optional: Auto-save reminder -[[hooks]] -event = "Notification" -[hooks.matcher] -time = "every 30 minutes" -command = "echo '💾 Auto-save reminder: Your work is being saved automatically'" - -# Optional: Simple test reminder (if user mentioned testing) -[[hooks]] -event = "UserPromptSubmit" -[hooks.matcher] -content_regex = "done|finished|complete" -command = "echo 'đŸ§Ē Remember to test your changes!'" - -# Optional: Basic formatting (customize per language) -# [[hooks]] -# event = "PostToolUse" -# [hooks.matcher] -# tool_name = "Write" -# file_paths = ["*.py"] # Adjust for project language -# command = "echo '🎨 Formatting code...' && black $CLAUDE_FILE_PATH 2>/dev/null || true" \ No newline at end of file diff --git a/hooks/writing/content-automation.toml b/hooks/writing/content-automation.toml deleted file mode 100644 index 89f5dcb..0000000 --- a/hooks/writing/content-automation.toml +++ /dev/null @@ -1,39 +0,0 @@ -# Writing Project Automation Hooks -# For blogs, articles, books, etc. - -# Optional: Word count tracking -[[hooks]] -event = "Stop" -command = """ -if [ -d "content" ]; then - count=$(wc -w content/*.md 2>/dev/null | tail -1 | awk '{print $1}') - echo "📝 Total word count: $count words" - echo "$(date): $count words" >> .writing-progress.log -fi -""" - -# Optional: Writing session timer -[[hooks]] -event = "UserPromptSubmit" -[hooks.matcher] -content_regex = "start writing|begin writing" -command = "echo '⏰ Writing session started at $(date +%H:%M)' && echo $(date +%s) > .session-start" - -[[hooks]] -event = "Stop" -command = """ -if [ -f .session-start ]; then - start=$(cat .session-start) - duration=$(( ($(date +%s) - start) / 60 )) - echo "âąī¸ Writing session duration: $duration minutes" - rm .session-start -fi -""" - -# Optional: Backup reminder -[[hooks]] -event = "Notification" -[hooks.matcher] -time = "17:00" -days = ["monday", "wednesday", "friday"] -command = "echo 'â˜ī¸ Remember to backup your writing to cloud storage!'" \ No newline at end of file diff --git a/pkg/exporters/prometheus/exporter_test.go b/pkg/exporters/prometheus/exporter_test.go index 169a1da..d34faff 100644 --- a/pkg/exporters/prometheus/exporter_test.go +++ b/pkg/exporters/prometheus/exporter_test.go @@ -1118,8 +1118,8 @@ func TestExportProblemBeforeStart(t *testing.T) { } func TestPrometheusExporter_StartBindFailure(t *testing.T) { - // Occupy a port so the exporter cannot bind to it. - ln, err := net.Listen("tcp", "127.0.0.1:0") + // Squat on the wildcard: BSD lets a wildcard bind coexist with a 127.0.0.1 bind. + ln, err := net.Listen("tcp", ":0") if err != nil { t.Fatalf("failed to grab a free port: %v", err) } diff --git a/pkg/health/liveness_readiness_test.go b/pkg/health/liveness_readiness_test.go index 1384310..465ea85 100644 --- a/pkg/health/liveness_readiness_test.go +++ b/pkg/health/liveness_readiness_test.go @@ -6,7 +6,6 @@ import ( "errors" "net" "net/http" - "path/filepath" "testing" "time" @@ -20,7 +19,7 @@ import ( func newSocketServer(t *testing.T) (*Server, *http.Client) { t.Helper() - socket := filepath.Join(t.TempDir(), "health.sock") + socket := testSocketPath(t) srv, err := NewServer(&Config{ Enabled: true, BindAddress: "127.0.0.1", diff --git a/pkg/health/server_test.go b/pkg/health/server_test.go index f4729c0..cf97ae7 100644 --- a/pkg/health/server_test.go +++ b/pkg/health/server_test.go @@ -13,6 +13,17 @@ import ( "time" ) +// t.TempDir() paths can exceed the unix socket path limit (104 bytes on macOS). +func testSocketPath(t *testing.T) string { + t.Helper() + dir, err := os.MkdirTemp("", "nd") + if err != nil { + t.Fatalf("mkdir temp: %v", err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + return filepath.Join(dir, "health.sock") +} + // getOverUnixSocket issues an HTTP GET for path against a health server listening // on a unix domain socket, returning the status code. func getOverUnixSocket(t *testing.T, socketPath, path string) int { @@ -37,7 +48,7 @@ func getOverUnixSocket(t *testing.T, socketPath, path string) int { // TestServer_UnixSocket verifies the health endpoints are served over the per-pod // unix socket and that the socket file is cleaned up on Stop. func TestServer_UnixSocket(t *testing.T) { - sockPath := filepath.Join(t.TempDir(), "health.sock") + sockPath := testSocketPath(t) srv, err := NewServer(&Config{ Enabled: true, Port: 0, // ephemeral TCP — no conflict @@ -81,7 +92,7 @@ func TestServer_TCPConflictNonFatal(t *testing.T) { defer func() { _ = squatter.Close() }() occupiedPort := squatter.Addr().(*net.TCPAddr).Port - sockPath := filepath.Join(t.TempDir(), "health.sock") + sockPath := testSocketPath(t) srv, err := NewServer(&Config{ Enabled: true, BindAddress: "127.0.0.1", // non-dual-stack: bind fails outright on the taken port diff --git a/pkg/monitors/custom/logpattern_mocks.go b/pkg/monitors/custom/logpattern_mocks.go index 5432e4c..9b5e6f4 100644 --- a/pkg/monitors/custom/logpattern_mocks.go +++ b/pkg/monitors/custom/logpattern_mocks.go @@ -145,13 +145,6 @@ func (m *mockKmsgReader) ReadKmsg(ctx context.Context, path string) ([]byte, err type mockCommandExecutor struct { outputs map[string]string // command key -> output errors map[string]error // command key -> error - calls []commandCall // Track all calls for verification -} - -// commandCall records a command execution -type commandCall struct { - command string - args []string } // newMockCommandExecutor creates a new mock command executor @@ -159,7 +152,6 @@ func newMockCommandExecutor() *mockCommandExecutor { return &mockCommandExecutor{ outputs: make(map[string]string), errors: make(map[string]error), - calls: make([]commandCall, 0), } } @@ -177,12 +169,6 @@ func (m *mockCommandExecutor) setError(command string, args []string, err error) // ExecuteCommand implements CommandExecutor.ExecuteCommand func (m *mockCommandExecutor) ExecuteCommand(ctx context.Context, command string, args []string) ([]byte, error) { - // Record the call - m.calls = append(m.calls, commandCall{ - command: command, - args: args, - }) - key := m.makeKey(command, args) // Check for error first @@ -199,22 +185,6 @@ func (m *mockCommandExecutor) ExecuteCommand(ctx context.Context, command string return []byte{}, nil } -// getCalls returns all recorded command calls -func (m *mockCommandExecutor) getCalls() []commandCall { - return m.calls -} - -// wasCalled checks if a command was called with specific args -func (m *mockCommandExecutor) wasCalled(command string, args []string) bool { - key := m.makeKey(command, args) - for _, call := range m.calls { - if m.makeKey(call.command, call.args) == key { - return true - } - } - return false -} - // makeKey creates a unique key for a command and its args func (m *mockCommandExecutor) makeKey(command string, args []string) string { return command + " " + strings.Join(args, " ") @@ -235,7 +205,6 @@ const ( testOOMKillLog = "<4>[12345.678901] Out of memory: Kill process 1234 (test-process) score 1000" testDiskErrorLog = "<3>[12346.789012] EXT4-fs (sda1): error count: 5" testNetworkErrorLog = "<4>[12347.890123] NETDEV WATCHDOG: eth0 (e1000e): transmit queue 0 timed out" - testKernelPanicLog = "<0>[12348.901234] Kernel panic - not syncing: VFS: Unable to mount root fs" testNormalLog = "<6>[12349.012345] systemd[1]: Started User Manager for UID 1000." testKubeletErrorLog = "kubelet: Failed to initialize CSI driver" testContainerdLog = "containerd: error creating container" diff --git a/pkg/monitors/custom/logpattern_security_test.go b/pkg/monitors/custom/logpattern_security_test.go index 111c2a5..1fc3a8d 100644 --- a/pkg/monitors/custom/logpattern_security_test.go +++ b/pkg/monitors/custom/logpattern_security_test.go @@ -781,32 +781,30 @@ func TestSecurity_MemoryEstimation_Accuracy(t *testing.T) { units[i] = "test-" + string(rune(i)) + ".service" } - _ = &LogPatternMonitorConfig{ + config := &LogPatternMonitorConfig{ Patterns: patterns, JournalUnits: units, MaxEventsPerPattern: cfg.events, DedupWindow: 5 * time.Minute, } - // estimate := estimateMemoryUsage(config) - // - // // Estimates should scale with config size - // expectedMin := cfg.patterns * 1024 // At least 1KB per pattern - // if estimate < expectedMin { - // t.Errorf("Estimate too low: got %d bytes, expected at least %d", - // estimate, expectedMin) - // } - // - // // Estimates should be conservative (reasonable upper bound) - // expectedMax := 50 * 1024 * 1024 // 50MB max - // if estimate > expectedMax { - // t.Errorf("Estimate too high: got %d bytes, expected at most %d", - // estimate, expectedMax) - // } - - // t.Logf("%s config: %d patterns, %d events, %d units = %.2f MB estimated", - // cfg.name, cfg.patterns, cfg.events, cfg.units, - // float64(estimate)/(1024*1024)) + estimate := config.estimateMemoryUsage() + + expectedMin := cfg.patterns * 1024 + if estimate < expectedMin { + t.Errorf("Estimate too low: got %d bytes, expected at least %d", + estimate, expectedMin) + } + + expectedMax := 50 * 1024 * 1024 + if estimate > expectedMax { + t.Errorf("Estimate too high: got %d bytes, expected at most %d", + estimate, expectedMax) + } + + t.Logf("%s config: %d patterns, %d events, %d units = %.2f MB estimated", + cfg.name, cfg.patterns, cfg.events, cfg.units, + float64(estimate)/(1024*1024)) }) } } diff --git a/pkg/monitors/network/clusterdnspod_test.go b/pkg/monitors/network/clusterdnspod_test.go index 2218635..1274405 100644 --- a/pkg/monitors/network/clusterdnspod_test.go +++ b/pkg/monitors/network/clusterdnspod_test.go @@ -22,6 +22,7 @@ import ( // with no registered result respond 404, simulating an unreachable/errored pod. func startFakeOverlayServers(t *testing.T, hostToResult map[string]clusterdns.ProbeResult) (port int, cleanup func()) { t.Helper() + requireLoopbackAliases(t, hostToResult) mux := http.NewServeMux() mux.HandleFunc("/clusterdns", func(w http.ResponseWriter, r *http.Request) { @@ -56,6 +57,18 @@ func startFakeOverlayServers(t *testing.T, hostToResult map[string]clusterdns.Pr return tcpAddr.Port, ts.Close } +// Linux routes all of 127.0.0.0/8 to lo; macOS only configures 127.0.0.1. +func requireLoopbackAliases(t *testing.T, hostToResult map[string]clusterdns.ProbeResult) { + t.Helper() + for host := range hostToResult { + ln, err := net.Listen("tcp", net.JoinHostPort(host, "0")) + if err != nil { + t.Skipf("loopback alias %s not available: %v", host, err) + } + _ = ln.Close() + } +} + func newTestClusterDNSPodMonitor(port int, peers []Peer, opts func(*ClusterDNSPodConfig)) *ClusterDNSPodMonitor { cfg := &ClusterDNSPodConfig{ ProbePort: port, diff --git a/pkg/monitors/network/cni_health_test.go b/pkg/monitors/network/cni_health_test.go index 4d6543b..f793995 100644 --- a/pkg/monitors/network/cni_health_test.go +++ b/pkg/monitors/network/cni_health_test.go @@ -1,6 +1,7 @@ package network import ( + "net" "os" "path/filepath" "testing" @@ -793,9 +794,25 @@ func TestCheckHealth(t *testing.T) { } } +func loopbackInterfaceName(t *testing.T) string { + t.Helper() + ifaces, err := net.Interfaces() + if err != nil { + t.Fatalf("net.Interfaces: %v", err) + } + for _, iface := range ifaces { + if iface.Flags&net.FlagLoopback != 0 { + return iface.Name + } + } + t.Fatal("no loopback interface found") + return "" +} + func TestCheckInterfaces(t *testing.T) { // This test relies on actual system interfaces, so we test behavior // rather than specific interface names + lo := loopbackInterfaceName(t) t.Run("no expected interfaces", func(t *testing.T) { checker := NewCNIHealthChecker(CNIHealthConfig{ @@ -834,20 +851,20 @@ func TestCheckInterfaces(t *testing.T) { } }) - t.Run("lo interface exists on all systems", func(t *testing.T) { + t.Run("loopback interface exists on all systems", func(t *testing.T) { checker := NewCNIHealthChecker(CNIHealthConfig{ ConfigPath: t.TempDir(), CheckInterfaces: true, - ExpectedInterfaces: []string{"lo"}, + ExpectedInterfaces: []string{lo}, }) result := checker.CheckInterfaces() if !result.Healthy { - t.Errorf("Should be healthy when 'lo' interface exists. Errors: %v", result.Errors) + t.Errorf("Should be healthy when %q interface exists. Errors: %v", lo, result.Errors) } - if len(result.FoundInterfaces) != 1 || result.FoundInterfaces[0] != "lo" { - t.Errorf("Expected 'lo' in FoundInterfaces, got %v", result.FoundInterfaces) + if len(result.FoundInterfaces) != 1 || result.FoundInterfaces[0] != lo { + t.Errorf("Expected %q in FoundInterfaces, got %v", lo, result.FoundInterfaces) } }) @@ -855,7 +872,7 @@ func TestCheckInterfaces(t *testing.T) { checker := NewCNIHealthChecker(CNIHealthConfig{ ConfigPath: t.TempDir(), CheckInterfaces: true, - ExpectedInterfaces: []string{"lo", "nonexistent-xyz123"}, + ExpectedInterfaces: []string{lo, "nonexistent-xyz123"}, }) result := checker.CheckInterfaces() @@ -864,7 +881,7 @@ func TestCheckInterfaces(t *testing.T) { t.Error("Should be unhealthy when some expected interfaces are missing") } if len(result.FoundInterfaces) != 1 { - t.Errorf("FoundInterfaces should have 1 entry (lo), got %v", result.FoundInterfaces) + t.Errorf("FoundInterfaces should have 1 entry (%s), got %v", lo, result.FoundInterfaces) } if len(result.MissingInterfaces) != 1 { t.Errorf("MissingInterfaces should have 1 entry, got %v", result.MissingInterfaces) diff --git a/pkg/monitors/system/disk.go b/pkg/monitors/system/disk.go index 27c7c69..b42ae3b 100644 --- a/pkg/monitors/system/disk.go +++ b/pkg/monitors/system/disk.go @@ -84,7 +84,7 @@ func (p *defaultStatfsProvider) Statfs(path string) (*FilesystemStats, error) { BlocksAvail: stat.Bavail, Files: stat.Files, FilesFree: stat.Ffree, - BlockSize: stat.Bsize, + BlockSize: int64(stat.Bsize), //nolint:unconvert // Bsize is uint32 on darwin Flags: uint64(stat.Flags), }, nil } diff --git a/pkg/reload/validator.go b/pkg/reload/validator.go index 403788f..a4964ba 100644 --- a/pkg/reload/validator.go +++ b/pkg/reload/validator.go @@ -444,7 +444,7 @@ func (v *ConfigValidator) validateRemediation(remediation *types.RemediationConf func (v *ConfigValidator) validateMonitorRemediation(remediation *types.MonitorRemediationConfig, prefix string, result *ValidationResult) { if remediation.Strategy == "" { v.addError(result, prefix+".strategy", "remediation strategy is required") - } else if !v.isValidRemediationStrategy(remediation.Strategy) { + } else if !types.IsValidRemediationStrategy(remediation.Strategy) { v.addError(result, prefix+".strategy", fmt.Sprintf("unsupported remediation strategy '%s'", remediation.Strategy)) } @@ -613,16 +613,6 @@ func (v *ConfigValidator) isValidMonitorType(monitorType string) bool { return false } -func (v *ConfigValidator) isValidRemediationStrategy(strategy string) bool { - validStrategies := []string{"restart", "recreate", "script", "webhook"} - for _, validStrategy := range validStrategies { - if strategy == validStrategy { - return true - } - } - return false -} - func (v *ConfigValidator) validatePort(port interface{}) error { var portNum int switch p := port.(type) { diff --git a/pkg/reload/validator_test.go b/pkg/reload/validator_test.go index 5bca040..9900be9 100644 --- a/pkg/reload/validator_test.go +++ b/pkg/reload/validator_test.go @@ -615,7 +615,7 @@ func TestValidate_RemediationConsistency(t *testing.T) { config.Remediation.Enabled = false config.Monitors[0].Remediation = &types.MonitorRemediationConfig{ Enabled: true, - Strategy: "restart", + Strategy: "systemd-restart", } validator := NewConfigValidator() @@ -817,7 +817,7 @@ func createValidConfig() *types.NodeDoctorConfig { MaxAttemptsGlobal: 3, HistorySize: 100, CircuitBreaker: types.CircuitBreakerConfig{ - Enabled: true, + Enabled: boolPtr(true), Threshold: 5, TimeoutString: "30m", Timeout: 30 * time.Minute, @@ -1853,3 +1853,42 @@ func TestValidate_ReloadRejectsUnregisteredMonitorType(t *testing.T) { t.Errorf("Expected monitors[0].type error, got: %v", result.Errors) } } + +func TestValidateMonitorRemediation_StrategyNames(t *testing.T) { + validator := NewConfigValidator() + tests := []struct { + strategy string + valid bool + }{ + {"systemd-restart", true}, + {"webhook", false}, + } + for _, tt := range tests { + t.Run(tt.strategy, func(t *testing.T) { + result := &ValidationResult{Valid: true} + validator.validateMonitorRemediation(&types.MonitorRemediationConfig{ + Enabled: true, + Strategy: tt.strategy, + }, "monitors[0].remediation", result) + if result.Valid != tt.valid { + t.Errorf("strategy %q valid = %v, want %v (errors: %v)", tt.strategy, result.Valid, tt.valid, result.Errors) + } + }) + } +} + +func TestValidate_MonitorRemediationSystemdRestartPasses(t *testing.T) { + config := createValidConfig() + config.Monitors[0].Remediation = &types.MonitorRemediationConfig{ + Enabled: true, + Strategy: "systemd-restart", + Service: "kubelet", + } + + result := NewConfigValidator().Validate(config) + if !result.Valid { + t.Fatalf("reload validation must accept systemd-restart, got errors: %v", result.Errors) + } +} + +func boolPtr(b bool) *bool { return &b } diff --git a/pkg/reload/watcher_test.go b/pkg/reload/watcher_test.go index e5dd2f7..8bee39e 100644 --- a/pkg/reload/watcher_test.go +++ b/pkg/reload/watcher_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "runtime" "testing" "time" ) @@ -215,6 +216,9 @@ func TestConfigWatcherDebouncing(t *testing.T) { // TestConfigWatcherKubernetesSymlink tests handling of Kubernetes ConfigMap atomic updates func TestConfigWatcherKubernetesSymlink(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("rename-over of an existing symlink only surfaces as a Create event with inotify") + } // This test simulates how Kubernetes updates ConfigMaps: // 1. Creates new directory with timestamp // 2. Writes files to new directory diff --git a/pkg/remediators/registry.go b/pkg/remediators/registry.go index 1540c83..8c9fad5 100644 --- a/pkg/remediators/registry.go +++ b/pkg/remediators/registry.go @@ -194,6 +194,9 @@ func (r RemediationRecord) MarshalJSON() ([]byte, error) { // CircuitBreakerConfig contains configuration for the circuit breaker. type CircuitBreakerConfig struct { + // Disabled makes every check pass and stops failures from tripping the circuit + Disabled bool + // Threshold is the number of consecutive failures before opening the circuit Threshold int @@ -309,6 +312,31 @@ func NewRegistry(maxPerHour, maxHistory int) *RemediatorRegistry { } } +// NewRegistryFromConfig builds a registry wired exactly as a hot reload would +// leave it, so startup and reload share one source of truth. +func NewRegistryFromConfig(cfg *types.RemediationConfig, dryRunMode bool) (*RemediatorRegistry, error) { + if cfg == nil { + return nil, fmt.Errorf("remediation config cannot be nil") + } + + maxPerHour := cfg.MaxRemediationsPerHour + if maxPerHour == 0 { + maxPerHour = types.DefaultMaxRemediationsPerHour + } + historySize := cfg.HistorySize + if historySize == 0 { + historySize = types.DefaultHistorySize + } + + r := NewRegistry(maxPerHour, historySize) + r.SetDryRun(cfg.DryRun || dryRunMode) + r.SetMaxRemediationsPerMinute(cfg.MaxRemediationsPerMinute) + if err := r.applyCircuitBreakerConfig(cfg.CircuitBreaker); err != nil { + return nil, err + } + return r, nil +} + // Register adds a new remediator type to the registry. // This function is typically called during initialization to register // available remediators. @@ -443,18 +471,8 @@ func (r *RemediatorRegistry) ApplyConfig(cfg *types.RemediationConfig, dryRunMod r.SetMaxRemediationsPerHour(cfg.MaxRemediationsPerHour) r.SetMaxRemediationsPerMinute(cfg.MaxRemediationsPerMinute) - // Only push a circuit-breaker update when the new values are actually - // usable. A zero/absent circuitBreaker block must not clobber the running - // configuration with invalid values, so treat it as "leave as-is". - cb := CircuitBreakerConfig{ - Threshold: cfg.CircuitBreaker.Threshold, - Timeout: cfg.CircuitBreaker.Timeout, - SuccessThreshold: cfg.CircuitBreaker.SuccessThreshold, - } - if cb.Threshold > 0 && cb.Timeout > 0 && cb.SuccessThreshold > 0 { - if err := r.SetCircuitBreakerConfig(cb); err != nil { - return fmt.Errorf("apply circuit breaker config: %w", err) - } + if err := r.applyCircuitBreakerConfig(cfg.CircuitBreaker); err != nil { + return err } r.logInfof("Remediation config reloaded in place (dryRun=%v maxPerHour=%d maxPerMinute=%d)", @@ -462,6 +480,25 @@ func (r *RemediatorRegistry) ApplyConfig(cfg *types.RemediationConfig, dryRunMod return nil } +// applyCircuitBreakerConfig converts the typed config and applies it. A +// zero/absent circuitBreaker block is treated as "leave as-is" so it cannot +// clobber the running configuration with invalid values. +func (r *RemediatorRegistry) applyCircuitBreakerConfig(cfg types.CircuitBreakerConfig) error { + cb := CircuitBreakerConfig{ + Disabled: !cfg.IsEnabled(), + Threshold: cfg.Threshold, + Timeout: cfg.Timeout, + SuccessThreshold: cfg.SuccessThreshold, + } + if cb.Threshold <= 0 || cb.Timeout <= 0 || cb.SuccessThreshold <= 0 { + return nil + } + if err := r.SetCircuitBreakerConfig(cb); err != nil { + return fmt.Errorf("apply circuit breaker config: %w", err) + } + return nil +} + // SetCircuitStateObserver registers an observer that is notified of circuit // breaker state changes. The observer is called once immediately with the // current state (so a backing metric is correct from the start) and then on @@ -537,10 +574,23 @@ func (r *RemediatorRegistry) SetCircuitBreakerConfig(config CircuitBreakerConfig r.mu.Lock() defer r.mu.Unlock() r.circuitConfig = config + if config.Disabled && r.circuitState != CircuitClosed { + r.circuitState = CircuitClosed + r.circuitOpenedAt = time.Time{} + r.circuitLastStateChange = time.Now() + r.notifyCircuitStateObserver(r.circuitState) + } r.logInfof("Circuit breaker config updated: %+v", config) return nil } +// GetCircuitBreakerConfig returns the active circuit breaker configuration. +func (r *RemediatorRegistry) GetCircuitBreakerConfig() CircuitBreakerConfig { + r.mu.RLock() + defer r.mu.RUnlock() + return r.circuitConfig +} + // GetRegisteredTypes returns a sorted list of all registered remediator types. func (r *RemediatorRegistry) GetRegisteredTypes() []string { r.mu.RLock() @@ -757,11 +807,12 @@ func (r *RemediatorRegistry) Remediate(ctx context.Context, remediatorType strin return remediationErr } + dryRun := r.IsDryRun() r.logInfof("Executing remediation: type=%s problem=%s dry-run=%v", - remediatorType, GenerateProblemKey(problem), r.dryRun) + remediatorType, GenerateProblemKey(problem), dryRun) // Phase 5: Execute remediation (or skip in dry-run mode) - if r.dryRun { + if dryRun { r.logInfof("[DRY-RUN] Would execute remediation for %s", GenerateProblemKey(problem)) success = true r.recordCircuitBreakerSuccess() @@ -843,6 +894,10 @@ func (r *RemediatorRegistry) RemediateWithStrategies(ctx context.Context, strate // checkCircuitBreaker checks if the circuit breaker allows remediation. // This must be called with the lock held. func (r *RemediatorRegistry) checkCircuitBreaker() error { + if r.circuitConfig.Disabled { + return nil + } + switch r.circuitState { case CircuitClosed: // Normal operation @@ -925,13 +980,13 @@ func (r *RemediatorRegistry) checkPerMinuteRate() error { // recordRateLimitEntry records a successful remediation for rate limiting. func (r *RemediatorRegistry) recordRateLimitEntry() { + r.mu.Lock() + defer r.mu.Unlock() + if r.maxPerHour == 0 { return // Rate limiting disabled } - r.mu.Lock() - defer r.mu.Unlock() - r.remediationTimes = append(r.remediationTimes, time.Now()) } @@ -940,6 +995,10 @@ func (r *RemediatorRegistry) recordCircuitBreakerSuccess() { r.mu.Lock() defer r.mu.Unlock() + if r.circuitConfig.Disabled { + return + } + r.consecutiveFailures = 0 r.consecutiveSuccesses++ @@ -960,6 +1019,10 @@ func (r *RemediatorRegistry) recordCircuitBreakerFailure() { r.mu.Lock() defer r.mu.Unlock() + if r.circuitConfig.Disabled { + return + } + r.consecutiveSuccesses = 0 r.consecutiveFailures++ diff --git a/pkg/remediators/registry_applyconfig_test.go b/pkg/remediators/registry_applyconfig_test.go index b33bfc5..2543ccd 100644 --- a/pkg/remediators/registry_applyconfig_test.go +++ b/pkg/remediators/registry_applyconfig_test.go @@ -1,6 +1,8 @@ package remediators import ( + "context" + "strings" "testing" "time" @@ -131,3 +133,141 @@ func TestApplyConfigRejectsNil(t *testing.T) { t.Error("ApplyConfig(nil) must return an error") } } + +func TestNewRegistryFromConfigAppliesCircuitBreakerAtStartup(t *testing.T) { + r, err := NewRegistryFromConfig(&types.RemediationConfig{ + Enabled: true, + CircuitBreaker: types.CircuitBreakerConfig{ + Enabled: boolPtr(true), + Threshold: 5, + Timeout: 7 * time.Minute, + SuccessThreshold: 2, + }, + }, false) + if err != nil { + t.Fatalf("NewRegistryFromConfig: %v", err) + } + + got := r.GetCircuitBreakerConfig() + if got.Timeout != 7*time.Minute { + t.Errorf("circuit breaker timeout = %v, want 7m applied at startup without a reload", got.Timeout) + } + if got.Disabled { + t.Error("circuit breaker must be enabled when config says enabled") + } +} + +func TestNewRegistryFromConfigDefaults(t *testing.T) { + r, err := NewRegistryFromConfig(&types.RemediationConfig{Enabled: true}, true) + if err != nil { + t.Fatalf("NewRegistryFromConfig: %v", err) + } + + stats := r.GetStats() + if stats.MaxPerHour != types.DefaultMaxRemediationsPerHour { + t.Errorf("MaxPerHour = %d, want %d", stats.MaxPerHour, types.DefaultMaxRemediationsPerHour) + } + if stats.MaxHistory != types.DefaultHistorySize { + t.Errorf("MaxHistory = %d, want %d", stats.MaxHistory, types.DefaultHistorySize) + } + if !stats.DryRun { + t.Error("process-wide dry-run must be honored at startup") + } + if r.GetCircuitBreakerConfig() != DefaultCircuitBreakerConfig { + t.Errorf("an absent circuitBreaker block must leave the default, got %+v", r.GetCircuitBreakerConfig()) + } +} + +func TestNewRegistryFromConfigRejectsNil(t *testing.T) { + if _, err := NewRegistryFromConfig(nil, false); err == nil { + t.Error("NewRegistryFromConfig(nil) must return an error") + } +} + +func TestCircuitBreakerDisabledNeverBlocks(t *testing.T) { + r, err := NewRegistryFromConfig(&types.RemediationConfig{ + Enabled: true, + CircuitBreaker: types.CircuitBreakerConfig{ + Enabled: boolPtr(false), + Threshold: 1, + Timeout: time.Hour, + SuccessThreshold: 1, + }, + }, false) + if err != nil { + t.Fatalf("NewRegistryFromConfig: %v", err) + } + + mock := newMockRemediator("test", true) + r.Register(RemediatorInfo{ + Type: "test", + Factory: func() (types.Remediator, error) { return mock, nil }, + }) + problem := createTestProblem("test-type", "test-resource") + + for i := 0; i < 5; i++ { + mock.ClearCooldown(problem) + mock.ResetAttempts(problem) + err := r.Remediate(context.Background(), "test", problem) + if err == nil { + t.Fatalf("attempt %d: expected the failing remediator's error", i) + } + if strings.Contains(err.Error(), "circuit breaker") { + t.Fatalf("attempt %d: disabled circuit breaker must never block, got %v", i, err) + } + } + if mock.getCallCount() != 5 { + t.Errorf("remediator called %d times, want 5", mock.getCallCount()) + } + if r.GetCircuitState() != CircuitClosed { + t.Errorf("circuit state = %v, want Closed while disabled", r.GetCircuitState()) + } +} + +func TestApplyConfigDisablingCircuitBreakerClosesOpenCircuit(t *testing.T) { + r := NewRegistry(100, 100) + if err := r.SetCircuitBreakerConfig(CircuitBreakerConfig{Threshold: 1, Timeout: time.Hour, SuccessThreshold: 1}); err != nil { + t.Fatal(err) + } + r.recordCircuitBreakerFailure() + if r.GetCircuitState() != CircuitOpen { + t.Fatalf("test setup: circuit should be Open, got %v", r.GetCircuitState()) + } + + err := r.ApplyConfig(&types.RemediationConfig{ + Enabled: true, + CircuitBreaker: types.CircuitBreakerConfig{ + Enabled: boolPtr(false), + Threshold: 1, + Timeout: time.Hour, + SuccessThreshold: 1, + }, + }, false) + if err != nil { + t.Fatalf("ApplyConfig: %v", err) + } + if r.GetCircuitState() != CircuitClosed { + t.Errorf("disabling the breaker must report Closed, got %v", r.GetCircuitState()) + } +} + +func boolPtr(b bool) *bool { return &b } + +func TestNewRegistryFromConfigAbsentCircuitBreakerBlockEnablesBreaker(t *testing.T) { + cfg := types.RemediationConfig{Enabled: true} + if err := cfg.ApplyDefaults(); err != nil { + t.Fatal(err) + } + + r, err := NewRegistryFromConfig(&cfg, false) + if err != nil { + t.Fatalf("NewRegistryFromConfig: %v", err) + } + got := r.GetCircuitBreakerConfig() + if got.Disabled { + t.Fatal("omitting the circuitBreaker block must leave the breaker enabled") + } + if got.Threshold != types.DefaultCircuitBreakerThreshold || got.Timeout != 30*time.Minute || got.SuccessThreshold != 2 { + t.Errorf("breaker must run with the config defaults, got %+v", got) + } +} diff --git a/pkg/remediators/registry_test.go b/pkg/remediators/registry_test.go index 216a080..a1990a6 100644 --- a/pkg/remediators/registry_test.go +++ b/pkg/remediators/registry_test.go @@ -1849,3 +1849,46 @@ func TestRemediateWithStrategies(t *testing.T) { } }) } + +func runRemediationsWhileToggling(t *testing.T, registry *RemediatorRegistry, toggle func(i int)) { + t.Helper() + mock := newMockRemediator("test", false) + registry.Register(RemediatorInfo{ + Type: "test", + Factory: func() (types.Remediator, error) { return mock, nil }, + }) + problem := createTestProblem("test-type", "test-resource") + + stop := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; ; i++ { + select { + case <-stop: + return + default: + toggle(i) + } + } + }() + + for i := 0; i < 200; i++ { + mock.ClearCooldown(problem) + mock.ResetAttempts(problem) + _ = registry.Remediate(context.Background(), "test", problem) + } + close(stop) + wg.Wait() +} + +func TestSetDryRunConcurrentWithRemediate(t *testing.T) { + registry := NewRegistry(0, 100) + runRemediationsWhileToggling(t, registry, func(i int) { registry.SetDryRun(i%2 == 0) }) +} + +func TestSetMaxRemediationsPerHourConcurrentWithRemediate(t *testing.T) { + registry := NewRegistry(0, 100) + runRemediationsWhileToggling(t, registry, func(i int) { registry.SetMaxRemediationsPerHour((i % 2) * 1000) }) +} diff --git a/pkg/types/config.go b/pkg/types/config.go index 483d62e..62db5e1 100644 --- a/pkg/types/config.go +++ b/pkg/types/config.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "regexp" + "sort" "strings" "time" ) @@ -89,6 +90,21 @@ var ( MinCooldownPeriod = 10 * time.Second // Minimum cooldown between remediation attempts ) +// IsValidRemediationStrategy reports whether strategy names a known remediation strategy. +func IsValidRemediationStrategy(strategy string) bool { + return validRemediationStrategies[strategy] +} + +// ValidRemediationStrategies returns the known remediation strategy names, sorted. +func ValidRemediationStrategies() []string { + names := make([]string, 0, len(validRemediationStrategies)) + for name := range validRemediationStrategies { + names = append(names, name) + } + sort.Strings(names) + return names +} + // MonitorRegistryValidator provides an interface for validating monitor types // without creating an import cycle between config and monitors packages. // This interface is implemented by monitors.Registry. @@ -488,13 +504,22 @@ type RemediationCoordinationConfig struct { // CircuitBreakerConfig configures circuit breaker behavior. type CircuitBreakerConfig struct { - Enabled bool `json:"enabled" yaml:"enabled"` + // Enabled is a *bool so an absent key defaults to true while an explicit false disables + Enabled *bool `json:"enabled,omitempty" yaml:"enabled,omitempty"` Threshold int `json:"threshold,omitempty" yaml:"threshold,omitempty"` TimeoutString string `json:"timeout,omitempty" yaml:"timeout,omitempty"` Timeout time.Duration `json:"-" yaml:"-"` SuccessThreshold int `json:"successThreshold,omitempty" yaml:"successThreshold,omitempty"` } +// IsEnabled reports whether the circuit breaker is on; unset means enabled. +func (c *CircuitBreakerConfig) IsEnabled() bool { + if c == nil || c.Enabled == nil { + return true + } + return *c.Enabled +} + // RemediationOverride allows problem-specific remediation overrides. type RemediationOverride struct { Problem string `json:"problem" yaml:"problem"` @@ -1032,21 +1057,23 @@ func (r *RemediationConfig) ApplyDefaults() error { return fmt.Errorf("invalid cooldownPeriod %q: %w", r.CooldownPeriodString, err) } - // Circuit breaker defaults - if r.CircuitBreaker.Enabled { - if r.CircuitBreaker.Threshold == 0 { - r.CircuitBreaker.Threshold = DefaultCircuitBreakerThreshold - } - if r.CircuitBreaker.TimeoutString == "" { - r.CircuitBreaker.TimeoutString = DefaultCircuitBreakerTimeout - } - r.CircuitBreaker.Timeout, err = time.ParseDuration(r.CircuitBreaker.TimeoutString) - if err != nil { - return fmt.Errorf("invalid circuit breaker timeout %q: %w", r.CircuitBreaker.TimeoutString, err) - } - if r.CircuitBreaker.SuccessThreshold == 0 { - r.CircuitBreaker.SuccessThreshold = 2 - } + if r.CircuitBreaker.Enabled == nil { + enabled := true + r.CircuitBreaker.Enabled = &enabled + } + // Threshold defaults apply even when disabled so a later enable or reload has usable values + if r.CircuitBreaker.Threshold == 0 { + r.CircuitBreaker.Threshold = DefaultCircuitBreakerThreshold + } + if r.CircuitBreaker.TimeoutString == "" { + r.CircuitBreaker.TimeoutString = DefaultCircuitBreakerTimeout + } + r.CircuitBreaker.Timeout, err = time.ParseDuration(r.CircuitBreaker.TimeoutString) + if err != nil { + return fmt.Errorf("invalid circuit breaker timeout %q: %w", r.CircuitBreaker.TimeoutString, err) + } + if r.CircuitBreaker.SuccessThreshold == 0 { + r.CircuitBreaker.SuccessThreshold = 2 } // Apply defaults to overrides @@ -1301,8 +1328,8 @@ func (r *MonitorRemediationConfig) Validate() error { if r.Strategy == "" { return fmt.Errorf("strategy is required when remediation is enabled") } - if !validRemediationStrategies[r.Strategy] { - return fmt.Errorf("invalid strategy %q, must be one of: systemd-restart, custom-script, node-reboot, pod-delete, flush-dns, restart-interface, reset-routing, flush-ipv6-route", r.Strategy) + if !IsValidRemediationStrategy(r.Strategy) { + return fmt.Errorf("invalid strategy %q, must be one of: %s", r.Strategy, strings.Join(ValidRemediationStrategies(), ", ")) } // Strategy-specific validation @@ -1635,8 +1662,8 @@ func (r *RemediationConfig) Validate() error { return fmt.Errorf("historySize must be positive, got %d", r.HistorySize) } - // Validate circuit breaker - if r.CircuitBreaker.Enabled { + // Validate circuit breaker only when explicitly enabled; a nil Enabled means defaults have not run yet + if r.CircuitBreaker.Enabled != nil && *r.CircuitBreaker.Enabled { if r.CircuitBreaker.Threshold <= 0 { return fmt.Errorf("circuitBreaker.threshold must be positive, got %d", r.CircuitBreaker.Threshold) } diff --git a/pkg/types/config_test.go b/pkg/types/config_test.go index 7e2ebd8..bb427ec 100644 --- a/pkg/types/config_test.go +++ b/pkg/types/config_test.go @@ -209,7 +209,7 @@ func TestRemediationConfigApplyDefaults(t *testing.T) { name: "circuit breaker enabled gets defaults", input: RemediationConfig{ CircuitBreaker: CircuitBreakerConfig{ - Enabled: true, + Enabled: boolPtr(true), }, }, expected: RemediationConfig{ @@ -220,7 +220,7 @@ func TestRemediationConfigApplyDefaults(t *testing.T) { MaxAttemptsGlobal: DefaultMaxAttemptsGlobal, HistorySize: DefaultHistorySize, CircuitBreaker: CircuitBreakerConfig{ - Enabled: true, + Enabled: boolPtr(true), Threshold: DefaultCircuitBreakerThreshold, TimeoutString: DefaultCircuitBreakerTimeout, Timeout: 30 * time.Minute, @@ -240,7 +240,7 @@ func TestRemediationConfigApplyDefaults(t *testing.T) { name: "invalid circuit breaker timeout", input: RemediationConfig{ CircuitBreaker: CircuitBreakerConfig{ - Enabled: true, + Enabled: boolPtr(true), TimeoutString: "invalid", }, }, @@ -1476,7 +1476,7 @@ func TestRemediationConfigValidation(t *testing.T) { MaxAttemptsGlobal: 3, HistorySize: 100, CircuitBreaker: CircuitBreakerConfig{ - Enabled: true, + Enabled: boolPtr(true), Threshold: 0, Timeout: 30 * time.Minute, }, @@ -1863,7 +1863,7 @@ func TestComplexConfigurationValidation(t *testing.T) { MaxAttemptsGlobal: 5, HistorySize: 200, CircuitBreaker: CircuitBreakerConfig{ - Enabled: true, + Enabled: boolPtr(true), Threshold: 10, Timeout: 45 * time.Minute, SuccessThreshold: 3, @@ -3220,3 +3220,56 @@ func TestPrometheusExporterConfigApplyDefaults(t *testing.T) { } }) } + +func TestRemediationStrategyHelpers(t *testing.T) { + for _, name := range ValidRemediationStrategies() { + if !IsValidRemediationStrategy(name) { + t.Errorf("%q listed as valid but rejected", name) + } + } + if IsValidRemediationStrategy("webhook") { + t.Error("webhook must not be a valid strategy") + } + + err := (&MonitorRemediationConfig{Enabled: true, Strategy: "webhook"}).Validate() + if err == nil { + t.Fatal("expected an error for an unknown strategy") + } + for _, name := range ValidRemediationStrategies() { + if !strings.Contains(err.Error(), name) { + t.Errorf("error %q should list %q", err.Error(), name) + } + } +} + +func boolPtr(b bool) *bool { return &b } + +func TestRemediationConfigApplyDefaults_AbsentCircuitBreakerIsEnabled(t *testing.T) { + r := RemediationConfig{} + if err := r.ApplyDefaults(); err != nil { + t.Fatal(err) + } + if r.CircuitBreaker.Enabled == nil || !*r.CircuitBreaker.Enabled { + t.Error("an absent circuitBreaker block must default to enabled") + } + if !r.CircuitBreaker.IsEnabled() { + t.Error("IsEnabled must report true after defaults") + } + if !(&CircuitBreakerConfig{}).IsEnabled() { + t.Error("IsEnabled on a zero-value block must be true") + } +} + +func TestRemediationConfigApplyDefaults_DisabledCircuitBreakerStillGetsDefaults(t *testing.T) { + r := RemediationConfig{CircuitBreaker: CircuitBreakerConfig{Enabled: boolPtr(false)}} + if err := r.ApplyDefaults(); err != nil { + t.Fatal(err) + } + cb := r.CircuitBreaker + if cb.IsEnabled() { + t.Error("ApplyDefaults must not flip an explicit false") + } + if cb.Threshold != DefaultCircuitBreakerThreshold || cb.Timeout != 30*time.Minute || cb.SuccessThreshold != 2 { + t.Errorf("disabled circuit breaker must still get usable defaults, got %+v", cb) + } +} diff --git a/scripts/README.md b/scripts/README.md index ed2ec5c..38320eb 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,359 +1,7 @@ -# Validation Scripts +# Scripts -This directory contains validation scripts that mirror CI/CD pipeline checks, allowing local validation before pushing code. - -## Core Validation Scripts - -### validate-pipeline-local.sh ⭐ PRIMARY SCRIPT - -**Purpose**: Mirrors the complete CI/CD pipeline validation locally - -**Usage**: -```bash -# Full validation (mirrors CI/CD exactly) -./scripts/validate-pipeline-local.sh - -# Quick validation (format, vet, staticcheck only - no tests) -./scripts/validate-pipeline-local.sh --quick - -# Validate specific component -./scripts/validate-pipeline-local.sh --component api -``` - -**What it validates**: -- Go version compliance -- Repository cleanliness -- Code formatting (gofmt) -- Go vet checks -- Static analysis (staticcheck) -- Security scanning (gosec) -- Unit tests with coverage -- Component-specific validations - -**Exit codes**: -- `0` - All validations passed -- `1` - One or more validations failed - ---- - -### validate-go-version.sh - -**Purpose**: Ensures Go version consistency across project - -**Usage**: -```bash -./scripts/validate-go-version.sh -``` - -**Checks**: -- Installed Go version matches go.mod requirement -- go.mod specifies correct Go version -- Dockerfiles use correct Go version - -**Customization**: -Edit the script to change the required Go version (currently 1.24) - ---- - -### validate-repo-cleanliness.sh - -**Purpose**: Enforces repository structure rules - -**Usage**: -```bash -./scripts/validate-repo-cleanliness.sh -``` - -**Checks**: -- No Go files in repository root -- No build artifacts in repository -- No temporary files committed -- No secrets or credentials -- Proper directory organization - -**Violations result in**: -- Detailed error messages -- Suggested fixes -- Non-zero exit code - ---- - -### validate-api.sh (Example Component Validation) - -**Purpose**: Component-specific validation for API - -**Usage**: -```bash -./scripts/validate-api.sh -``` - -**What it does**: -- Runs API-specific tests -- Validates API configuration -- Checks OpenAPI/Swagger documentation -- Verifies handler structure - -**Customization**: Copy this pattern for other components - ---- - -## Integration with Workflow - -### Pre-Push Validation - -Enable automatic pre-push validation: - -```bash -# Setup git hooks (in .githooks/) -git config core.hooksPath .githooks -``` - -### Makefile Integration - -These scripts are integrated into the makefile: - -```bash -# Full validation -make validate-pipeline-local - -# Quick validation -make validate-quick - -# Component validation -make validate-component COMPONENT=api -``` - -### CI/CD Integration - -The CI/CD pipeline uses these same scripts: - -```yaml -# .github/workflows/pipeline-v2.yml -validation: - - run: ./scripts/validate-go-version.sh - - run: ./scripts/validate-repo-cleanliness.sh - - run: ./scripts/validate-pipeline-local.sh -``` - -## Creating Component Validation Scripts - -To add validation for a new component: - -### 1. Create Component Script - -```bash -cp scripts/validate-api.sh scripts/validate-mycomponent.sh -``` - -### 2. Customize for Component - -```bash -#!/bin/bash -# validate-mycomponent.sh - -set -e - -echo "Validating my component..." - -# Component-specific checks -cd cmd/mycomponent -go test ./... -v -go vet ./... - -# Additional validation -# ... - -echo "✅ My component validation passed" -``` - -### 3. Make Executable - -```bash -chmod +x scripts/validate-mycomponent.sh -``` - -### 4. Add to Pipeline Script - -Edit `validate-pipeline-local.sh` to include your component: - -```bash -COMPONENTS=("api" "agent" "mycomponent") -``` - -### 5. Add Makefile Target - -```makefile -validate-mycomponent-local: - @./scripts/validate-mycomponent.sh -``` - -## Validation Configuration - -Configuration is stored in `.validation.json`: - -```json -{ - "validation": { - "enabled": true, - "parallel_jobs": 4, - "quick_mode": { - "skip_tests": true, - "skip_gosec": true - } - }, - "components": { - "api": { - "enabled": true, - "test_timeout": "10m", - "coverage_threshold": 80 - } - }, - "tools": { - "staticcheck": { - "enabled": true, - "checks": ["all", "-ST1000"] - }, - "gosec": { - "enabled": true, - "severity": "medium" - } - } -} -``` - -## Exit Codes - -All validation scripts follow these exit code conventions: - -- `0` - Success, all checks passed -- `1` - Failure, one or more checks failed -- `2` - Configuration error or missing dependencies - -## Best Practices - -### 1. Always Validate Locally Before Pushing - -```bash -make validate-pipeline-local -``` - -This prevents CI/CD failures and saves time. - -### 2. Use Quick Mode During Development - -```bash -make validate-quick -``` - -Run quick validation frequently, full validation before push. - -### 3. Fix Issues Systematically - -When validation fails: -1. Read the error message carefully -2. Fix the reported issue -3. Re-run validation -4. Don't commit until validation passes - -### 4. Keep Scripts Updated - -When changing: -- Go version → Update validate-go-version.sh -- Repository structure → Update validate-repo-cleanliness.sh -- Component structure → Update component validation scripts - -### 5. Test Scripts Locally - -Before committing script changes: -```bash -# Test success case -./scripts/validate-pipeline-local.sh - -# Test failure case (introduce an error) -echo "bad code" > test.go -./scripts/validate-pipeline-local.sh -git checkout test.go -``` - -## Troubleshooting - -### Script Permission Denied - -```bash -chmod +x scripts/*.sh -``` - -### Go Version Mismatch - -```bash -# Check installed version -go version - -# Check required version -grep "^go" go.mod - -# Update Go if needed -``` - -### Validation Takes Too Long - -Use quick mode during development: -```bash -./scripts/validate-pipeline-local.sh --quick -``` - -### Component Not Found - -Add component to validate-pipeline-local.sh: -```bash -COMPONENTS=("api" "agent" "your-component") -``` - -## Integration Testing - -Test that validation correctly catches issues: - -```bash -# Test 1: Should pass -make validate-pipeline-local - -# Test 2: Should fail on formatting -echo "package main;func main(){}" > cmd/test/bad.go -make validate-pipeline-local -git checkout cmd/test/bad.go - -# Test 3: Should fail on cleanliness -touch root-file.go -make validate-pipeline-local -rm root-file.go -``` - -## Performance - -### Validation Times (Typical) - -- **Quick mode**: 30-60 seconds -- **Full validation**: 2-5 minutes -- **Component validation**: 30-120 seconds per component - -### Optimization Tips - -1. **Use parallel execution** (already configured) -2. **Skip tests in quick mode** for rapid iteration -3. **Cache Go modules** to speed up builds -4. **Use component validation** when working on specific areas - -## Documentation - -For complete validation workflow documentation, see: -- [Local CI/CD Validation Guide](../docs/development/local-cicd-validation-guide.md) -- [Task Execution Workflow](../docs/development/task-execution-workflow.md) - -## Contributing - -When adding new validation: -1. Create a focused validation script -2. Add to validate-pipeline-local.sh -3. Document in this README -4. Test both success and failure cases -5. Update .validation.json if needed +- `validate-pipeline-local.sh` - runs the CI checks locally (gofmt, go vet, golangci-lint, gosec, go test, helm lint, helm-verify-generated). `--quick` skips tests; `SKIP_LINT=1` allows a missing golangci-lint. Wired to `make validate-pipeline-local` and `make validate-quick`. +- `validate-go-version.sh` - checks that go.mod and the Dockerfiles use the required Go minor version. Wired to `make check-go-version`. +- `validate-repo-cleanliness.sh` - checks for stray Go files in the repo root, compiled binaries outside `bin/`, and editor temp files. +- `test-monitors.sh` - exercises the monitors on a real cluster node, grouped by risk level (`--safe`, `--caution`, `--destructive`). Run as root on a dedicated test node. +- `test-vmxnet3-detection.sh` - injects synthetic vmxnet3 kernel messages on a lab node to verify pattern detection. Run as root. diff --git a/scripts/check-doc-links.sh b/scripts/check-doc-links.sh new file mode 100755 index 0000000..0b60405 --- /dev/null +++ b/scripts/check-doc-links.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Reports relative markdown links whose target does not exist. Exits 1 if any are broken. +set -u +cd "$(dirname "$0")/.." || exit 2 + +status=0 +while IFS= read -r file; do + dir=$(dirname "$file") + while IFS= read -r target; do + target=${target%% \"*} + target=${target%%#*} + [ -z "$target" ] && continue + case "$target" in http://*|https://*|mailto:*) continue ;; esac + if [ ! -e "$dir/$target" ]; then + echo "$file: broken -> $target" + status=1 + fi + done < <(grep -o '](<\{0,1\}[^)]*)' "$file" | sed 's/^](<\{0,1\}//; s/>\{0,1\})$//') +done < <(find . -name '*.md' -not -path './.git/*' -not -path './.claude/*' | sort) + +exit $status diff --git a/scripts/init-from-template.sh b/scripts/init-from-template.sh deleted file mode 100755 index 808bf31..0000000 --- a/scripts/init-from-template.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/bin/bash -# Template Initialization Script -# -# This script initializes a new project from the Nexmonyx repository template. -# It performs variable substitution, sets up git hooks, and validates the setup. - -set -e - -# Colors -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' - -echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" -echo -e "${BLUE} Nexmonyx Template Initialization${NC}" -echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" -echo "" -echo "This script will:" -echo " 1. Substitute template variables with your project values" -echo " 2. Set up git hooks for validation" -echo " 3. Validate the setup" -echo " 4. Provide next steps" -echo "" - -# Check prerequisites -echo -e "${BLUE}[1/5] Checking prerequisites...${NC}" - -command -v git >/dev/null 2>&1 || { echo -e "${RED}Error: git not found${NC}"; exit 1; } -command -v go >/dev/null 2>&1 || { echo -e "${YELLOW}Warning: go not found (required for Go projects)${NC}"; } - -echo -e "${GREEN}✓ Prerequisites checked${NC}" -echo "" - -# Run variable substitution -echo -e "${BLUE}[2/5] Variable substitution...${NC}" - -if [ ! -f "scripts/substitute-variables.sh" ]; then - echo -e "${RED}Error: substitute-variables.sh not found${NC}" - exit 1 -fi - -chmod +x scripts/substitute-variables.sh -./scripts/substitute-variables.sh - -echo "" - -# Set up git hooks -echo -e "${BLUE}[3/5] Setting up git hooks...${NC}" - -if [ -f ".githooks/setup.sh" ]; then - chmod +x .githooks/setup.sh - cd .githooks && ./setup.sh && cd .. - echo -e "${GREEN}✓ Git hooks configured${NC}" -else - echo -e "${YELLOW}Warning: Git hooks not found, skipping${NC}" -fi - -echo "" - -# Validate setup -echo -e "${BLUE}[4/5] Validating setup...${NC}" - -# Check for remaining template variables -remaining=$(grep -r "{{.*}}" . \ - --exclude-dir=".git" \ - --exclude-dir="node_modules" \ - --exclude-dir="bin" \ - --exclude="*.png" --exclude="*.jpg" --exclude="*.jpeg" \ - 2>/dev/null | wc -l | tr -d ' ') - -if [ "$remaining" -gt 0 ]; then - echo -e "${YELLOW}Warning: $remaining files still contain template variables${NC}" - echo " Run: grep -r '{{.*}}' . to find them" -else - echo -e "${GREEN}✓ No remaining template variables found${NC}" -fi - -echo "" - -# Provide next steps -echo -e "${BLUE}[5/5] Next steps...${NC}" -echo "" -echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" -echo -e "${GREEN} Initialization Complete!${NC}" -echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" -echo "" -echo "Your project has been initialized. Next steps:" -echo "" -echo "1. Review the changes:" -echo " git diff" -echo "" -echo "2. Customize for your project:" -echo " - Update CLAUDE.md with project-specific instructions" -echo " - Add your specialized agents to .claude/agents/specialized/" -echo " - Update makefile with your components" -echo " - Configure .validation.json for your components" -echo "" -echo "3. Initialize or update git repository:" -echo " git init # If new repository" -echo " git add ." -echo " git commit -m \"Initialize from Nexmonyx template\"" -echo "" -echo "4. Set up remote repository (optional):" -echo " git remote add origin https://github.com/YOUR_ORG/YOUR_REPO.git" -echo " git push -u origin main" -echo "" -echo "5. Read the documentation:" -echo " - docs/README.md # Documentation hub" -echo " - docs/development/task-execution-workflow.md # Development process" -echo " - .claude/AGENT_GUIDE.md # How to use agents" -echo "" -echo "6. Start development:" -echo " make help # See available commands" -echo " make build-local # Build your project" -echo " make test-local # Run tests" -echo " make validate-pipeline-local # Validate before pushing" -echo "" -echo -e "${BLUE}For complete setup instructions, see: TEMPLATE_USAGE.md${NC}" -echo "" diff --git a/scripts/setup-gpg-signing-subkey.sh b/scripts/setup-gpg-signing-subkey.sh deleted file mode 100755 index 5679957..0000000 --- a/scripts/setup-gpg-signing-subkey.sh +++ /dev/null @@ -1,253 +0,0 @@ -#!/bin/bash -set -e - -# setup-gpg-signing-subkey.sh -# Creates a dedicated signing subkey for GitHub Actions (safer than using main key) - -echo "🔐 Setting up GPG Signing Subkey for Release Automation" -echo "================================================================" -echo "" -echo "WHY USE A SUBKEY?" -echo "- Protects your main GPG key from compromise" -echo "- Subkey can be revoked independently if GitHub is compromised" -echo "- Best practice for automated signing systems" -echo "" -echo "================================================================" -echo "" - -# Colors -GREEN='\033[0;32m' -RED='\033[0;31m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -MAIN_KEY_ID="1AF32A8B0481A7F3" - -# Step 1: Verify main key exists -echo "1ī¸âƒŖ Verifying main GPG key..." -if ! gpg --list-secret-keys "$MAIN_KEY_ID" > /dev/null 2>&1; then - echo -e "${RED}❌ Main key $MAIN_KEY_ID not found${NC}" - exit 1 -fi -echo -e "${GREEN}✅ Main key found${NC}" -echo "" - -# Step 2: Check if signing subkey already exists -echo "2ī¸âƒŖ Checking for existing signing subkey..." -EXISTING_SUBKEY=$(gpg --list-secret-keys --with-subkey-fingerprints "$MAIN_KEY_ID" | grep -A 1 "\[S\]" | tail -1 | awk '{print $1}' || echo "") - -if [ -n "$EXISTING_SUBKEY" ]; then - echo -e "${YELLOW}âš ī¸ Found existing signing subkey: $EXISTING_SUBKEY${NC}" - echo "" - read -p "Do you want to use this existing subkey? (y/n): " USE_EXISTING - if [ "$USE_EXISTING" = "y" ]; then - SUBKEY_ID="$EXISTING_SUBKEY" - echo -e "${GREEN}✅ Using existing subkey${NC}" - SKIP_CREATION=true - else - echo "Will create a new signing subkey..." - SKIP_CREATION=false - fi -else - echo "No existing signing subkey found. Will create new one." - SKIP_CREATION=false -fi -echo "" - -# Step 3: Create signing subkey (if needed) -if [ "$SKIP_CREATION" != "true" ]; then - echo "3ī¸âƒŖ Creating dedicated signing subkey..." - echo "" - echo -e "${BLUE}â„šī¸ You will be prompted to:${NC}" - echo " 1. Enter your main key passphrase" - echo " 2. Choose subkey type: Select '(4) RSA (sign only)'" - echo " 3. Choose key size: Recommend 4096" - echo " 4. Choose expiration: Recommend 2y (2 years)" - echo "" - read -p "Press Enter to continue..." - echo "" - - # Interactive subkey creation - gpg --expert --edit-key "$MAIN_KEY_ID" < "$EXPORT_FILE" 2>/dev/null; then - echo -e "${GREEN}✅ Subkey exported to: $EXPORT_FILE${NC}" - echo "" - - # Verify it's only the subkey (not main key) - echo "5ī¸âƒŖ Verifying export contains only subkey..." - TEMP_KEYRING=$(mktemp -d) - if gpg --homedir "$TEMP_KEYRING" --import "$EXPORT_FILE" 2>/dev/null; then - # Check if main key secret is present - if gpg --homedir "$TEMP_KEYRING" --list-secret-keys | grep -q "sec#"; then - echo -e "${GREEN}✅ Export verified: Contains subkey only (main key stub)${NC}" - echo " The '#' after 'sec' means main key is NOT included (safe!)" - else - echo -e "${YELLOW}âš ī¸ Warning: Main key might be included${NC}" - fi - fi - rm -rf "$TEMP_KEYRING" -else - echo -e "${RED}❌ Failed to export subkey${NC}" - exit 1 -fi -echo "" - -# Step 5: Create passphrase for automated signing -echo "6ī¸âƒŖ Subkey passphrase setup..." -echo "" -echo -e "${BLUE}â„šī¸ The subkey currently uses your main key's passphrase.${NC}" -echo "For GitHub Actions, you have two options:" -echo "" -echo "Option A (RECOMMENDED): Remove passphrase from subkey" -echo " - Safer because subkey can be revoked independently" -echo " - No passphrase to manage in GitHub Secrets" -echo " - Still requires passphrase for main key operations" -echo "" -echo "Option B: Use main key passphrase in GitHub Secrets" -echo " - Keep subkey passphrase protection" -echo " - Must store passphrase in GitHub Secrets" -echo "" -read -p "Remove passphrase from subkey? (recommended: y/n): " REMOVE_PASS - -if [ "$REMOVE_PASS" = "y" ]; then - echo "" - echo "Removing passphrase from signing subkey..." - echo -e "${YELLOW}You will be prompted for your CURRENT passphrase, then press Enter (blank) for new passphrase${NC}" - - gpg --edit-key "$MAIN_KEY_ID" < "$EXPORT_FILE" - NEEDS_PASSPHRASE=false - echo -e "${GREEN}✅ Passphrase removed from subkey${NC}" -else - NEEDS_PASSPHRASE=true - echo -e "${YELLOW}âš ī¸ Subkey still requires passphrase${NC}" -fi -echo "" - -# Step 6: GitHub configuration instructions -echo "================================================================" -echo "7ī¸âƒŖ GitHub Secrets Configuration" -echo "================================================================" -echo "" -echo -e "${GREEN}✅ Subkey ready for GitHub Actions!${NC}" -echo "" -echo "Next steps:" -echo "" -echo "1. Add the subkey to GitHub Secrets:" -echo "" -echo -e " ${BLUE}Via GitHub CLI:${NC}" -echo " gh secret set GPG_PRIVATE_KEY < $EXPORT_FILE" -echo "" -echo -e " ${BLUE}Via GitHub UI:${NC}" -echo " - Go to: https://github.com/supporttools/node-doctor/settings/secrets/actions" -echo " - Click 'New repository secret'" -echo " - Name: GPG_PRIVATE_KEY" -echo " - Value: " -echo "" - -if [ "$NEEDS_PASSPHRASE" = "true" ]; then - echo "2. Add passphrase to GitHub Secrets:" - echo "" - echo -e " ${BLUE}Via GitHub CLI:${NC}" - echo " gh secret set GPG_PASSPHRASE # will prompt for value" - echo "" - echo -e " ${BLUE}Via GitHub UI:${NC}" - echo " - Click 'New repository secret'" - echo " - Name: GPG_PASSPHRASE" - echo " - Value: " - echo "" -else - echo "2. No passphrase needed (you removed it from the subkey)" - echo "" -fi - -echo "3. Publish public key for verification:" -echo "" -echo " gpg --armor --export $MAIN_KEY_ID > node-doctor-signing-key.asc" -echo " # Upload to: docs/signing-keys/node-doctor-signing-key.asc" -echo " # Or publish to keyserver:" -echo " gpg --send-keys $MAIN_KEY_ID" -echo "" - -echo "4. Update release documentation with key ID:" -echo " Main key ID: $MAIN_KEY_ID" -echo " Signing subkey: $SUBKEY_ID" -echo "" - -echo "================================================================" -echo "🔐 Security Notes" -echo "================================================================" -echo "" -echo -e "${GREEN}✅ Your main GPG key is safe!${NC}" -echo " - Main key never leaves your machine" -echo " - Only signing subkey is uploaded to GitHub" -echo " - Subkey can be revoked independently if needed" -echo "" -echo -e "${BLUE}â„šī¸ To revoke the subkey later (if GitHub is compromised):${NC}" -echo " gpg --edit-key $MAIN_KEY_ID" -echo " > key $SUBKEY_ID" -echo " > revkey" -echo " > save" -echo "" -echo -e "${YELLOW}âš ī¸ Remember to securely delete the exported file after uploading:${NC}" -echo " shred -u $EXPORT_FILE" -echo "" - -# Step 7: Verification -echo "================================================================" -echo "8ī¸âƒŖ Verification" -echo "================================================================" -echo "" -echo "To verify the setup works:" -echo "" -echo "1. Run the verification script:" -echo " ./scripts/verify-gpg-setup.sh" -echo "" -echo "2. Test with a pre-release tag:" -echo " git tag v0.1.0-rc.1" -echo " git push origin v0.1.0-rc.1" -echo "" -echo "3. Check GitHub Actions workflow for GPG signing steps" -echo "" -echo "4. Verify release artifacts have both signatures:" -echo " - *.tar.gz.cosign.sig (Cosign/GitHub OIDC)" -echo " - *.tar.gz.asc (GPG subkey signature)" -echo "" diff --git a/scripts/substitute-variables.sh b/scripts/substitute-variables.sh deleted file mode 100755 index b36edec..0000000 --- a/scripts/substitute-variables.sh +++ /dev/null @@ -1,135 +0,0 @@ -#!/bin/bash -# Variable Substitution Script -# -# This script replaces template variables with your project values. -# It searches for {{VARIABLE}} patterns and replaces them with your values. - -set -e - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" -echo -e "${BLUE} Nexmonyx Template Variable Substitution${NC}" -echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" -echo "" - -# Check if running in repository root -if [ ! -f "CLAUDE.md" ] || [ ! -d ".claude" ]; then - echo -e "${RED}Error: Must run from repository root${NC}" - exit 1 -fi - -# Function to prompt for variable -prompt_variable() { - local var_name=$1 - local description=$2 - local default_value=$3 - local value - - echo -e "${YELLOW}${var_name}${NC}" - echo -e " ${description}" - if [ -n "$default_value" ]; then - read -p " Enter value (default: ${default_value}): " value - value=${value:-$default_value} - else - read -p " Enter value: " value - fi - - echo "$value" -} - -# Collect variables from user -echo -e "${BLUE}Please provide values for template variables:${NC}" -echo "" - -PROJECT_NAME=$(prompt_variable "PROJECT_NAME" "Your project name (lowercase, no spaces)" "my-project") -GITHUB_ORG=$(prompt_variable "GITHUB_ORG" "Your GitHub organization or username" "") -REGISTRY=$(prompt_variable "REGISTRY" "Container registry (e.g., ghcr.io/myorg, harbor.example.com/myproject)" "ghcr.io/${GITHUB_ORG}") -HELM_REPO=$(prompt_variable "HELM_REPO" "Helm chart repository URL" "https://charts.example.com") -PRIMARY_DOMAIN=$(prompt_variable "PRIMARY_DOMAIN" "Primary domain for your project" "example.com") -GO_VERSION=$(prompt_variable "GO_VERSION" "Go version" "1.24") - -echo "" -echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" -echo -e "${BLUE} Summary of Values${NC}" -echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" -echo "PROJECT_NAME: $PROJECT_NAME" -echo "GITHUB_ORG: $GITHUB_ORG" -echo "REGISTRY: $REGISTRY" -echo "HELM_REPO: $HELM_REPO" -echo "PRIMARY_DOMAIN: $PRIMARY_DOMAIN" -echo "GO_VERSION: $GO_VERSION" -echo "" - -read -p "Proceed with substitution? (y/n): " confirm -if [ "$confirm" != "y" ] && [ "$confirm" != "Y" ]; then - echo -e "${YELLOW}Substitution cancelled${NC}" - exit 0 -fi - -echo "" -echo -e "${BLUE}Performing substitutions...${NC}" - -# Find all files that might contain variables (excluding binary files, .git, and node_modules) -files=$(find . -type f \ - -not -path "*/\.git/*" \ - -not -path "*/node_modules/*" \ - -not -path "*/bin/*" \ - -not -path "*/.claude/settings.local.json" \ - -not -name "*.png" \ - -not -name "*.jpg" \ - -not -name "*.jpeg" \ - -not -name "*.gif" \ - -not -name "*.ico" \ - -not -name "*.pdf" \ - -not -name "*.woff*" \ - -not -name "*.ttf" \ - -not -name "*.eot") - -count=0 - -for file in $files; do - # Check if file contains any template variables - if grep -q "{{.*}}" "$file" 2>/dev/null; then - echo -e " ${GREEN}✓${NC} $file" - - # Perform substitutions (macOS and Linux compatible) - if [[ "$OSTYPE" == "darwin"* ]]; then - # macOS - sed -i '' "s|{{PROJECT_NAME}}|$PROJECT_NAME|g" "$file" - sed -i '' "s|{{GITHUB_ORG}}|$GITHUB_ORG|g" "$file" - sed -i '' "s|{{REGISTRY}}|$REGISTRY|g" "$file" - sed -i '' "s|{{HELM_REPO}}|$HELM_REPO|g" "$file" - sed -i '' "s|{{PRIMARY_DOMAIN}}|$PRIMARY_DOMAIN|g" "$file" - sed -i '' "s|{{GO_VERSION}}|$GO_VERSION|g" "$file" - else - # Linux - sed -i "s|{{PROJECT_NAME}}|$PROJECT_NAME|g" "$file" - sed -i "s|{{GITHUB_ORG}}|$GITHUB_ORG|g" "$file" - sed -i "s|{{REGISTRY}}|$REGISTRY|g" "$file" - sed -i "s|{{HELM_REPO}}|$HELM_REPO|g" "$file" - sed -i "s|{{PRIMARY_DOMAIN}}|$PRIMARY_DOMAIN|g" "$file" - sed -i "s|{{GO_VERSION}}|$GO_VERSION|g" "$file" - fi - - ((count++)) - fi -done - -echo "" -echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" -echo -e "${GREEN} Substitution Complete!${NC}" -echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" -echo -e "${GREEN} Updated $count files${NC}" -echo "" -echo "Next steps:" -echo "1. Review the changes: git diff" -echo "2. Update additional project-specific files as needed" -echo "3. Initialize git: git init (if not already initialized)" -echo "4. Commit the changes: git add . && git commit -m \"Initialize from template\"" -echo "" diff --git a/scripts/validate-api.sh b/scripts/validate-api.sh deleted file mode 100755 index f569b2b..0000000 --- a/scripts/validate-api.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -# API-specific validation script - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -"$SCRIPT_DIR/validate-all.sh" --component api \ No newline at end of file diff --git a/scripts/validate-pipeline-local.sh b/scripts/validate-pipeline-local.sh index ad23355..410f48d 100755 --- a/scripts/validate-pipeline-local.sh +++ b/scripts/validate-pipeline-local.sh @@ -1,383 +1,122 @@ -#!/bin/bash -# validate-pipeline-local.sh -# Mirrors CI/CD pipeline validation exactly for local testing -# This script validates node-doctor with the same checks as .github/workflows/ci.yml - -set -e - -# Color codes for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Counters for summary -TOTAL_CHECKS=0 -PASSED_CHECKS=0 -FAILED_CHECKS=0 - -# Components to validate (node-doctor is a single binary) -COMPONENTS=("node-doctor") - -# Parse command line arguments -SPECIFIC_COMPONENT="" -QUICK_MODE=false -PARALLEL_MODE=true -FAIL_FAST=false +#!/usr/bin/env bash +# Runs the checks from .github/workflows/ci.yml against the local tree. +set -uo pipefail usage() { - cat << EOF -Usage: $0 [OPTIONS] - -Validates node-doctor using the same checks as CI/CD pipeline. - -OPTIONS: - --component=NAME Validate only specific component (node-doctor) - --quick Skip tests and gosec (format, vet, staticcheck only) - --sequential Run components sequentially instead of parallel - --fail-fast Stop on first failure - -h, --help Show this help message - -EXAMPLES: - $0 # Validate node-doctor - $0 --component=node-doctor # Validate node-doctor (explicit) - $0 --quick # Quick validation (no tests) - $0 --fail-fast # Stop on first error + cat <&2; exit 1 ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $arg" >&2; usage >&2; exit 1 ;; esac done -# Validate specific component if provided -if [ -n "$SPECIFIC_COMPONENT" ]; then - VALID_COMPONENT=false - for comp in "${COMPONENTS[@]}"; do - if [ "$comp" = "$SPECIFIC_COMPONENT" ]; then - VALID_COMPONENT=true - break - fi - done - - if [ "$VALID_COMPONENT" = false ]; then - echo -e "${RED}Error: Invalid component '$SPECIFIC_COMPONENT'${NC}" - echo "Valid components: ${COMPONENTS[*]}" - exit 1 - fi - - COMPONENTS=("$SPECIFIC_COMPONENT") -fi - -# Print header -echo -e "${BLUE}╔════════════════════════════════════════════════════════════════╗${NC}" -echo -e "${BLUE}║ Local CI/CD Pipeline Validation ║${NC}" -echo -e "${BLUE}╚════════════════════════════════════════════════════════════════╝${NC}" -echo "" -echo -e "${BLUE}Components:${NC} ${COMPONENTS[*]}" -echo -e "${BLUE}Quick Mode:${NC} $QUICK_MODE" -echo -e "${BLUE}Parallel:${NC} $PARALLEL_MODE" -echo -e "${BLUE}Fail Fast:${NC} $FAIL_FAST" -echo "" - -# Check for required tools -echo -e "${BLUE}Checking required tools...${NC}" -MISSING_TOOLS=() - -if ! command -v gofmt &> /dev/null; then - MISSING_TOOLS+=("gofmt") -fi - -if ! command -v go &> /dev/null; then - MISSING_TOOLS+=("go") -else - GO_VERSION=$(go version | awk '{print $3}' | sed 's/go//') - if [[ ! "$GO_VERSION" =~ ^1\.24 ]]; then - echo -e "${YELLOW}Warning: Go version $GO_VERSION detected. CI/CD uses Go 1.24${NC}" - fi -fi - -if ! command -v staticcheck &> /dev/null; then - echo -e "${YELLOW}Installing staticcheck...${NC}" - go install honnef.co/go/tools/cmd/staticcheck@latest -fi - -if [ "$QUICK_MODE" = false ] && ! command -v gosec &> /dev/null; then - echo -e "${YELLOW}Installing gosec...${NC}" - go install github.com/securego/gosec/v2/cmd/gosec@latest -fi - -if [ ${#MISSING_TOOLS[@]} -gt 0 ]; then - echo -e "${RED}Error: Missing required tools: ${MISSING_TOOLS[*]}${NC}" - exit 1 -fi - -echo -e "${GREEN}✓ All required tools available${NC}" -echo "" - -# Function to run a validation check -run_check() { - local component=$1 - local check_name=$2 - local command=$3 +cd "$(dirname "$0")/.." - TOTAL_CHECKS=$((TOTAL_CHECKS + 1)) +PASSED=() +FAILED=() +SKIPPED=() - echo -e "${BLUE}[$component]${NC} Running $check_name..." - - if eval "$command" > /tmp/validate_${component}_${check_name}.log 2>&1; then - echo -e "${GREEN}✓ [$component] $check_name passed${NC}" - PASSED_CHECKS=$((PASSED_CHECKS + 1)) - return 0 +run() { + local name=$1 + shift + echo "==> $name" + if "$@"; then + PASSED+=("$name") else - echo -e "${RED}✗ [$component] $check_name FAILED${NC}" - cat /tmp/validate_${component}_${check_name}.log - FAILED_CHECKS=$((FAILED_CHECKS + 1)) - - if [ "$FAIL_FAST" = true ]; then - echo -e "${RED}Fail-fast enabled. Stopping validation.${NC}" - exit 1 - fi - - return 1 + FAILED+=("$name") + echo "FAILED: $name" >&2 fi } -# Function to validate a single component -validate_component() { - local component=$1 - local component_failed=false - - echo -e "${BLUE}════════════════════════════════════════════════════════════════${NC}" - echo -e "${BLUE}Validating: $component${NC}" - echo -e "${BLUE}════════════════════════════════════════════════════════════════${NC}" - - # Step 1: Format check (gofmt) - case "$component" in - api) - run_check "$component" "format" "test -z \"\$(find api -name '*.go' -not -path '*/vendor/*' -exec gofmt -l {} +)\"" || component_failed=true - ;; - pkg) - run_check "$component" "format" "test -z \"\$(find pkg -name '*.go' -not -path '*/vendor/*' -exec gofmt -l {} +)\"" || component_failed=true - ;; - org-management-controller) - run_check "$component" "format" "test -z \"\$(find pkg/controller/org-management -name '*.go' -not -path '*/vendor/*' -exec gofmt -l {} +)\"" || component_failed=true - ;; - probe-controller) - run_check "$component" "format" "test -z \"\$(find pkg/probe -name '*.go' -not -path '*/vendor/*' -exec gofmt -l {} +)\"" || component_failed=true - ;; - ingestor) - run_check "$component" "format" "test -z \"\$(find pkg/ingestor -name '*.go' -not -path '*/vendor/*' -exec gofmt -l {} +)\"" || component_failed=true - ;; - customer-probe-agent) - run_check "$component" "format" "test -z \"\$(find cmd/customer-probe-agent -name '*.go' -not -path '*/vendor/*' -exec gofmt -l {} +)\"" || component_failed=true - ;; - esac - - # Step 2: Go vet - case "$component" in - api) - run_check "$component" "go-vet" "(cd api && go vet ./...)" || component_failed=true - ;; - pkg) - run_check "$component" "go-vet" "go vet ./pkg/..." || component_failed=true - ;; - org-management-controller) - run_check "$component" "go-vet" "go vet ./pkg/controller/org-management/..." || component_failed=true - ;; - probe-controller) - run_check "$component" "go-vet" "go vet ./pkg/probe/..." || component_failed=true - ;; - ingestor) - run_check "$component" "go-vet" "go vet ./pkg/ingestor/..." || component_failed=true - ;; - customer-probe-agent) - # NOTE: This is the CORRECT way - CI/CD workflow has a bug here - run_check "$component" "go-vet" "(cd cmd/customer-probe-agent && go vet ./...)" || component_failed=true - ;; - esac - - # Step 3: Staticcheck - case "$component" in - api) - run_check "$component" "staticcheck" "(cd api && staticcheck ./...)" || component_failed=true - ;; - pkg) - run_check "$component" "staticcheck" "staticcheck ./pkg/..." || component_failed=true - ;; - org-management-controller) - run_check "$component" "staticcheck" "staticcheck ./pkg/controller/org-management/..." || component_failed=true - ;; - probe-controller) - run_check "$component" "staticcheck" "staticcheck ./pkg/probe/..." || component_failed=true - ;; - ingestor) - run_check "$component" "staticcheck" "staticcheck ./pkg/ingestor/..." || component_failed=true - ;; - customer-probe-agent) - # NOTE: This is the CORRECT way - CI/CD workflow has a bug here - run_check "$component" "staticcheck" "(cd cmd/customer-probe-agent && staticcheck ./...)" || component_failed=true - ;; - esac - - # Step 4: Gosec (skip in quick mode) - if [ "$QUICK_MODE" = false ]; then - case "$component" in - api) - run_check "$component" "gosec" "(cd api && gosec -quiet ./...)" || component_failed=true - ;; - pkg) - run_check "$component" "gosec" "gosec -quiet ./pkg/..." || component_failed=true - ;; - org-management-controller) - run_check "$component" "gosec" "gosec -quiet ./pkg/controller/org-management/..." || component_failed=true - ;; - probe-controller) - run_check "$component" "gosec" "gosec -quiet ./pkg/probe/..." || component_failed=true - ;; - ingestor) - run_check "$component" "gosec" "gosec -quiet ./pkg/ingestor/..." || component_failed=true - ;; - customer-probe-agent) - # NOTE: This is the CORRECT way - CI/CD workflow has a bug here - run_check "$component" "gosec" "(cd cmd/customer-probe-agent && gosec -quiet ./...)" || component_failed=true - ;; - esac - fi +skip() { + SKIPPED+=("$1 ($2)") + echo "==> $1 skipped: $2" +} - # Step 5: Unit tests with race detection (skip in quick mode) - if [ "$QUICK_MODE" = false ]; then - case "$component" in - api) - run_check "$component" "tests" "(cd api && go test -v -race -coverprofile=coverage.out -covermode=atomic ./...)" || component_failed=true - ;; - org-management-controller) - run_check "$component" "tests" "go test -v -race -coverprofile=coverage.out -covermode=atomic ./pkg/controller/org-management/..." || component_failed=true - ;; - probe-controller) - run_check "$component" "tests" "go test -v -race -coverprofile=coverage.out -covermode=atomic ./pkg/probe/..." || component_failed=true - ;; - ingestor) - run_check "$component" "tests" "go test -v -race -coverprofile=coverage.out -covermode=atomic ./pkg/ingestor/..." || component_failed=true - ;; - customer-probe-agent) - run_check "$component" "tests" "(cd cmd/customer-probe-agent && go test -v -race -coverprofile=coverage.out -covermode=atomic ./...)" || component_failed=true - ;; - pkg) - run_check "$component" "tests" "go test -v -race -coverprofile=coverage.out -covermode=atomic ./pkg/..." || component_failed=true - ;; - esac - fi +check_gofmt() { + local unformatted + unformatted=$(find . -name '*.go' -not -path './.*' -not -path './vendor/*' -print0 | xargs -0 gofmt -l) + [ -z "$unformatted" ] && return 0 + echo "gofmt -l reports unformatted files:" + echo "$unformatted" + return 1 +} - echo "" +check_vet() { + GOOS=linux go vet ./... +} - if [ "$component_failed" = true ]; then - return 1 - fi - return 0 +check_lint() { + golangci-lint run --timeout=5m } -# Main validation loop -echo -e "${BLUE}Starting validation...${NC}" -echo "" +check_gosec() { + gosec -quiet ./... +} -COMPONENT_RESULTS=() +check_tests() { + go test -short -race ./pkg/... ./cmd/... ./config/... +} -if [ "$PARALLEL_MODE" = true ] && [ ${#COMPONENTS[@]} -gt 1 ]; then - echo -e "${YELLOW}Running components in parallel (max 6 workers)...${NC}" - echo "" +check_helm_lint() { + helm lint helm/node-doctor +} - # Run components in parallel using background jobs - PIDS=() - for component in "${COMPONENTS[@]}"; do - validate_component "$component" & - PIDS+=($!) +check_helm_generated() { + make -s helm-verify-generated +} - # Limit to 6 parallel workers - if [ ${#PIDS[@]} -ge 6 ]; then - wait -n - PIDS=($(jobs -p)) - fi - done +run gofmt check_gofmt +run "go vet (GOOS=linux)" check_vet - # Wait for all remaining jobs - for pid in "${PIDS[@]}"; do - wait "$pid" - COMPONENT_RESULTS+=($?) - done +if command -v golangci-lint >/dev/null 2>&1; then + run golangci-lint check_lint +elif [ "${SKIP_LINT:-0}" = "1" ]; then + skip golangci-lint "not installed, SKIP_LINT=1" else - # Sequential execution - for component in "${COMPONENTS[@]}"; do - validate_component "$component" - COMPONENT_RESULTS+=($?) - done + echo "==> golangci-lint" + echo "golangci-lint is not installed. Install it (make install-deps) or set SKIP_LINT=1." >&2 + FAILED+=("golangci-lint") fi -# Cleanup temp files -rm -f /tmp/validate_*.log +if command -v gosec >/dev/null 2>&1; then + run gosec check_gosec +else + skip gosec "not installed" +fi -# Print summary -echo -e "${BLUE}════════════════════════════════════════════════════════════════${NC}" -echo -e "${BLUE}Validation Summary${NC}" -echo -e "${BLUE}════════════════════════════════════════════════════════════════${NC}" -echo "" -echo -e "Total checks: $TOTAL_CHECKS" -echo -e "${GREEN}Passed: $PASSED_CHECKS${NC}" -echo -e "${RED}Failed: $FAILED_CHECKS${NC}" -echo "" +if [ "$QUICK" = true ]; then + skip "go test" "--quick" +else + run "go test" check_tests +fi -# Component-level results -echo -e "${BLUE}Component Results:${NC}" -for i in "${!COMPONENTS[@]}"; do - component="${COMPONENTS[$i]}" - result="${COMPONENT_RESULTS[$i]}" +run "helm lint" check_helm_lint +run helm-verify-generated check_helm_generated - if [ "$result" -eq 0 ]; then - echo -e " ${GREEN}✓ $component${NC}" - else - echo -e " ${RED}✗ $component${NC}" - fi -done -echo "" +echo +echo "Summary" +for name in "${PASSED[@]}"; do echo " PASS $name"; done +for name in "${SKIPPED[@]}"; do echo " SKIP $name"; done +for name in "${FAILED[@]}"; do echo " FAIL $name"; done -# Exit with appropriate code -if [ $FAILED_CHECKS -eq 0 ]; then - echo -e "${GREEN}╔════════════════════════════════════════════════════════════════╗${NC}" - echo -e "${GREEN}║ ✓ All validations passed! Safe to push. ║${NC}" - echo -e "${GREEN}╚════════════════════════════════════════════════════════════════╝${NC}" - exit 0 -else - echo -e "${RED}╔════════════════════════════════════════════════════════════════╗${NC}" - echo -e "${RED}║ ✗ Validation failed! Fix issues before pushing. ║${NC}" - echo -e "${RED}╚════════════════════════════════════════════════════════════════╝${NC}" - echo "" - echo -e "${YELLOW}See logs above for details.${NC}" - echo -e "${YELLOW}Run with --component=NAME to validate specific component.${NC}" - echo -e "${YELLOW}Run with --quick for faster feedback (skips tests).${NC}" +if [ ${#FAILED[@]} -gt 0 ]; then + echo + echo "${#FAILED[@]} check(s) failed." >&2 exit 1 fi +echo +echo "All checks passed." diff --git a/scripts/validate-repo-cleanliness.sh b/scripts/validate-repo-cleanliness.sh index b0d75b6..ddc179d 100755 --- a/scripts/validate-repo-cleanliness.sh +++ b/scripts/validate-repo-cleanliness.sh @@ -13,31 +13,16 @@ echo "Checking repository cleanliness..." ROOT_BLACKLIST=( "*.go" # Go files belong in proper directories "test-*" # Test files - "*test*" # Any test files (except .gitignore, etc) "*.tmp" # Temporary files "*.log" # Log files "*.bak" # Backup files "*~" # Editor backup files "*.swp" # Vim swap files "*.swo" # Vim swap files - "Dockerfile.*" # Test Dockerfiles "*.out" # Output files "*.exe" # Executables ) -# Allowed files in root (whitelist) -ROOT_WHITELIST=( - ".gitignore" - ".gitattributes" - ".validation.json" - "README.md" - "LICENSE" - "CLAUDE.md" - "Makefile" - "makefile" - "makefile.windows" -) - echo "" echo "Checking root directory..." @@ -130,8 +115,6 @@ echo "" if [ $ERRORS -gt 0 ]; then echo "❌ Repository cleanliness check FAILED with $ERRORS errors!" echo "" - echo "To fix these issues, run: ./scripts/cleanup-repo.sh" - echo "" echo "Rules for keeping the repository clean:" echo "1. No Go files in the root directory - use proper package structure" echo "2. No test files or temporary files in the root" diff --git a/scripts/verify-gpg-setup.sh b/scripts/verify-gpg-setup.sh deleted file mode 100755 index c09082b..0000000 --- a/scripts/verify-gpg-setup.sh +++ /dev/null @@ -1,185 +0,0 @@ -#!/bin/bash -set -e - -# verify-gpg-setup.sh -# Verifies GPG signing configuration for GitHub Actions - -echo "🔍 Verifying GPG Setup for Node Doctor Release Automation" -echo "================================================================" -echo "" - -# Colors -GREEN='\033[0;32m' -RED='\033[0;31m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -ERRORS=0 -WARNINGS=0 - -# Check 1: Verify GPG key exists locally -echo "1ī¸âƒŖ Checking local GPG key..." -if gpg --list-secret-keys 1AF32A8B0481A7F3 > /dev/null 2>&1; then - echo -e " ${GREEN}✅ GPG key 1AF32A8B0481A7F3 found locally${NC}" - - # Get key details - KEY_CREATED=$(gpg --list-secret-keys --with-colons 1AF32A8B0481A7F3 | grep ^sec | cut -d: -f6) - KEY_EMAIL=$(gpg --list-secret-keys 1AF32A8B0481A7F3 | grep uid | grep -oP '<.*>' | tr -d '<>') - echo " 📧 Email: $KEY_EMAIL" - echo " 📅 Created: $(date -d @$KEY_CREATED 2>/dev/null || date -r $KEY_CREATED 2>/dev/null || echo 'Unknown')" -else - echo -e " ${RED}❌ GPG key 1AF32A8B0481A7F3 NOT found locally${NC}" - ERRORS=$((ERRORS + 1)) -fi -echo "" - -# Check 2: Verify GitHub secrets are configured -echo "2ī¸âƒŖ Checking GitHub repository secrets..." -if command -v gh &> /dev/null; then - SECRET_LIST=$(gh secret list 2>&1) - if echo "$SECRET_LIST" | grep -q "404"; then - echo -e " ${YELLOW}âš ī¸ Cannot verify secrets (permission issue)${NC}" - echo " Run manually: gh secret list" - WARNINGS=$((WARNINGS + 1)) - else - # Check for GPG_PRIVATE_KEY - if echo "$SECRET_LIST" | grep -q "GPG_PRIVATE_KEY"; then - echo -e " ${GREEN}✅ GPG_PRIVATE_KEY secret is configured${NC}" - else - echo -e " ${RED}❌ GPG_PRIVATE_KEY secret NOT configured${NC}" - ERRORS=$((ERRORS + 1)) - fi - - # Check for GPG_PASSPHRASE - if echo "$SECRET_LIST" | grep -q "GPG_PASSPHRASE"; then - echo -e " ${GREEN}✅ GPG_PASSPHRASE secret is configured${NC}" - else - echo -e " ${RED}❌ GPG_PASSPHRASE secret NOT configured${NC}" - ERRORS=$((ERRORS + 1)) - fi - fi -else - echo -e " ${YELLOW}âš ī¸ GitHub CLI (gh) not installed - skipping secret check${NC}" - WARNINGS=$((WARNINGS + 1)) -fi -echo "" - -# Check 3: Verify key can be exported (test without actually exporting) -echo "3ī¸âƒŖ Checking if key can be exported..." -if gpg --list-secret-keys 1AF32A8B0481A7F3 > /dev/null 2>&1; then - echo -e " ${GREEN}✅ Key is accessible for export${NC}" - echo " â„šī¸ To export: gpg --armor --export-secret-key 1AF32A8B0481A7F3 > /tmp/gpg-private-key.asc" -else - echo -e " ${RED}❌ Cannot access key for export${NC}" - ERRORS=$((ERRORS + 1)) -fi -echo "" - -# Check 4: Verify release workflow configuration -echo "4ī¸âƒŖ Checking release workflow GPG configuration..." -RELEASE_WORKFLOW=".github/workflows/release.yml" -if [ -f "$RELEASE_WORKFLOW" ]; then - if grep -q "GPG_PRIVATE_KEY" "$RELEASE_WORKFLOW"; then - echo -e " ${GREEN}✅ Release workflow references GPG_PRIVATE_KEY${NC}" - else - echo -e " ${RED}❌ Release workflow missing GPG_PRIVATE_KEY reference${NC}" - ERRORS=$((ERRORS + 1)) - fi - - if grep -q "GPG_PASSPHRASE" "$RELEASE_WORKFLOW"; then - echo -e " ${GREEN}✅ Release workflow references GPG_PASSPHRASE${NC}" - else - echo -e " ${RED}❌ Release workflow missing GPG_PASSPHRASE reference${NC}" - ERRORS=$((ERRORS + 1)) - fi - - # Check for GPG signing step - if grep -q "gpg --batch --import" "$RELEASE_WORKFLOW"; then - echo -e " ${GREEN}✅ GPG key import step configured${NC}" - else - echo -e " ${YELLOW}âš ī¸ GPG key import step not found${NC}" - WARNINGS=$((WARNINGS + 1)) - fi - - # Check for GPG signing - if grep -q "gpg.*--detach-sign" "$RELEASE_WORKFLOW"; then - echo -e " ${GREEN}✅ GPG artifact signing configured${NC}" - else - echo -e " ${YELLOW}âš ī¸ GPG artifact signing not found${NC}" - WARNINGS=$((WARNINGS + 1)) - fi -else - echo -e " ${RED}❌ Release workflow not found at $RELEASE_WORKFLOW${NC}" - ERRORS=$((ERRORS + 1)) -fi -echo "" - -# Check 5: Test GPG signing capability (if key is available) -echo "5ī¸âƒŖ Testing GPG signing capability..." -TEST_FILE="/tmp/gpg-test-$$" -echo "Test content for GPG signing verification" > "$TEST_FILE" - -if gpg --list-secret-keys 1AF32A8B0481A7F3 > /dev/null 2>&1; then - # Try to sign with --batch (will fail if passphrase needed) - if echo "test-passphrase" | gpg --batch --yes --passphrase-fd 0 --armor --detach-sign --output "${TEST_FILE}.asc" "$TEST_FILE" 2>/dev/null; then - echo -e " ${YELLOW}âš ī¸ GPG signing works but test passphrase was accepted${NC}" - echo " â„šī¸ Make sure to use your actual passphrase in GitHub secrets" - WARNINGS=$((WARNINGS + 1)) - rm -f "${TEST_FILE}.asc" - else - echo -e " ${GREEN}✅ GPG key requires passphrase (as expected)${NC}" - echo " â„šī¸ GitHub Actions will use GPG_PASSPHRASE secret during release" - fi -else - echo -e " ${YELLOW}âš ī¸ Cannot test signing - key not accessible${NC}" - WARNINGS=$((WARNINGS + 1)) -fi -rm -f "$TEST_FILE" -echo "" - -# Summary -echo "================================================================" -echo "📊 Verification Summary" -echo "================================================================" -if [ $ERRORS -eq 0 ] && [ $WARNINGS -eq 0 ]; then - echo -e "${GREEN}✅ ALL CHECKS PASSED!${NC}" - echo "" - echo "GPG signing is properly configured. Next steps:" - echo "1. Test with a pre-release: git tag v0.1.0-rc.1 && git push origin v0.1.0-rc.1" - echo "2. Monitor the release workflow in GitHub Actions" - echo "3. Verify dual signatures are created (.cosign.sig and .asc files)" -elif [ $ERRORS -eq 0 ]; then - echo -e "${YELLOW}âš ī¸ PASSED WITH WARNINGS ($WARNINGS warnings)${NC}" - echo "" - echo "GPG configuration is mostly ready, but review warnings above." -else - echo -e "${RED}❌ VERIFICATION FAILED ($ERRORS errors, $WARNINGS warnings)${NC}" - echo "" - echo "Please address the errors above before proceeding with releases." - exit 1 -fi - -echo "" -echo "================================================================" -echo "📚 Quick Reference" -echo "================================================================" -echo "" -echo "Export GPG private key:" -echo " gpg --armor --export-secret-key 1AF32A8B0481A7F3 > /tmp/gpg-private-key.asc" -echo "" -echo "Add GitHub secrets (manual via UI):" -echo " 1. Go to: https://github.com/supporttools/node-doctor/settings/secrets/actions" -echo " 2. Click 'New repository secret'" -echo " 3. Name: GPG_PRIVATE_KEY" -echo " Value: " -echo " 4. Click 'New repository secret'" -echo " 5. Name: GPG_PASSPHRASE" -echo " Value: " -echo "" -echo "Or add via CLI:" -echo " gh secret set GPG_PRIVATE_KEY < /tmp/gpg-private-key.asc" -echo " gh secret set GPG_PASSPHRASE # will prompt for value" -echo "" -echo "Verify secrets:" -echo " gh secret list" -echo "" diff --git a/test/e2e/cluster/image.go b/test/e2e/cluster/image.go index 601efc8..8c1c30b 100644 --- a/test/e2e/cluster/image.go +++ b/test/e2e/cluster/image.go @@ -18,42 +18,45 @@ import ( "time" ) -// BuildAndLoadImage builds the Node Doctor Docker image and loads it into KIND cluster +var e2eImages = []struct { + name string + dockerfile string +}{ + {name: "node-doctor:e2e-test", dockerfile: "Dockerfile"}, + {name: "node-doctor-controller:e2e-test", dockerfile: "Dockerfile.controller"}, +} + +// BuildAndLoadImage builds the agent and controller images and loads them into the KIND cluster func BuildAndLoadImage(ctx context.Context, clusterName string) error { - // Get project root (should be 3 levels up from test/e2e/cluster) projectRoot, err := getProjectRoot() if err != nil { return fmt.Errorf("failed to find project root: %w", err) } - imageName := "node-doctor:e2e-test" + for _, img := range e2eImages { + fmt.Printf("Building Docker image: %s\n", img.name) + if err := buildDockerImage(ctx, projectRoot, img.dockerfile, img.name); err != nil { + return fmt.Errorf("failed to build image %s: %w", img.name, err) + } - // Step 1: Build Docker image - fmt.Printf("Building Docker image: %s\n", imageName) - if err := buildDockerImage(ctx, projectRoot, imageName); err != nil { - return fmt.Errorf("failed to build image: %w", err) - } + fmt.Printf("Loading image into KIND cluster: %s\n", clusterName) + if err := loadImageIntoKIND(ctx, clusterName, img.name); err != nil { + return fmt.Errorf("failed to load image %s into KIND: %w", img.name, err) + } - // Step 2: Load image into KIND cluster - fmt.Printf("Loading image into KIND cluster: %s\n", clusterName) - if err := loadImageIntoKIND(ctx, clusterName, imageName); err != nil { - return fmt.Errorf("failed to load image into KIND: %w", err) + fmt.Printf("Image %s built and loaded successfully\n", img.name) } - fmt.Printf("Image %s built and loaded successfully\n", imageName) return nil } -// buildDockerImage builds the Node Doctor Docker image -func buildDockerImage(ctx context.Context, projectRoot, imageName string) error { - // Create timeout for build operation +func buildDockerImage(ctx context.Context, projectRoot, dockerfile, imageName string) error { buildCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) defer cancel() - // Check if Dockerfile exists - dockerfilePath := filepath.Join(projectRoot, "Dockerfile") + dockerfilePath := filepath.Join(projectRoot, dockerfile) if _, err := os.Stat(dockerfilePath); err != nil { - return fmt.Errorf("Dockerfile not found at %s: %w", dockerfilePath, err) + return fmt.Errorf("%s not found at %s: %w", dockerfile, dockerfilePath, err) } // Build the image