diff --git a/vars/devopsPipeline.groovy b/vars/devopsPipeline.groovy index 133e143..f03290e 100644 --- a/vars/devopsPipeline.groovy +++ b/vars/devopsPipeline.groovy @@ -1,20 +1,19 @@ -// Ek hi pipeline, har module ke liye. Module ki Jenkinsfile sirf itni hoti hai: +// Ek hi pipeline, har module ke liye. Module ki Jenkinsfile sirf itni: // // @Library('devops-platform') _ // devopsPipeline(module: 'demo-app') // -// Poora raasta: secret scan -> test -> SAST -> build -> IaC scan -> CVE gate -> -// SBOM -> smoke -> push -> sign -> verify -> staging -> approval -> production. +// Raasta: secret scan -> test -> SAST -> IaC scan -> build -> CVE gate -> SBOM +// -> smoke -> push -> sign -> verify -> GitOps promote -> VM deploy. // // CI builder VM par chalta hai (wahan deploy key nahi hai), -// CD controller par (wahan docker.sock nahi hai). Dono taraf blast radius chhota. +// CD + signing controller par (wahan docker.sock nahi hai). def call(Map cfg = [:]) { def P = platformConfig() if (!cfg.module) { error "devopsPipeline: 'module' dena zaroori hai" } - // ---- defaults: module kuch na de to sensible value ---- cfg.imageName = cfg.imageName ?: "swim/${cfg.module}" cfg.vaultMount = cfg.vaultMount ?: 'swim' cfg.testImage = cfg.testImage ?: 'python:3.12-slim' @@ -25,251 +24,227 @@ def call(Map cfg = [:]) { cfg.sonarKey = cfg.sonarKey ?: "swim-${cfg.module}" cfg.inventory = cfg.inventory ?: 'deploy/inventory.ini' cfg.playbook = cfg.playbook ?: 'deploy/deploy.yml' - cfg.runSonar = cfg.containsKey('runSonar') ? cfg.runSonar : true - cfg.runDeploy = cfg.containsKey('runDeploy') ? cfg.runDeploy : true - cfg.approval = cfg.containsKey('approval') ? cfg.approval : true - // GitOps lane: k8s cluster me deploy ArgoCD karta hai, Jenkins sirf tag commit karta hai. - cfg.gitops = cfg.containsKey('gitops') ? cfg.gitops : false + cfg.runSonar = cfg.containsKey('runSonar') ? cfg.runSonar : true + cfg.runDeploy = cfg.containsKey('runDeploy') ? cfg.runDeploy : true + cfg.approval = cfg.containsKey('approval') ? cfg.approval : true + cfg.gitops = cfg.containsKey('gitops') ? cfg.gitops : false + cfg.deployVM = cfg.containsKey('deployVM') ? cfg.deployVM : true cfg.gitopsRepo = cfg.gitopsRepo ?: 'https://ns31240276.ip-51-195-4.eu/devops/gitops.git' cfg.gitopsPath = cfg.gitopsPath ?: "manifests/${cfg.module}" cfg.k8sVerifyUrl = cfg.k8sVerifyUrl ?: '' - // VM lane: ansible se app-vm/staging-vm par deploy. - cfg.deployVM = cfg.containsKey('deployVM') ? cfg.deployVM : true properties([ parameters([ string(name: 'ROLLBACK_TAG', defaultValue: '', - description: 'Khaali = normal build. Purana tag (jaise 7) = bina rebuild seedha us image par rollback.') + description: 'Khaali = normal build. Purana image tag = bina rebuild seedha us image par rollback.') ]), buildDiscarder(logRotator(numToKeepStr: '30')), disableConcurrentBuilds() ]) - def branch = env.BRANCH_NAME ?: 'main' - def isPR = branch.startsWith('PR-') - def isMain = (branch == 'main' || branch == 'master') - // PR par sirf CI. Production sirf main/master se jaata hai. - def deployable = cfg.runDeploy && !isPR && isMain - + def branch = env.BRANCH_NAME ?: 'main' + def isPR = branch.startsWith('PR-') + def isMain = (branch == 'main' || branch == 'master') + // PR aur feature branch par sirf CI gates. Production sirf main/master se. + def deployable = cfg.runDeploy && !isPR && isMain + def safeBranch = branch.replaceAll('[^A-Za-z0-9._-]', '-').toLowerCase() def rollbackTag = (env.ROLLBACK_TAG ?: '').trim() def isRollback = rollbackTag != '' - // Multibranch me HAR branch ka build number 1 se shuru hota hai. Agar tag me - // sirf build number rakha to main#1 aur feature#1 ek hi image tag bana ke - // ek doosre ko overwrite kar denge. Isliye non-main branch ka naam tag me. - def safeBranch = branch.replaceAll('[^A-Za-z0-9._-]', '-').toLowerCase() - def imageTag = isRollback ? rollbackTag - : (isMain ? "${env.BUILD_NUMBER}" - : "${safeBranch}-${env.BUILD_NUMBER}") - def fullImage = "${P.registry}/${cfg.imageName}:${imageTag}" - def commonEnv = [ - "MODULE=${cfg.module}", - "REGISTRY=${P.registry}", - "IMAGE_NAME=${cfg.imageName}", - "IMAGE_TAG=${imageTag}", - "FULL_IMAGE=${fullImage}", - "VAULT_ADDR=${P.vaultAddr}", - "VAULT_MOUNT=${cfg.vaultMount}", - "SONAR_URL=${P.sonarUrl}", - "SONAR_KEY=${cfg.sonarKey}", - "BRIDGE_IP=${P.bridgeIp}", - "TRIVY_SEVERITY=${cfg.trivySeverity}", - "APP_PORT=${cfg.appPort}", - "HEALTH_PATH=${cfg.healthPath}", - "INVENTORY=${cfg.inventory}", - "PLAYBOOK=${cfg.playbook}", - "IMG_GITLEAKS=${P.gitleaks}", - "IMG_TRIVY=${P.trivy}", - "IMG_SYFT=${P.syft}", - "IMG_SCANNER=${P.sonarScanner}", - "TEST_IMAGE=${cfg.testImage}" - ] + def imageTag, fullImage, commonEnv timestamps { timeout(time: 60, unit: 'MINUTES') { - try { - // ===================== CI: builder VM ===================== - node(P.builderLabel) { - withEnv(commonEnv) { - try { - stage('Init') { - echo "module=${cfg.module} node=${env.NODE_NAME} branch=${branch} " + - "mode=${isRollback ? 'ROLLBACK' : 'BUILD'} tag=${imageTag} deploy=${deployable}" + // ===================== CI: builder VM ===================== + node(P.builderLabel) { + try { + stage('Checkout') { + checkout scm + def sha = sh(script: 'git rev-parse --short=7 HEAD', returnStdout: true).trim() + // Tag me commit SHA isliye: multibranch me har branch ka build + // number 1 se shuru hota hai, to sirf build number rakhne par + // tag dobara use ho jaata hai aur purani image overwrite hoti hai. + // SHA ke saath har tag hamesha unique aur traceable rehta hai. + imageTag = isRollback ? rollbackTag + : (isMain ? "${env.BUILD_NUMBER}-${sha}" + : "${safeBranch}-${env.BUILD_NUMBER}-${sha}") + fullImage = "${P.registry}/${cfg.imageName}:${imageTag}" + commonEnv = [ + "MODULE=${cfg.module}", "REGISTRY=${P.registry}", + "IMAGE_NAME=${cfg.imageName}", "IMAGE_TAG=${imageTag}", + "FULL_IMAGE=${fullImage}", "VAULT_ADDR=${P.vaultAddr}", + "VAULT_MOUNT=${cfg.vaultMount}", "SONAR_URL=${P.sonarUrl}", + "SONAR_KEY=${cfg.sonarKey}", "BRIDGE_IP=${P.bridgeIp}", + "TRIVY_SEVERITY=${cfg.trivySeverity}", "APP_PORT=${cfg.appPort}", + "HEALTH_PATH=${cfg.healthPath}", "INVENTORY=${cfg.inventory}", + "PLAYBOOK=${cfg.playbook}", "IMG_GITLEAKS=${P.gitleaks}", + "IMG_TRIVY=${P.trivy}", "IMG_SYFT=${P.syft}", + "IMG_SCANNER=${P.sonarScanner}", "TEST_IMAGE=${cfg.testImage}" + ] + currentBuild.displayName = "#${env.BUILD_NUMBER} ${imageTag}" + echo "module=${cfg.module} node=${env.NODE_NAME} branch=${branch} " + + "mode=${isRollback ? 'ROLLBACK' : 'BUILD'} tag=${imageTag} deploy=${deployable}" + } + + withEnv(commonEnv) { + stage('Secret scan') { + // Poori git history scan hoti hai, sirf latest commit nahi. + sh ''' + docker run --rm -v "$PWD":/repo "$IMG_GITLEAKS" \ + detect --source=/repo --redact --no-banner -v + ''' + } + + if (!isRollback) { + stage('Test') { + sh """ + docker run --rm -v "\$PWD":/src -w /src "\$TEST_IMAGE" sh -c '${cfg.testCommand}' + """ + junit allowEmptyResults: true, testResults: 'test-results.xml' } - stage('Checkout') { - checkout scm - sh 'echo "commit: $(git rev-parse --short HEAD)"' + if (cfg.runSonar) { + stage('SAST (SonarQube)') { + withCredentials([string(credentialsId: 'sonar-token', variable: 'SONAR_TOKEN')]) { + sh ''' + set +x + docker run --rm \ + --add-host "ns31240276.ip-51-195-4.eu:$BRIDGE_IP" \ + -e SONAR_HOST_URL="$SONAR_URL" \ + -e SONAR_TOKEN="$SONAR_TOKEN" \ + -v "$PWD":/usr/src "$IMG_SCANNER" \ + -Dsonar.projectKey="$SONAR_KEY" + ''' + } + } } - stage('Secret scan') { - // Poori git history scan hoti hai, sirf latest commit nahi - - // purana leak bhi pakda jaata hai. Findings = build fail. + stage('IaC / Dockerfile scan') { sh ''' - docker run --rm -v "$PWD":/repo "$IMG_GITLEAKS" \ - detect --source=/repo --redact --no-banner -v + docker run --rm -v "$PWD":/work "$IMG_TRIVY" config /work \ + --severity "$TRIVY_SEVERITY" --exit-code 1 ''' } - if (!isRollback) { - stage('Test') { - sh """ - docker run --rm -v "\$PWD":/src -w /src "\$TEST_IMAGE" sh -c '${cfg.testCommand}' - """ - junit allowEmptyResults: true, testResults: 'test-results.xml' - } + stage('Build image') { + sh ''' + docker build --build-arg APP_VERSION="$IMAGE_TAG" \ + -t "$FULL_IMAGE" -t "$REGISTRY/$IMAGE_NAME:latest" . + ''' + } - if (cfg.runSonar) { - stage('SAST (SonarQube)') { - withCredentials([string(credentialsId: 'sonar-token', variable: 'SONAR_TOKEN')]) { - sh ''' - set +x - docker run --rm \ - --add-host "ns31240276.ip-51-195-4.eu:$BRIDGE_IP" \ - -e SONAR_HOST_URL="$SONAR_URL" \ - -e SONAR_TOKEN="$SONAR_TOKEN" \ - -v "$PWD":/usr/src "$IMG_SCANNER" \ - -Dsonar.projectKey="$SONAR_KEY" - ''' - } - } - } + stage('CVE gate (Trivy)') { + sh ''' + docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v trivy-cache:/root/.cache/ \ + -v "$PWD/.trivyignore":/.trivyignore:ro \ + "$IMG_TRIVY" image \ + --severity "$TRIVY_SEVERITY" --ignore-unfixed \ + --ignorefile /.trivyignore --exit-code 1 \ + --format table --no-progress "$FULL_IMAGE" + ''' + } - stage('IaC / Dockerfile scan') { - // Dockerfile, compose, k8s manifests ki misconfig pakadta hai - // (root user, missing healthcheck, privileged, etc.) - sh ''' - docker run --rm -v "$PWD":/work "$IMG_TRIVY" config /work \ - --severity "$TRIVY_SEVERITY" --exit-code 1 - ''' - } + stage('SBOM') { + sh ''' + docker run --rm -v /var/run/docker.sock:/var/run/docker.sock \ + "$IMG_SYFT" "docker:$FULL_IMAGE" -o spdx-json > sbom.spdx.json + docker run --rm -v /var/run/docker.sock:/var/run/docker.sock \ + "$IMG_SYFT" "docker:$FULL_IMAGE" -o cyclonedx-json > sbom.cyclonedx.json + echo " SBOM packages: $(python3 -c "import json;print(len(json.load(open('sbom.spdx.json'))['packages']))")" + ''' + archiveArtifacts artifacts: 'sbom.*.json', fingerprint: true + } - stage('Build image') { - sh ''' - docker build --build-arg APP_VERSION="$IMAGE_TAG" \ - -t "$FULL_IMAGE" -t "$REGISTRY/$IMAGE_NAME:latest" . - ''' - } - - stage('CVE gate (Trivy)') { - sh ''' - docker run --rm \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v trivy-cache:/root/.cache/ \ - -v "$PWD/.trivyignore":/.trivyignore:ro \ - "$IMG_TRIVY" image \ - --severity "$TRIVY_SEVERITY" --ignore-unfixed \ - --ignorefile /.trivyignore --exit-code 1 \ - --format table --no-progress "$FULL_IMAGE" - ''' - } - - stage('SBOM') { - // Audit ka pehla sawaal: "is image me kya-kya hai?" - sh ''' - docker run --rm -v /var/run/docker.sock:/var/run/docker.sock \ - "$IMG_SYFT" "docker:$FULL_IMAGE" -o spdx-json > sbom.spdx.json - docker run --rm -v /var/run/docker.sock:/var/run/docker.sock \ - "$IMG_SYFT" "docker:$FULL_IMAGE" -o cyclonedx-json > sbom.cyclonedx.json - echo " SBOM packages: $(python3 -c "import json;print(len(json.load(open('sbom.spdx.json'))['packages']))")" - ''' - archiveArtifacts artifacts: 'sbom.*.json', fingerprint: true - } - - stage('Smoke test') { - sh ''' - CID=$(docker run -d -P "$FULL_IMAGE") - trap "docker rm -f $CID >/dev/null 2>&1" EXIT - sleep 5 - docker exec $CID python -c " + stage('Smoke test') { + sh ''' + CID=$(docker run -d -P "$FULL_IMAGE") + trap "docker rm -f $CID >/dev/null 2>&1" EXIT + sleep 5 + docker exec $CID python -c " import urllib.request, json r = json.loads(urllib.request.urlopen('http://127.0.0.1:${APP_PORT}${HEALTH_PATH}').read()) print(' live:', r) assert r['version'] == '${IMAGE_TAG}', 'version mismatch' " + ''' + } + + stage('Push') { + withCredentials([string(credentialsId: 'vault-builder-token', variable: 'VAULT_TOKEN')]) { + sh ''' + set +x + curl -sf -H "X-Vault-Token: $VAULT_TOKEN" \ + "$VAULT_ADDR/v1/$VAULT_MOUNT/data/registry" > /tmp/reg.$$.json + U=$(python3 -c "import json;print(json.load(open('/tmp/reg.$$.json'))['data']['data']['username'])") + P=$(python3 -c "import json;print(json.load(open('/tmp/reg.$$.json'))['data']['data']['password'])") + rm -f /tmp/reg.$$.json + echo "$P" | docker login "$REGISTRY" -u "$U" --password-stdin + docker push "$FULL_IMAGE" + docker push "$REGISTRY/$IMAGE_NAME:latest" + docker logout "$REGISTRY" ''' } - - stage('Push') { - withCredentials([string(credentialsId: 'vault-builder-token', variable: 'VAULT_TOKEN')]) { - sh ''' - set +x - curl -sf -H "X-Vault-Token: $VAULT_TOKEN" \ - "$VAULT_ADDR/v1/$VAULT_MOUNT/data/registry" > /tmp/reg.$$.json - U=$(python3 -c "import json;print(json.load(open('/tmp/reg.$$.json'))['data']['data']['username'])") - P=$(python3 -c "import json;print(json.load(open('/tmp/reg.$$.json'))['data']['data']['password'])") - rm -f /tmp/reg.$$.json - echo "$P" | docker login "$REGISTRY" -u "$U" --password-stdin - docker push "$FULL_IMAGE" - docker push "$REGISTRY/$IMAGE_NAME:latest" - docker logout "$REGISTRY" - ''' - } - } } - } finally { - sh 'docker image prune -f --filter "until=24h" >/dev/null 2>&1 || true' - cleanWs() - } - } - } - - // ============ Sign: controller par (key builder ko nahi milti) ============ - if (!isRollback) { - node(P.deployLabel) { - withEnv(commonEnv) { - stage('Sign image') { cosignSign() } - } } } + } finally { + sh 'docker image prune -f --filter "until=24h" >/dev/null 2>&1 || true' + cleanWs() + } + } - if (!deployable) { - echo isPR ? "PR build - deploy nahi hoga, sirf CI gates." : "Branch '${branch}' deployable nahi hai." - return + // ============ Sign: controller par (key builder ko nahi milti) ============ + if (!isRollback) { + node(P.deployLabel) { + withEnv(commonEnv) { stage('Sign image') { cosignSign() } } } + } - // ===================== CD: controller ===================== + if (!deployable) { + echo isPR ? "PR build - deploy nahi hoga, sirf CI gates." + : "Branch '${branch}' deployable nahi hai - sirf CI gates chale." + return + } + + // ===================== CD ===================== + node(P.deployLabel) { + withEnv(commonEnv) { + stage('Verify signature') { cosignVerify() } + if (cfg.gitops) { + stage('Promote to GitOps') { gitopsPromote(cfg) } + if (cfg.k8sVerifyUrl) { + // ArgoCD ko Gitea webhook turant jagata hai, phir bhi image + // pull + rollout me waqt lagta hai - 10 min ka window. + stage('Verify k8s (ArgoCD)') { waitForRollout(cfg.k8sVerifyUrl, imageTag, 60) } + } + } + if (cfg.deployVM) { + stage('Deploy STAGING') { ansibleDeploy('staging') } + stage('Verify STAGING') { verifyEnv('staging') } + } + } + } + + if (cfg.approval && cfg.deployVM) { + stage('Production approval') { + // Node ke bahar - intezaar me koi executor block nahi hota. + timeout(time: 30, unit: 'MINUTES') { + input message: "${cfg.module}:${imageTag} staging par verified hai. PRODUCTION par bhejein?", + ok: 'Deploy to Production' + } + } + } + + if (cfg.deployVM) { node(P.deployLabel) { withEnv(commonEnv) { - stage('Verify signature') { cosignVerify() } - if (cfg.gitops) { - stage('Promote to GitOps') { gitopsPromote(cfg) } - if (cfg.k8sVerifyUrl) { - stage('Verify k8s (ArgoCD)') { waitForRollout(cfg.k8sVerifyUrl, imageTag) } - } - } - if (cfg.deployVM) { - stage('Deploy STAGING') { ansibleDeploy('staging') } - stage('Verify STAGING') { verifyEnv('staging') } - } + stage('Deploy PRODUCTION') { ansibleDeploy('production') } + stage('Verify PRODUCTION') { verifyEnv('production') } } } - - if (cfg.approval && cfg.deployVM) { - stage('Production approval') { - // Node ke bahar hai - intezaar me koi executor block nahi hota. - timeout(time: 30, unit: 'MINUTES') { - input message: "${cfg.module}:${imageTag} staging par verified hai. PRODUCTION par bhejein?", - ok: 'Deploy to Production' - } - } - } - - if (cfg.deployVM) { - node(P.deployLabel) { - withEnv(commonEnv) { - stage('Deploy PRODUCTION') { ansibleDeploy('production') } - stage('Verify PRODUCTION') { verifyEnv('production') } - } - } - } - - currentBuild.description = "${cfg.module}:${imageTag}${isRollback ? ' (rollback)' : ''}" - - } catch (e) { - currentBuild.result = 'FAILURE' - throw e } } }