Administrator / elk-aws-eks · Files

pipeline {
    agent {
        label 'jenkins-agent'
    }

    environment {
        CLUSTER_NAME = "comic-website-prod"
        AWS_REGION = "us-east-1"
        NAMESPACE = "logging"
        DOMAIN_NAME = "mpc.run.place"
        KIBANA_DOMAIN = "kibana.mpc.run.place"
        TF_STATE_BUCKET = "terraformstatefile090909"
        TF_LOCK_TABLE = "terraform-locks"
    }

    options {
        timeout(time: 1, unit: 'HOURS')
        disableConcurrentBuilds()
        buildDiscarder(logRotator(numToKeepStr: '10'))
    }

    parameters {
        choice(
            name: 'DEPLOYMENT_TYPE',
            choices: ['full', 'infrastructure-only', 'elk-only', 'dns-only'],
            description: 'Select deployment type'
        )
        booleanParam(
            name: 'DESTROY',
            defaultValue: false,
            description: 'Destroy resources instead of creating'
        )
    }

    stages {
        stage('Verify Environment') {
            steps {
                script {
                    echo "Starting ELK Stack Deployment on EKS"
                    echo "Deployment Type: ${params.DEPLOYMENT_TYPE}"
                    echo "Destroy Mode: ${params.DESTROY}"
                    
                    // 验证工具是否已安装
                    sh '''
                        echo "=== Verifying Required Tools ==="
                        terraform version
                        kubectl version --client
                        aws --version
                        jq --version
                        echo "All tools are available!"
                    '''
                    
                    // 配置 kubectl 访问 EKS 集群
                    sh '''
                        echo "=== Configuring kubectl for EKS ==="
                        aws eks update-kubeconfig --region ${AWS_REGION} --name ${CLUSTER_NAME}
                        kubectl get nodes
                    '''
                }
            }
        }

        stage('Destroy Resources') {
            when {
                expression { params.DESTROY == true }
            }
            steps {
                script {
                    echo "Starting destruction of ELK Stack resources..."
                    destroyResources()
                }
            }
        }

        stage('Deploy Infrastructure') {
            when {
                expression { 
                    params.DESTROY == false && 
                    (params.DEPLOYMENT_TYPE == 'full' || params.DEPLOYMENT_TYPE == 'infrastructure-only') 
                }
            }
            steps {
                script {
                    deployInfrastructure()
                }
            }
        }

        stage('Deploy ELK Stack') {
            when {
                expression { 
                    params.DESTROY == false && 
                    (params.DEPLOYMENT_TYPE == 'full' || params.DEPLOYMENT_TYPE == 'elk-only') 
                }
            }
            steps {
                script {
                    deployELKStack()
                }
            }
        }

        stage('Update DNS') {
            when {
                expression { 
                    params.DESTROY == false && 
                    (params.DEPLOYMENT_TYPE == 'full' || params.DEPLOYMENT_TYPE == 'dns-only') 
                }
            }
            steps {
                script {
                    updateDNS()
                }
            }
        }

        stage('Health Check') {
            when {
                expression { params.DESTROY == false }
            }
            steps {
                script {
                    healthCheck()
                }
            }
        }
    }

    post {
        always {
            script {
                echo "Build completed: ${currentBuild.result}"
                echo "Build URL: ${env.BUILD_URL}"
            }
        }
        success {
            script {
                echo "✅ ELK Stack deployment completed successfully!"
                echo "🌐 Kibana URL: https://${env.KIBANA_DOMAIN}"
            }
        }
        failure {
            script {
                echo "❌ ELK Stack deployment failed!"
                emailext (
                    subject: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
                    body: """
                    ELK Stack deployment failed!
                    
                    Job: ${env.JOB_NAME}
                    Build: ${env.BUILD_NUMBER}
                    URL: ${env.BUILD_URL}
                    
                    Please check the build logs for details.
                    """,
                    to: "devops@example.com"
                )
            }
        }
    }
}

// Pipeline Functions
def deployInfrastructure() {
    echo "🚀 Stage 1: Deploying Infrastructure (Route53 + ACM)"
    
    // 部署 Route53
    dir('terraform/route53') {
        sh '''
            echo "Initializing and applying Route53 Terraform..."
            terraform init
            terraform plan -out=tfplan
            terraform apply -auto-approve tfplan
        '''
        
        // 获取输出
        def route53ZoneId = sh(
            script: 'terraform output -raw route53_zone_id',
            returnStdout: true
        ).trim()
        
        def nameServers = sh(
            script: 'terraform output -json route53_name_servers',
            returnStdout: true
        ).trim()
        
        echo "Route53 Zone ID: ${route53ZoneId}"
        echo "Name Servers: ${nameServers}"
        
        env.ROUTE53_ZONE_ID = route53ZoneId
    }
    
    // 部署 ACM
    dir('terraform/acm') {
        sh '''
            echo "Initializing and applying ACM Terraform..."
            terraform init
            terraform plan -out=tfplan
            terraform apply -auto-approve tfplan
        '''
        
        // 获取 ACM 证书 ARN
        def acmCertArn = sh(
            script: 'terraform output -raw acm_certificate_arn',
            returnStdout: true
        ).trim()
        
        echo "ACM Certificate ARN: ${acmCertArn}"
        env.ACM_CERT_ARN = acmCertArn
        
        // 更新 Kibana Ingress 使用 ACM ARN
        sh """
            echo "Updating Kibana Ingress with ACM Certificate ARN..."
            sed -i 's|alb.ingress.kubernetes.io/certificate-arn:.*|alb.ingress.kubernetes.io/certificate-arn: \"${acmCertArn}\"|' k8s/kibana/kibana-ingress.yaml
        """
    }
    
    echo "=== 🔔 IMPORTANT: Update Your Domain Nameservers ==="
    echo "Please update your domain registrar for ${env.DOMAIN_NAME} to use the nameservers shown above"
    echo "====================================================="
}

def deployELKStack() {
    echo "🚀 Stage 2: Deploying ELK Stack"
    
    // 应用 Kubernetes 资源
    sh '''
        echo "Creating logging namespace..."
        kubectl apply -f k8s/namespaces/logging-namespace.yaml
        
        echo "Deploying Elasticsearch..."
        kubectl apply -f k8s/elasticsearch/ -n ${NAMESPACE}
    '''
    
    // 等待 Elasticsearch 就绪
    sh '''
        echo "Waiting for Elasticsearch to be ready..."
        for i in {1..60}; do
            if kubectl wait --for=condition=ready pod -l app=elasticsearch -n ${NAMESPACE} --timeout=60s 2>/dev/null; then
                echo "✅ Elasticsearch is ready!"
                break
            fi
            echo "⏳ Elasticsearch not ready yet, attempt $i/60..."
            sleep 10
        done
    '''
    
    // 部署其他组件
    sh '''
        echo "Deploying Kibana..."
        kubectl apply -f k8s/kibana/ -n ${NAMESPACE}
        
        echo "Deploying Filebeat..."
        kubectl apply -f k8s/filebeat/ -n ${NAMESPACE}
        
        echo "Deploying Logstash..."
        kubectl apply -f k8s/logstash/ -n ${NAMESPACE}
    '''
    
    // 等待 ALB 配置完成
    echo "⏳ Waiting for ALB to be provisioned..."
    sh '''
        for i in {1..30}; do
            ALB_DNS_NAME=$(kubectl get ingress -n ${NAMESPACE} kibana-ingress -o jsonpath="{.status.loadBalancer.ingress[0].hostname}" 2>/dev/null || echo "")
            if [ -n "$ALB_DNS_NAME" ]; then
                echo "✅ ALB is ready: $ALB_DNS_NAME"
                break
            fi
            echo "⏳ ALB not ready yet, attempt $i/30..."
            sleep 10
        done
    '''
    
    // 获取 ALB 信息
    def albDnsName = sh(
        script: 'kubectl get ingress -n ${NAMESPACE} kibana-ingress -o jsonpath="{.status.loadBalancer.ingress[0].hostname}"',
        returnStdout: true
    ).trim()
    
    if (!albDnsName) {
        error "❌ ALB DNS name not found. ALB may not be provisioned yet."
    }
    
    def albZoneId = sh(
        script: "aws elbv2 describe-load-balancers --region ${env.AWS_REGION} --query \"LoadBalancers[?DNSName=='${albDnsName}'].CanonicalHostedZoneId\" --output text",
        returnStdout: true
    ).trim()
    
    echo "ALB DNS Name: ${albDnsName}"
    echo "ALB Zone ID: ${albZoneId}"
    
    // 保存 ALB 信息到 S3
    sh """
        echo "Saving ALB information to S3..."
        cat > alb-info.json << EOF
        {
            "alb_dns_name": "${albDnsName}",
            "alb_zone_id": "${albZoneId}"
        }
        EOF
        
        aws s3 cp alb-info.json s3://${env.TF_STATE_BUCKET}/elk-eks/alb-info.json
        echo "✅ ALB information saved to S3"
    """
    
    env.ALB_DNS_NAME = albDnsName
    env.ALB_ZONE_ID = albZoneId
}

def updateDNS() {
    echo "🚀 Stage 3: Updating DNS Records"
    
    // 从 S3 获取 ALB 信息（如果尚未设置）
    if (!env.ALB_DNS_NAME || !env.ALB_ZONE_ID) {
        echo "📥 Retrieving ALB information from S3..."
        sh '''
            aws s3 cp s3://${TF_STATE_BUCKET}/elk-eks/alb-info.json alb-info.json || echo "ALB info file not found in S3"
        '''
        
        if (fileExists('alb-info.json')) {
            env.ALB_DNS_NAME = sh(
                script: 'jq -r ".alb_dns_name" alb-info.json',
                returnStdout: true
            ).trim()
            
            env.ALB_ZONE_ID = sh(
                script: 'jq -r ".alb_zone_id" alb-info.json',
                returnStdout: true
            ).trim()
        } else {
            error "❌ ALB information not found. Please run ELK Stack deployment first or provide ALB information."
        }
    }
    
    echo "Using ALB DNS Name: ${env.ALB_DNS_NAME}"
    echo "Using ALB Zone ID: ${env.ALB_ZONE_ID}"
    
    // 更新 DNS 记录
    dir('terraform/dns') {
        sh """
            echo "Initializing and applying DNS Terraform..."
            terraform init
            terraform plan \
                -var="alb_dns_name=${env.ALB_DNS_NAME}" \
                -var="alb_zone_id=${env.ALB_ZONE_ID}" \
                -out=tfplan
            terraform apply -auto-approve tfplan
        """
    }
    
    echo "✅ DNS records updated successfully!"
    echo "🌐 Kibana will be available at: https://${env.KIBANA_DOMAIN}"
}

def healthCheck() {
    echo "🔍 Performing Health Checks..."
    
    sh '''
        echo "=== 📊 ELK Stack Status ==="
        kubectl get pods -n ${NAMESPACE}
        
        echo ""
        echo "=== 🔗 Services Status ==="
        kubectl get svc -n ${NAMESPACE}
        
        echo ""
        echo "=== 🌐 Ingress Status ==="
        kubectl get ingress -n ${NAMESPACE}
        
        echo ""
        echo "=== 📈 Elasticsearch Health ==="
        ES_POD=$(kubectl get pods -n ${NAMESPACE} -l app=elasticsearch -o jsonpath="{.items[0].metadata.name}" 2>/dev/null || echo "")
        if [ -n "$ES_POD" ]; then
            kubectl exec -n ${NAMESPACE} $ES_POD -- curl -s http://localhost:9200/_cluster/health | jq . || echo "❌ Failed to check Elasticsearch health"
        else
            echo "❌ Elasticsearch pod not found"
        fi
        
        echo ""
        echo "=== 📊 Kibana Status ==="
        KIBANA_POD=$(kubectl get pods -n ${NAMESPACE} -l app=kibana -o jsonpath="{.items[0].metadata.name}" 2>/dev/null || echo "")
        if [ -n "$KIBANA_POD" ]; then
            kubectl exec -n ${NAMESPACE} $KIBANA_POD -- curl -s http://localhost:5601/api/status | jq '.status.overall' || echo "❌ Failed to check Kibana status"
        else
            echo "❌ Kibana pod not found"
        fi
    '''
    
    echo "✅ Health checks completed!"
}

def destroyResources() {
    echo "🗑️  Destroying ELK Stack resources..."
    
    // 删除 Kubernetes 资源
    sh '''
        echo "Deleting Kubernetes resources..."
        kubectl delete -f k8s/logstash/ -n ${NAMESPACE} --ignore-not-found=true || true
        kubectl delete -f k8s/filebeat/ -n ${NAMESPACE} --ignore-not-found=true || true
        kubectl delete -f k8s/kibana/ -n ${NAMESPACE} --ignore-not-found=true || true
        kubectl delete -f k8s/elasticsearch/ -n ${NAMESPACE} --ignore-not-found=true || true
        kubectl delete -f k8s/namespaces/logging-namespace.yaml --ignore-not-found=true || true
        echo "✅ Kubernetes resources deleted"
    '''
    
    // 销毁 DNS 记录
    dir('terraform/dns') {
        sh '''
            echo "Destroying DNS records..."
            terraform init
            terraform destroy -auto-approve
            echo "✅ DNS records destroyed"
        '''
    }
    
    // 销毁 ACM 证书
    dir('terraform/acm') {
        sh '''
            echo "Destroying ACM certificate..."
            terraform init
            terraform destroy -auto-approve
            echo "✅ ACM certificate destroyed"
        '''
    }
    
    // 销毁 Route53 区域
    dir('terraform/route53') {
        sh '''
            echo "Destroying Route53 zone..."
            terraform init
            terraform destroy -auto-approve
            echo "✅ Route53 zone destroyed"
        '''
    }
    
    // 清理 S3 中的 ALB 信息
    sh '''
        echo "Cleaning up ALB info from S3..."
        aws s3 rm s3://${TF_STATE_BUCKET}/elk-eks/alb-info.json || true
        echo "✅ ALB info cleaned up from S3"
    '''
    
    echo "✅ All ELK Stack resources have been destroyed successfully!"
}