[Jenkins][KDT 2차 프로젝트] Jenkins Pipeline : Rolling

1. 롤링 코드

pipeline {
    tools {
        gradle "GRADLE"
    }
    agent any
    stages {
        stage('Clone') {
            steps {
                git branch: 'master', url: 'https://github.com/nayeo2/KDT_prj_application.git'
            }
        }

        stage('Set Permissions') {
            steps {
                sh 'chmod +x ./gradlew'
            }
        }

        stage('Test') {
            steps {
                script {
                    sh './gradlew test'
                }
            }
        }

        stage('Build') {
            steps {
                sh './gradlew clean build'
                sh './gradlew clean build -Pdb.url=jdbc:mysql://10.0.3.31:3306/spring_security_inclass?useSSL=false -Pdb.username=master -Pdb.password=0808 -Pdb.driver=com.mysql.cj.jdbc.Driver'
                script {
                    def appServerIps = ['10.0.2.99', '10.0.2.83']
                    copyJarToRemote(appServerIps[0]) // Copy JAR to first server
                }
            }
        }

        stage('Deploy') {
            steps {
                script {
                    def appServerIps = ['10.0.2.99','10.0.2.83']
                    def nginxIp = '10.0.10.7'

                    // Deploy on first server
                    deployOnServer(appServerIps[0], nginxIp)

                    // Copy JAR to the second server
                    copyJarToRemote(appServerIps[1])

                    // Deploy on second server
                    deployOnServer(appServerIps[1], nginxIp)
                }
            }
        }
    }
    post {
        success {
            echo "Build and deployment succeeded."
        }
        failure {
            echo "Build or deployment failed."
        }
    }
}

def copyJarToRemote(targetServerIp) {
    def jarFile = '/var/lib/jenkins/workspace/A-KDT-pipeline-webhook/build/libs/inclass-spring-security-0.0.1-SNAPSHOT.jar'
    def deployPath = '/home/ubuntu'

    sshagent(['A-jenkins-key']) {
        def scpCmd = "scp -o StrictHostKeyChecking=no $jarFile ubuntu@$targetServerIp:$deployPath/"
        def scpResult = sh(script: scpCmd, returnStatus: true)
        if (scpResult != 0) {
            error "Failed to copy jar file to $targetServerIp"
        } else {
            echo "Successfully copied jar file to $targetServerIp"
        }
    }
}

def deployOnServer(ip, nginxIp) {
    def jarFile = '/home/ubuntu/inclass-spring-security-0.0.1-SNAPSHOT.jar'
    def deployPath = '/home/ubuntu'
    def runAppCommand = "nohup java -jar $jarFile > $deployPath/log.log 2>&1 &"
    def checkLogCommand = "grep -q 'Started' $deployPath/log.log"
    def backupJarFile = 'inclass-spring-security-0.0.1-SNAPSHOT-backup.jar'
    def maxAttempts = 10
    def sleepInterval = 5

    sshagent(['A-jenkins-key']) {
        echo "Starting deployment on $ip"

        // 1. Stop existing process and free port
        stopExistingProcess(ip)

        // 2. Create backup if not exists
        def checkBackupCmd = "ssh -o StrictHostKeyChecking=no ubuntu@$ip '[ ! -f $deployPath/$backupJarFile ]'"
        def checkBackupResult = sh(script: checkBackupCmd, returnStatus: true)
        if (checkBackupResult == 0) {
            def backupCmd = "ssh -o StrictHostKeyChecking=no ubuntu@$ip 'cp $deployPath/inclass-spring-security-0.0.1-SNAPSHOT.jar $deployPath/$backupJarFile'"
            def backupResult = sh(script: backupCmd, returnStatus: true)
            if (backupResult != 0) {
                error "Failed to create backup file on $ip"
            }
        } else {
            echo "Backup file already exists on $ip"
        }

        // 3. Deploy new version
        def runCmd = "ssh -o StrictHostKeyChecking=no ubuntu@$ip '$runAppCommand'"
        def runResult = sh(script: runCmd, returnStatus: true)
        if (runResult != 0) {
            error "Failed to start application on $ip"
            rollbackToPreviousVersion(ip, backupJarFile, deployPath, runAppCommand, checkLogCommand)
        }

        // 4. Check if the new version started successfully
        def attempts = 0
        def deploymentSuccess = false
        while (attempts < maxAttempts) {
            int result = sh(script: "ssh -o StrictHostKeyChecking=no ubuntu@$ip '$checkLogCommand'", returnStatus: true)
            if (result == 0) {
                echo "Deployment to $ip was successful."
                deploymentSuccess = true
                break
            }
            attempts++
            sleep sleepInterval
        }

        if (!deploymentSuccess) {
            error "Deployment to $ip failed. Rolling back to previous version."
            rollbackToPreviousVersion(ip, backupJarFile, deployPath, runAppCommand, checkLogCommand)
        }

        // 5. Run CD test
        def cdTestResult = checkApplicationStatus(ip)
        echo "CD test result for $ip: '${cdTestResult}'"
        if (cdTestResult != "200") {
            error "CD test failed for $ip. Rolling back to previous version."
            rollbackToPreviousVersion(ip, backupJarFile, deployPath, runAppCommand, checkLogCommand)
        } else {
            echo "CD test passed for $ip."
        }

        // 6. Update load balancer with new server IP
        updateLoadBalancer(ip, nginxIp, 'add')

        // Wait before moving to the next server to ensure stability
        sleep 10
    }
}

def checkApplicationStatus(ip) {
    def checkStatusCmd = "ssh -o StrictHostKeyChecking=no ubuntu@$ip 'curl -s -o /dev/null -w \"%{http_code}\" http://localhost:8080'"
    def statusCode = sh(script: checkStatusCmd, returnStdout: true).trim()
    return statusCode
}

def stopExistingProcess(ip) {
    sh script: "ssh -o StrictHostKeyChecking=no ubuntu@$ip 'pgrep -f inclass-spring-security-0.0.1-SNAPSHOT.jar && pkill -f inclass-spring-security-0.0.1-SNAPSHOT.jar || echo \"No process found\"'", returnStatus: true
    sh script: "ssh -o StrictHostKeyChecking=no ubuntu@$ip 'sudo lsof -ti:8080 | xargs sudo kill -9'", returnStatus: true
}

def rollbackToPreviousVersion(targetServerIp, backupJarFile, deployPath, runAppCommand, checkLogCommand) {
    def maxAttempts = 10
    def sleepInterval = 3

    sshagent(['A-jenkins-key']) {
        sh script: "ssh -o StrictHostKeyChecking=no ubuntu@$targetServerIp 'pgrep -f inclass-spring-security-0.0.1-SNAPSHOT.jar && pkill -f inclass-spring-security-0.0.1-SNAPSHOT.jar || echo \"No process found\"'", returnStatus: true

        sh script: """
            ssh -o StrictHostKeyChecking=no ubuntu@$targetServerIp 'if [ -f $deployPath/$backupJarFile ]; then
                mv -f $deployPath/$backupJarFile $deployPath/inclass-spring-security-0.0.1-SNAPSHOT.jar
                $runAppCommand
            else
                echo "Backup file not found on $targetServerIp"
                exit 1
            fi'
        """, returnStatus: true

        def attempts = 0
        def rollbackSuccess = false
        while (attempts < maxAttempts) {
            int result = sh(script: "ssh -o StrictHostKeyChecking=no ubuntu@$targetServerIp '$checkLogCommand'", returnStatus: true)
            if (result == 0) {
                echo "Rollback to previous version on $targetServerIp was successful."
                rollbackSuccess = true
                break
            }
            attempts++
            sleep sleepInterval
        }

        if (!rollbackSuccess) {
            error "Rollback to previous version on $targetServerIp failed."
        }
    }
}

def updateLoadBalancer(serverIp, nginxIp, action) {
    def nginxConfigPath = '/etc/nginx/nginx.conf'

    def serverCommand = action == 'remove' ?
        """
        ssh -o StrictHostKeyChecking=no ubuntu@$nginxIp 'bash -c "
            # 서버 IP 제거
            sudo sed -i \\"/server $serverIp:8080/d\\" $nginxConfigPath && \\
            
            # Nginx 구성을 테스트
            sudo nginx -t && \\
            
            # Nginx를 재로드
            sudo systemctl reload nginx
        "'
        """ :
        """
        ssh -o StrictHostKeyChecking=no ubuntu@$nginxIp 'bash -c "
            # 서버 IP 추가
            sudo sed -i \\"/upstream app {/a \\
            \\            server $serverIp:8080 weight=100 max_fails=3 fail_timeout=3s;\\" $nginxConfigPath && \\
            
            # Nginx 구성을 테스트
            sudo nginx -t && \\
            
            # Nginx를 재로드
            sudo systemctl reload nginx
        "'
        """

    def result = sh(script: serverCommand, returnStatus: true)
    if (result != 0) {
        error "Failed to ${action == 'remove' ? 'remove' : 'add'} $serverIp ${action == 'remove' ? 'from' : 'to'} load balancer"
    } else {
        echo "Successfully ${action == 'remove' ? 'removed' : 'added'} $serverIp ${action == 'remove' ? 'from' : 'to'} load balancer"
    }
}

간단한 함수 설명

• copyJarRemote(targerServerIp)
  • 빌드된 Jar 파일을 앱 서버로 복사
• deployAndTest(targetServerIp)
  • 애플리케이션 배포&테스트
• stopCurrentApplication(targetServerIp)
  • 실행 중인 애플리케이션 중지
• backupCurrentJar(targetServerIp, deployPath, backupJarFile)
  • 현재 jar 파일의 백업 생성
• startApplication(targetServerIp, runAppCommand)
  • 애플리케이션 시작
• waitForDeployment(targetServerIp, checkLogCommand, maxAttempts)
  • 배포 확인을 위한 로그 검사
• performCDTest(targetServerIp)
  • CD 테스트 수행
• rollbackToPreviousVersion(targetServerIp, backupJarFile, deployPath, runAppCommand, checkLogCommand)
  • 배포 실패 시에 이전 버전으로 롤백

 

Sucess !