Relk's 工作手札

在 Cloud-Native 汪洋中載浮載沉的 SRE - Service Restart Engineer

0%

[DevOps] 利用 Jenkins 與 Ansible 搭建 CI/CD Pipeline

簡易 CI/CD Pipeline 設計原則

範例用 GitHub Repository

  1. 遵循 GitHub Flow,master 分支永遠是經過驗證且可佈署的
  2. 建立新 Pull Request 時,自動 deploy 到 Staging 環境並執行 Smoke Testing,等團隊其他成員或主管完成 Code Review 以及測試後才能 Mergemaster
  3. 只有被 Tag 的 commit,才能 deploy 到 Production 環境(手動)
  4. 為了方便測試,可以手動 deploy 其他 branch 到 Staging 環境

Pipeline 設定

Jenkinsfile 說明

  1. 全域環境變數設定

    1
    2
    3
    4
    5
    6
    7
    8
    environment {
    # 登入 Docker Hub 的 credential
    DOCKER_HUB_CREDS = credentials('dockerhub-downager')
    # 設定 Staging 和 Production 環境 Postgresql UserName & Password
    FLASK_POSTGRES_CREDS = credentials('flask-app-postgres-user-pass')
    # 決定本次佈署是否要執行 db_migrate.sh 這隻腳本
    DB_MIGRATION = false
    }
  2. 在 Jenkins Node 上 build 出 UnitTest 專用的 docker image,並進行測試

    1
    2
    3
    4
    5
    6
    stage('Test') {
    steps {
    sh 'docker build -t flask-realworld-example-app:UnitTest -f ./UnitTest_Dockerfile .'
    sh 'docker run flask-realworld-example-app:UnitTest'
    }
    }
  3. Build 出 Production 專用的 Image,並推至 Private Docker Registry,並用 COMMIT_ID 當作 Image Tag

    1
    2
    3
    4
    5
    6
    7
    stage('Build') {
    steps {
    sh 'docker login -u $DOCKER_HUB_CREDS_USR -p $DOCKER_HUB_CREDS_PSW'
    sh 'docker build -t downager/flask-realworld-example-app:$GIT_COMMIT .'
    sh 'docker push downager/flask-realworld-example-app:$GIT_COMMIT'
    }
    }
  4. 每次觸發 Jenkins Job 時都會佈署至 Staging 環境,並執行 Smoke Testing

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    stage('Deploy - Staging') {
    environment {
    # 設定本次 Ansible Playbook 要執行的 Host Group
    HOST_GROUP = 'Staging'
    # 設定本次 docker-compose 的 image tag,Staging 環境是 COMMIT_ID
    IMAGE_TAG = sh(returnStdout: true, script: 'echo $GIT_COMMIT')
    }
    steps {
    echo 'Deploy to staging servers'
    ansiColor(colorMapName: 'xterm') {
    ansiblePlaybook(
    disableHostKeyChecking: true,
    # 設定登入 Staging 環境的 SSH KEY
    credentialsId: 'devops-ssh-key',
    playbook: 'ansible/playbook-deploy-flask-app.yml',
    inventory: 'ansible/hosts',
    colorized: true)
    }
    echo 'Make a smoke testing on staging servers (MAX_RETRIES = 3)'
    # 進行 Smoke Testing,如果連續三次都失敗才報錯退出
    retry(3) {
    ansiColor(colorMapName: 'xterm') {
    ansiblePlaybook(
    disableHostKeyChecking: true,
    # 設定登入 Staging 環境的 SSH KEY
    credentialsId: 'devops-ssh-key',
    playbook: 'ansible/playbook-smoke-testing.yml',
    inventory: 'ansible/hosts',
    colorized: true)
    }
    }
    }
    }
  5. 佈署至 Production 環境,只有觸發帶 Tag 的 Job 才會執行此階段

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    stage('Deploy - Production') {
    when {
    tag '*'
    }
    environment {
    # 設定本次 Ansible Playbook 要執行的 Host Group
    HOST_GROUP = 'Production'
    # 設定本次 docker-compose 的 image tag,Production 環境是 Git tag
    IMAGE_TAG = sh(returnStdout: true, script: 'echo $TAG_NAME')
    }
    steps {
    echo 'Deploying only because this commit is tagged'
    # 將本次 commit 的 image 打上 Git Tag,並推上 docker registry
    sh 'docker tag downager/flask-realworld-example-app:$GIT_COMMIT downager/flask-realworld-example-app:$TAG_NAME'
    sh 'docker push downager/flask-realworld-example-app:$TAG_NAME'
    echo 'Deploy to production servers'
    ansiColor(colorMapName: 'xterm') {
    ansiblePlaybook(
    disableHostKeyChecking: true,
    # 設定登入 Production 環境的 SSH KEY
    credentialsId: 'devops-ssh-key',
    playbook: 'ansible/playbook-deploy-flask-app.yml',
    inventory: 'ansible/hosts',
    colorized: true)
    }
    echo 'Make a smoke testing on production servers (MAX_RETRIES = 3)'
    # 進行 Smoke Testing,如果連續三次都失敗才報錯退出
    retry(3) {
    ansiColor(colorMapName: 'xterm') {
    ansiblePlaybook(
    disableHostKeyChecking: true,
    # 設定登入 Production 環境的 SSH KEY
    credentialsId: 'devops-ssh-key',
    playbook: 'ansible/playbook-smoke-testing.yml',
    inventory: 'ansible/hosts',
    colorized: true)
    }
    }
    }
    }

    Ansible Playbook 說明

    playbook-deploy-flask-app.yml

  6. 建立資料夾以及設定權限

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    # 取得 Jenkinsfile 設定的 HOST_GROUP 環境變數,決定要 deploy 到 Staging 或 Production 環境
    - hosts: "{{ lookup('env','HOST_GROUP') }}"
    remote_user: devops
    vars:
    # 取得 DB_MIGRATION 環境變數,決定是否執行 DB migrate 步驟
    DB_MIGRATION: "{{ lookup('env','DB_MIGRATION') }}"
    tasks:
    - name: Change ownership of /opt directory
    file:
    path: /opt
    state: directory
    owner: devops
    group: devops
    mode: "0775"
    become: yes

    - name: Create flask-realworld-example-app directory
    file:
    path: /opt/flask-realworld-example-app
    state: directory
    owner: devops
    group: devops
    mode: "0775"
  7. 複製 docker-compose.yaml,以及將 container 所需環境變數寫入 .env

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    - name: Copy docker-compose file
    copy:
    src: docker-compose.yaml
    dest: /opt/flask-realworld-example-app

    - name: Create .env file
    copy:
    dest: /opt/flask-realworld-example-app/.env
    content: |
    TAG={{ lookup('env','IMAGE_TAG') }}
    POSTGRES_USER={{ lookup('env','FLASK_POSTGRES_CREDS_USR') }}
    POSTGRES_PASSWORD={{ lookup('env','FLASK_POSTGRES_CREDS_PSW') }}
  8. 登入 Private Docker Registry,以及使用 docker-compose 管理容器編排

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    - name: Log into DockerHub
    docker_login:
    username: "{{ lookup('env','DOCKER_HUB_CREDS_USR') }}"
    password: "{{ lookup('env','DOCKER_HUB_CREDS_PSW') }}"

    - name: docker-compose down 'flask-realworld-example-app'
    docker_compose:
    project_src: /opt/flask-realworld-example-app
    state: absent
    register: output

    - name: docker-compose up 'flask-realworld-example-app'
    docker_compose:
    project_src: /opt/flask-realworld-example-app
    state: present
    # 強迫重新拉取 image
    pull: yes
    register: output
  9. 根據環境變數設定決定是否執行 ./db_migrate.sh 腳本

    1
    2
    3
    - name: DB migrate - flask-realworld-example-app 
    command: docker exec flask-realworld-example-app bash -c './db_migrate.sh'
    when: DB_MIGRATION == true

playbook-smoke-testing.yml

  • 登入到主機中,進行簡易的本機測試,只要回傳的不是 HTTP 200 就會報錯,退出 Jenkins Job
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    # 取得 Jenkinsfile 設定的 HOST_GROUP 環境變數,決定要測試 Staging 或 Production 環境
    - hosts: "{{ lookup('env','HOST_GROUP') }}"
    remote_user: devops
    vars:
    HOST_NAME: "http://localhost:8080"
    tasks:
    - name: GET '/api/tags'
    uri:
    url: "{{ HOST_NAME }}/api/tags"
    method: GET
    status_code: 200

    - name: GET '/api/articles'
    uri:
    url: "{{ HOST_NAME }}/api/articles"
    method: GET
    status_code: 200

    docker-compose 說明

  • docker-compose.yaml 上的環境變數設定,都是由 Ansible Playbook 寫入 .env 檔案中,如果需要人工介入重啟 docker-compose 時才不會缺少環境變數
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    version: '3'
    services:
    web:
    container_name: flask-realworld-example-app
    image: downager/flask-realworld-example-app:${TAG}
    environment:
    - DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db/flask-app
    volumes:
    - ./app/migrations:/app/migrations
    ports:
    - 8080:80
    restart: always
    networks:
    - web
    db:
    container_name: postgres
    image: postgres:12.1
    environment:
    - POSTGRES_USER=${POSTGRES_USER:-postgres}
    - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-changeme}
    - POSTGRES_DB=flask-app
    volumes:
    - pgdata:/var/lib/postgresql/data/
    ports:
    - 5432:5432
    restart: always
    networks:
    - web
    networks:
    web:
    driver: bridge
    volumes:
    pgdata:
    driver: local

    Jenkins & GitHub 設定

  1. 請先參考本篇進行初步設定:
    GitHub Repository 有變動時自動通知 Jenkins 2 進行編譯建置 (GitHub Plugin)

  2. 調整 GitHub Webhook 設定,只勾選Branch or tag creation / deletionPull requestsPushes

  3. 回到 Jenkins,設定 Discover tags

  4. 設定預設行為 Suppress automatic SCM triggering,只有 masterPull Requests 觸發 Jenkins Job

成果截圖

  • 當新的 Pull Request 建立時,會自動 Trigger Jenkins 執行測試,以及回報結果至 GitHub PR 上

  • 只有帶 Tag 的事件才會進入 Deploy - Production 階段

  • Ansible Playbook 執行截圖

  • Jenkins BlueOcean 截圖

  • Jenkins Stage View

參考資料

Follow me at the links below