.gitlab-ci.yml

default:
  image: $CI_REGISTRY_IMAGE:latest

variables:
  CAMPUS_IP: 138.26.48.47
  CHEAHA_IP: 172.20.10.9
  TEST_IP: 138.26.49.134
  ANSIBLE_REMOTE_TMP: "/tmp"
  AWS_DEFAULT_REGION: "bhm"
  AWS_HOST: "s3.lts.rc.uab.edu"
  OS_AUTH_TYPE: "v3applicationcredential"
  OS_AUTH_URL: "https://keystone.cloud.rc.uab.edu:5000/v3"
  OS_IDENTITY_API_VERSION: "3"
  OS_INTERFACE: "public"
  OS_REGION_NAME: "bhm1"
  PKR_VAR_flavor: "m1.small"
  PKR_VAR_source_image: "CentOS-7-x86_64-GenericCloud-2009"
  PKR_VAR_floating_ip_network: "uab-campus"
  PKR_VAR_security_groups: '["allow-ssh"]'
  PKR_VAR_skip_create_image: "false"
  PKR_VAR_ssh_username: "centos"
  PKR_VAR_networks: '["ec11e2be-8fac-46cf-8fa2-9dffb74ba5df"]'
  GIT_AUTHOR_NAME: "Gitlab runner"
  GIT_AUTHOR_EMAIL: "gitlab@runner"
  NUM_SERVER_TO_KEEP: 1
  NUM_IMAGE_TO_KEEP: 30
  PKR_VAR_root_ssh_key: "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAFqqWgmYpEaGtHBeTu27ntVJpYjwq/x5aBefrvfhk8Z9lE3cuZ26vJ9n/9tGE4Zn2Pew1mpZgi6PzfJ3vMt8yA= root@master"
  DEV_KEY: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCpncAcYosVHt7HsUcE2XOYDuCi4HQnmFJv279LOcpZgXtZ6o0BM1fe5FgJS0X1ohBXQUFRuYJuJSW/GSmC1K8T+wCrKjZLJdMbqrubHV27diUZfdoVkoJy1vcAQF5nEcoTC7MpAFbBomdn2rsrpgQe8DGiURV7+soqybXV1OsIR3FFf6npnUaskHYT/oVtG9eBOnscyBxoVgbxzlmyoBLXED/sHKFw4nQSF/glYKEFiDu6TRTsBBEGvv23Qo/66QpQiFJ6TNfApNiyY9L1X+Dy8EWU6lozmNgwGDjXQ70Lr6xHnA0QGVALJlHXa6QjpgtpC5Nefsdvtf1hpfFo2VutpbSB+aq9jk3gWNN+XkhrWN5PiwP7YYJNw/WozyfL+IhwjfHZGxkuws+wGR6ZKxlX9W9Vrsq9ncYNKuhy2SdsR6s2XECQtrEQ6ZlX5jRt6Yh5M9ls5fMsWEqknDPmr1Ui6wV7NxprYngo9fLSdYO/ETIO3S6PB0aEHOZOyGitGaM06EmNpvjQn/QkkaVgt/O8wKL1o1AVzXhDMAFvtG6ejppV6kuTUHXFgSGZF6N9fnP91HuytyzC09F+NMWcmnRdrgXlHapjuuL3zzi+XLCQvk8+aYTzBKx1nU2FPMDRZ9sInGmqdTuM002E7qVbaCy4OxcWaAS/L2UVhGnHr+egYw== louistw@uab.edu"

stages:
  - pre-build
  - build
  - test
  - deploy

workflow:
  rules:
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    - if: $CI_PIPELINE_SOURCE == 'schedule'

.get_build_date: &get_build_date
  - export BUILD_DATE=$(TZ=America/Chicago date +%Y%m%d)
  - echo BUILD_DATE=${BUILD_DATE}

.update_ansible_repo: &update_ansible_repo
  - *get_build_date
  - |
    if [ ! -d $CI_PROJECT_DIR/CRI_XCBC ]; then
      git clone https://github.com/uabrc/CRI_XCBC.git
      cd CRI_XCBC
      git remote add upstream https://github.com/jprorama/CRI_XCBC.git
      cd ..
    fi
  - cd CRI_XCBC
  - git config user.name "${GIT_AUTHOR_NAME}"
  - git config user.email "${GIT_AUTHOR_EMAIL}"
  - git fetch origin uab-prod
  - git fetch upstream dev
  - git checkout uab-prod
  - git merge origin/uab-prod
  - git checkout -b integration
  - git merge upstream/dev
  - export CRI_XCBC_HEAD=$(git rev-parse --short HEAD)
  - export CRI_XCBC_dev=$(git rev-parse --short upstream/dev)
  - export CRI_XCBC_prod=$(git rev-parse --short origin/uab-prod)
  - cd ..
  - export PACKER_IMAGE_HEAD=$(git rev-parse --short HEAD)
  - echo CRI_XCBC_HEAD=${CRI_XCBC_HEAD} | tee -a $CI_PROJECT_DIR/image.env
  - echo CRI_XCBC_dev=${CRI_XCBC_dev} | tee -a $CI_PROJECT_DIR/image.env
  - echo CRI_XCBC_prod=${CRI_XCBC_prod} | tee -a $CI_PROJECT_DIR/image.env
  - echo PACKER_IMAGE_HEAD=${PACKER_IMAGE_HEAD} | tee -a $CI_PROJECT_DIR/image.env

.get_ansible_files: &get_ansible_files
  - s3cmd get --force -r --host=$AWS_HOST --host-bucket=$AWS_HOST s3://cheaha-cloud-ansible-files/ ansible/files/

build_docker_image:
  image: docker:20.10.17
  stage: pre-build
  services:
    - docker:20.10.16-dind
  tags:
    - dind
  before_script:
    - *get_build_date
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
  script:
    - docker build -t $CI_REGISTRY_IMAGE:$BUILD_DATE -t $CI_REGISTRY_IMAGE:latest .
    - >
      docker run --rm $CI_REGISTRY_IMAGE bash -c
      'ansible --version &&
      openstack --version &&
      packer version &&
      s3cmd --version &&
      terraform --version'
    - docker push --all-tags $CI_REGISTRY_IMAGE
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      changes:
        - Dockerfile
      allow_failure: true

build_base_image:
  stage: build
  tags:
    - build
  script:
    - |
      if [ -n "${BUILT_BASE_IMAGE_ID}" ]; then
        exit 0
      fi
    - *update_ansible_repo
    - *get_ansible_files
    - export REPO_HEAD=$(git rev-parse --short HEAD)
    - export PKR_VAR_flavor="m1.medium"
    - export PKR_VAR_build_instance_name="base-${REPO_HEAD}"
    - export PKR_VAR_image_date_suffix=false
    - |
      if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then
        export PKR_VAR_image_name="base-PR-${CI_MERGE_REQUEST_IID}"
      elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then
        export PKR_VAR_image_name="base-${BUILD_DATE}"
      fi
    - packer validate openstack
    - packer build -machine-readable openstack | tee base_build.log
    - export BUILT_BASE_IMAGE_ID=$(grep 'Image:' base_build.log | awk '{print $4}')
    - echo BUILT_BASE_IMAGE_ID=${BUILT_BASE_IMAGE_ID} | tee -a $CI_PROJECT_DIR/image.env
  artifacts:
    reports:
      dotenv: image.env
    expire_in: 30 days

build_compute_image:
  stage: build
  needs: [build_base_image]
  tags:
    - build
  script:
    - *update_ansible_repo
    - *get_ansible_files
    - export PKR_VAR_source_image=${BUILT_BASE_IMAGE_ID}
    - export REPO_HEAD=$(git rev-parse --short HEAD)
    - export PKR_VAR_flavor="m1.medium"
    - export PKR_VAR_build_instance_name="compute-${REPO_HEAD}"
    - export PKR_VAR_image_date_suffix=false
    - |
      if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then
        export PKR_VAR_image_name="compute-PR-${CI_MERGE_REQUEST_IID}"
      elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then
        export PKR_VAR_image_name="compute-${BUILD_DATE}"
      fi
    - packer validate openstack-compute
    - packer build -machine-readable openstack-compute | tee compute_build.log

build_gpu_image:
  stage: build
  needs: [build_base_image]
  tags:
    - build
  script:
    - *update_ansible_repo
    - *get_ansible_files
    - FAILED=false
    - export GPU_PLACEHOLDER_NAME="gpu1-placeholder"
    - export GPU_PLACEHOLDER_FLAVOR="gpu1.medium"
    - export GPU_PLACEHOLDER_IMAGE="CentOS-7-x86_64-GenericCloud-2009"
    - export PKR_VAR_source_image=${BUILT_BASE_IMAGE_ID}
    - export REPO_HEAD=$(git rev-parse --short HEAD)
    - export PKR_VAR_flavor="gpu1.medium"
    - export PKR_VAR_build_instance_name="gpu-${REPO_HEAD}"
    - export PKR_VAR_image_date_suffix=false
    - |
      if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then
        export PKR_VAR_image_name="gpu-PR-${CI_MERGE_REQUEST_IID}"
      elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then
        export PKR_VAR_image_name="gpu-${BUILD_DATE}"
      fi
    - packer validate openstack-gpu
    - openstack server delete --wait $GPU_PLACEHOLDER_NAME
    - packer build -machine-readable openstack-gpu | tee gpu_build.log || FAILED=true
    - openstack server create --image $GPU_PLACEHOLDER_IMAGE --network cicd-net --flavor $GPU_PLACEHOLDER_FLAVOR $GPU_PLACEHOLDER_NAME
    - |
      if [ "$FAILED" = true ]; then
        exit 1
      fi

build_ood_image:
  stage: build
  environment:
    name: knightly
  tags:
    - build
  script:
    - *update_ansible_repo
    - *get_ansible_files
    - >
      curl --header "PRIVATE-TOKEN: ${ANSIBLE_VAR_TOKEN}"
      "${CI_API_V4_URL}/projects/2836/repository/files/knightly/raw?ref=main"
      -o CRI_XCBC/group_vars/knightly
    - 'sed -i -E "s/(lts_access_key: ).*/\1\"${AWS_ACCESS_KEY_ID}\"/" CRI_XCBC/group_vars/knightly'
    - 'sed -i -E "s/(lts_secret_key: ).*/\1\"${AWS_SECRET_ACCESS_KEY}\"/" CRI_XCBC/group_vars/knightly'
    - 'sed -i -E "s/(user_register_app_key: ).*/\1\"${SELF_REG_APP_KEY}\"/" CRI_XCBC/group_vars/knightly'
    - 'sed -i -E "s/(celery_user_password: ).*/\1\"${CELERY_PASSWD}\"/" CRI_XCBC/group_vars/knightly'
    - 'sed -i -E "s|(ssh_pub_key: ).*|\1\"{{ lookup(''file'', ''${SSH_PUB_KEY}'') }}\"|" CRI_XCBC/group_vars/knightly'
    - packer validate openstack-ood
    - |
      if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then
        export PKR_VAR_image_name="ood-PR-${CI_MERGE_REQUEST_IID}"
        echo INSTANCE_FLAVOR="m1.small" | tee -a $CI_PROJECT_DIR/image.env
        echo OOD_INSTANCE_NAME="ood-PR-${CI_MERGE_REQUEST_IID}" | tee -a $CI_PROJECT_DIR/image.env
        export FLOATING_IP=$(openstack floating ip create uab-campus -f value -c floating_ip_address)
        echo FLOATING_IP=$FLOATING_IP | tee -a $CI_PROJECT_DIR/image.env
        sed -i -E "s/(ood_servername: ).*/\1\"$CI_COMMIT_REF_SLUG.$FLOATING_IP.nip.io\"/" CRI_XCBC/group_vars/knightly
      elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then
        export PKR_VAR_image_name="ood-${BUILD_DATE}"
        echo INSTANCE_FLAVOR="cpu16-64g" | tee -a $CI_PROJECT_DIR/image.env
        echo OOD_INSTANCE_NAME="ood-knightly" | tee -a $CI_PROJECT_DIR/image.env
        echo FLOATING_IP=$TEST_IP | tee -a $CI_PROJECT_DIR/image.env
      fi
    - >
      PKR_VAR_build_instance_name="ood-${CRI_XCBC_HEAD}"
      PKR_VAR_image_date_suffix=false
      packer build -machine-readable openstack-ood | tee ood_build.log
    - export BUILT_OOD_IMAGE_ID=$(grep 'Image:' ood_build.log | awk '{print $4}')
    - echo BUILT_OOD_IMAGE_ID=${BUILT_OOD_IMAGE_ID} | tee -a $CI_PROJECT_DIR/image.env
    - openstack image set --property CRI_XCBC_prod=${CRI_XCBC_prod} --property CRI_XCBC_dev=${CRI_XCBC_dev} --property PACKER_IMAGE_HEAD=${PACKER_IMAGE_HEAD} ${BUILT_OOD_IMAGE_ID}
  artifacts:
    reports:
      dotenv: image.env

test_ood_image:
  stage: test
  needs: [build_ood_image]
  environment:
    name: knightly
  tags:
    - build
  script:
    - FAILED=false
    - |
      eval $(ssh-agent -s)
      chmod 400 "$SSH_PRIV_KEY"
      ssh-add "$SSH_PRIV_KEY"
      mkdir ~/.ssh
      chmod 700 ~/.ssh
    - OLD_INSTANCE_IP=$(openstack floating ip list --floating-ip-address $CHEAHA_IP -c "Fixed IP Address" -f value)
    - echo $OLD_INSTANCE_IP
    - |
      if [ ! -z $OLD_INSTANCE_IP ]; then
        export OLD_INSTANCE_ID=$(openstack server list --name $OOD_INSTANCE_NAME --ip $OLD_INSTANCE_IP -c ID -f value)
      fi
    - echo OLD_INSTANCE_ID=$OLD_INSTANCE_ID | tee -a instance.env
    - |
      cat > user_data.txt << OEOF
      #!/bin/bash
      cat > /etc/resolv.conf << EOF
      search openstack.internal cm.cluster rc.uab.edu ib.cluster drac.cluster eth.cluster ib-hdr.cluster
      nameserver 172.20.0.25
      EOF
      echo "$DEV_KEY" >> /root/.ssh/authorized_keys
      ip route replace default via 10.250.0.3 dev eth0
      ip route add 172.20.0.0/16 via 10.250.0.1 dev eth0
      mkdir -p /run/shibboleth
      chown shibd:shibd /run/shibboleth
      pip3 install s3cmd
      s3cmd get --force -r --access_key=$AWS_ACCESS_KEY_ID --secret_key=$AWS_SECRET_ACCESS_KEY --host=$AWS_HOST --host-bucket=$AWS_HOST s3://knightly-key/ /etc/ssh/
      mount | grep home || mount /home
      OEOF
    - >
      export NEW_INSTANCE_ID=$(openstack server create
      -c id -f value --image $BUILT_OOD_IMAGE_ID
      --network openstack-cheaha-internal
      --security-group ood-https-ports
      --security-group allow-ssh
      --user-data user_data.txt
      --flavor $INSTANCE_FLAVOR
      --wait
      $OOD_INSTANCE_NAME)
    - echo NEW_INSTANCE_ID=$NEW_INSTANCE_ID | tee -a instance.env
    - openstack server add floating ip $NEW_INSTANCE_ID $FLOATING_IP
    - >
      curl --retry 10 --retry-delay 20 --retry-connrefused https://knightly.rc.uab.edu/Shibboleth.sso/Metadata --resolve knightly.rc.uab.edu:443:$FLOATING_IP -kf
      || FAILED=true
    - |
      if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then
        ssh -o StrictHostKeyChecking=no acctsvc@$FLOATING_IP '[ $(mount | grep "type nfs" | wc -l) -eq 6 ]' || FAILED=true
      elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then
        cp "$SSH_KNOWN_HOSTS" ~/.ssh/known_hosts
        chmod 644 ~/.ssh/known_hosts
        ssh acctsvc@$FLOATING_IP '[ $(mount | grep "type nfs" | wc -l) -eq 6 ]' || FAILED=true
      fi
    - |
      if [ "$FAILED" = true ]; then
        openstack server delete $NEW_INSTANCE_ID
        openstack image delete $BUILT_OOD_IMAGE_ID
        false
      fi
    - |
      if [ $CI_PIPELINE_SOURCE = "schedule" ]; then
        openstack server remove floating ip $NEW_INSTANCE_ID $FLOATING_IP
      fi
  artifacts:
    reports:
      dotenv: instance.env

deploy_review:
  stage: deploy
  script:
    - echo "Deploy Review App"
  environment:
    name: review/$CI_COMMIT_REF_SLUG
    url: https://$CI_COMMIT_REF_SLUG.$FLOATING_IP.nip.io
    on_stop: stop_review
    auto_stop_in: 2 days
  tags:
    - build
  rules:
    - if: $CI_MERGE_REQUEST_ID

stop_review:
  stage: deploy
  script:
    - openstack server delete $NEW_INSTANCE_ID
    - openstack image delete $BUILT_OOD_IMAGE_ID
    - openstack floating ip delete $FLOATING_IP
  environment:
    name: review/$CI_COMMIT_REF_SLUG
    action: stop
  tags:
    - build
  rules:
    - if: $CI_MERGE_REQUEST_ID
      when: manual

deploy_knightly:
  stage: deploy
  environment:
    name: knightly
  tags:
    - build
  script:
    - |
      if [ ! -z $OLD_INSTANCE_ID ]; then
        openstack server remove floating ip $OLD_INSTANCE_ID $CAMPUS_IP
        openstack server remove floating ip $OLD_INSTANCE_ID $CHEAHA_IP
      fi
    - |
      if [ ! -z $NEW_INSTANCE_ID ]; then
        openstack server add floating ip $NEW_INSTANCE_ID $CAMPUS_IP
        openstack server add floating ip $NEW_INSTANCE_ID $CHEAHA_IP
      fi
    - |
      SERVER_TO_BE_DELETE=($(openstack server list --name $OOD_INSTANCE_NAME --sort-column Image --sort-descending -f value -c ID | sed -n $(($NUM_SERVER_TO_KEEP+1))',$p'))
      IMAGE_TO_BE_DELETE=($(openstack image list --sort-column Name --sort-descending -f value -c Name -c ID | grep -P ' ood-\d{8}$' | sed -n $(($NUM_IMAGE_TO_KEEP+1))',$p' | awk '{print $1}'))
      for svr in ${SERVER_TO_BE_DELETE[@]}; do
        openstack server delete ${svr}
      done
      for img in ${IMAGE_TO_BE_DELETE[@]}; do
        openstack image delete ${img}
      done
  only:
    - schedules

deploy_cheaha:
  stage: deploy
  environment:
    name: cheaha
  tags:
    - build
  script:
    - echo "Job placeholder to deploy to Cheaha"
  when: manual
  only:
    - main