From d568a47618bca7576d24e409dc41efcf06a0687b Mon Sep 17 00:00:00 2001 From: Bo-Chun Louis Chen <louistw@uab.edu> Date: Mon, 2 Dec 2024 13:04:27 -0600 Subject: [PATCH 1/4] feat: remove base compute and gpu build from pipeline --- .gitlab-ci.yml | 92 -------------------------------------------------- 1 file changed, 92 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4bd9f80..c8a45b2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -104,98 +104,6 @@ build_docker_image: - Dockerfile allow_failure: true -build_base_image: - stage: build - tags: - - build - script: - - | - if [ -n "${BUILT_BASE_IMAGE_ID}" ]; then - exit 0 - fi - - *update_ansible_repo - - *get_ansible_files - - export REPO_HEAD=$(git rev-parse --short HEAD) - - export PKR_VAR_flavor="${BASE_BUILD_FLAVOR:-$PKR_VAR_flavor}" - - export PKR_VAR_build_instance_name="base-${REPO_HEAD}" - - export PKR_VAR_image_date_suffix=false - - | - if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then - export PKR_VAR_image_name="base-PR-${CI_MERGE_REQUEST_IID}" - elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then - export PKR_VAR_image_name="base-${BUILD_DATE}" - fi - - packer init openstack - - packer validate openstack - - packer build -machine-readable openstack | tee base_build.log - - export BUILT_BASE_IMAGE_ID=$(grep 'Image:' base_build.log | awk '{print $4}') - - echo BUILT_BASE_IMAGE_ID=${BUILT_BASE_IMAGE_ID} | tee -a $CI_PROJECT_DIR/image.env - - openstack image unset --property signature_verified $BUILT_BASE_IMAGE_ID - artifacts: - reports: - dotenv: image.env - expire_in: 30 days - -build_compute_image: - stage: build - needs: [build_base_image] - tags: - - build - script: - - *update_ansible_repo - - *get_ansible_files - - export PKR_VAR_source_image=${BUILT_BASE_IMAGE_ID} - - export REPO_HEAD=$(git rev-parse --short HEAD) - - export PKR_VAR_flavor="${COMPUTE_BUILD_FLAVOR:-$PKR_VAR_flavor}" - - export PKR_VAR_build_instance_name="compute-${REPO_HEAD}" - - export PKR_VAR_image_date_suffix=false - - | - if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then - export PKR_VAR_image_name="compute-PR-${CI_MERGE_REQUEST_IID}" - elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then - export PKR_VAR_image_name="compute-${BUILD_DATE}" - fi - - packer init openstack-compute - - packer validate openstack-compute - - packer build -machine-readable openstack-compute | tee compute_build.log - -build_gpu_image: - stage: build - needs: [build_base_image] - tags: - - build - script: - - *update_ansible_repo - - *get_ansible_files - - FAILED=false - - export GPU_PLACEHOLDER_NAME="gpu1-placeholder" - - export GPU_PLACEHOLDER_FLAVOR="gpu1.medium" - - export GPU_PLACEHOLDER_IMAGE="CentOS-7-x86_64-GenericCloud-2009" - - export PKR_VAR_source_image=${BUILT_BASE_IMAGE_ID} - - export REPO_HEAD=$(git rev-parse --short HEAD) - - export PKR_VAR_flavor="${GPU_BUILD_FLAVOR:-gpu1.medium}" - - export PKR_VAR_build_instance_name="gpu-${REPO_HEAD}" - - export PKR_VAR_image_date_suffix=false - - | - if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then - export PKR_VAR_image_name="gpu-PR-${CI_MERGE_REQUEST_IID}" - elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then - export PKR_VAR_image_name="gpu-${BUILD_DATE}" - fi - - packer init openstack-gpu - - packer validate openstack-gpu - - openstack server delete --wait $GPU_PLACEHOLDER_NAME - - packer build -machine-readable openstack-gpu | tee gpu_build.log || FAILED=true - - openstack server create --image $GPU_PLACEHOLDER_IMAGE --network cicd-net --flavor $GPU_PLACEHOLDER_FLAVOR $GPU_PLACEHOLDER_NAME - - | - if [ "$FAILED" = true ]; then - exit 1 - fi - rules: - - if: $SKIP_GPU_BUILD == "true" - when: never - - when: always - build_ood_image: stage: build tags: -- GitLab From e72c55852142a6d2439cb11697137b894beb52a5 Mon Sep 17 00:00:00 2001 From: Bo-Chun Louis Chen <louistw@uab.edu> Date: Mon, 2 Dec 2024 15:44:50 -0600 Subject: [PATCH 2/4] feat: remove cleanup for base comput and gpu image --- .gitlab-ci.yml | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c8a45b2..d1cb167 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -377,41 +377,11 @@ cleanup_integration: | awk -v NITK=$NUM_IMAGE_TO_KEEP -v REGEX=ood-$TIMESTAMP_REGEX '{if ($0 ~ REGEX) result[count++] = $1} END {for(i=NITK;i<count;i++) print result[i]}')) - - > - BASE_IMAGE_TO_BE_DELETE=($(cat images.txt - | awk -v NITK=$NUM_IMAGE_TO_KEEP -v REGEX=base-$TIMESTAMP_REGEX - '{if ($0 ~ REGEX) result[count++] = $1} - END {for(i=NITK;i<count;i++) print result[i]}')) - - > - COMPUTE_IMAGE_TO_BE_DELETE=($(cat images.txt - | awk -v NITK=$NUM_IMAGE_TO_KEEP -v REGEX=compute-$TIMESTAMP_REGEX - '{if ($0 ~ REGEX) result[count++] = $1} - END {for(i=NITK;i<count;i++) print result[i]}')) - - > - GPU_IMAGE_TO_BE_DELETE=($(cat images.txt - | awk -v NITK=$NUM_IMAGE_TO_KEEP -v REGEX=gpu-$TIMESTAMP_REGEX - '{if ($0 ~ REGEX) result[count++] = $1} - END {for(i=NITK;i<count;i++) print result[i]}')) - | for img in ${OOD_IMAGE_TO_BE_DELETE[@]}; do echo "Deleting image $img" openstack image delete ${img} done - - | - for img in ${BASE_IMAGE_TO_BE_DELETE[@]}; do - echo "Deleting image $img" - openstack image delete ${img} - done - - | - for img in ${COMPUTE_IMAGE_TO_BE_DELETE[@]}; do - echo "Deleting image $img" - openstack image delete ${img} - done - - | - for img in ${GPU_IMAGE_TO_BE_DELETE[@]}; do - echo "Deleting image $img" - openstack image delete ${img} - done rules: - if: $CI_PIPELINE_SOURCE == "schedule" when: always -- GitLab From 59cac1a9672051a2b523f61a198b1707a74ec0fd Mon Sep 17 00:00:00 2001 From: Dylan Wheeler <dwheeler@Dylans-MacBook-Pro.local> Date: Wed, 8 Jan 2025 14:57:15 -0600 Subject: [PATCH 3/4] adding scripts file to allow copying of variables from established project to a peronsal copy --- scripts/gitlab_script.py | 23 +++++++++++++++++++++++ scripts/requirements.txt | 7 +++++++ 2 files changed, 30 insertions(+) create mode 100644 scripts/gitlab_script.py create mode 100644 scripts/requirements.txt diff --git a/scripts/gitlab_script.py b/scripts/gitlab_script.py new file mode 100644 index 0000000..1a0f044 --- /dev/null +++ b/scripts/gitlab_script.py @@ -0,0 +1,23 @@ +import gitlab +import json + +config_json = [] + +with open('config.json','r') as file: + config_json = json.load(file) + +gl = gitlab.Gitlab( url='https://gitlab.rc.uab.edu', private_token=config_json["access_token"]) + + +hpc_project_main = gl.projects.get(config_json["from_project_id"]) +personal_hpc_project = gl.projects.get(config_json["to_project_id"]) + +from_variables = hpc_project_main.variables.list(get_all=True) + + +for vars in from_variables: + print(vars.key) + if vars.environment_scope == '*' or vars.environment_scope == 'dev': + personal_hpc_project.variables.create({'key':vars.key,'value':vars.value, 'environment_scope':vars.environment_scope}) + +print(personal_hpc_project.variables.list(get_all=True).count) \ No newline at end of file diff --git a/scripts/requirements.txt b/scripts/requirements.txt new file mode 100644 index 0000000..9d9e96f --- /dev/null +++ b/scripts/requirements.txt @@ -0,0 +1,7 @@ +certifi==2024.12.14 +charset-normalizer==3.4.1 +idna==3.10 +python-gitlab==5.3.1 +requests==2.32.3 +requests-toolbelt==1.0.0 +urllib3==2.3.0 -- GitLab From 8c0b7634d06c21a964f850021daf04b77b20b88c Mon Sep 17 00:00:00 2001 From: Dylan Wheeler <dwheeler@Dylans-MacBook-Pro.local> Date: Wed, 8 Jan 2025 15:03:25 -0600 Subject: [PATCH 4/4] added small readme file to explain the required config file --- scripts/README.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 scripts/README.md diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..d2a885e --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,7 @@ +You will need to create a config.json file in the scripts directory containing the following fields: + + { + access_token - this will be your personal access token as a string + from_project_id - this will be the project id you are copying from + to_project_id - this will be the project id that you are copying to + } \ No newline at end of file -- GitLab