ansible/roles/cuda_driver/tasks/main.yml

+---
+
+- name: Install prerequisites
+  ansible.builtin.yum:
+    name:
+      - "kernel-devel-{{ ansible_kernel }}"
+      - freeglut
+      - freeglut-devel
+    state: present
+
+- name: Install cuda-driver and dcgm packages
+  ansible.builtin.yum:
+    name:
+      - cuda-driver
+      - cuda-dcgm-nvvs.x86_64
+      - cuda-dcgm.x86_64
+      - cuda-dcgm-libs
+      - cuda-dcgm-devel
+    state: present
+
+- name: start cuda-driver service
+  ansible.builtin.systemd:
+    name: cuda-driver
+    state: started
+    enabled: yes
+  loop:
+    - cuda-driver
+    - nvidia-persistenced
+    - cuda-dcgm
+

ansible/roles/install_packages/tasks/main.yml

+---
+- name: Copy yum repo files into place
+  ansible.builtin.copy:
+    src: "{{ item }}"
+    dest: "/etc/yum.repos.d/{{ item }}"
+    owner: root
+    group: root
+    mode: 0644
+  loop: "{{ yum_repo_files }}"
+
+- name: Copy CM repo GPG key
+  ansible.builtin.copy:
+    src: RPM-GPG-KEY-cm
+    dest: /etc/pki/rpm-gpg/RPM-GPG-KEY-cm
+    owner: root
+    group: root
+    mode: 0644
+  when: "'cm.repo' in yum_repo_files"
+
+- name: Enable epel
+  ansible.builtin.yum:
+    state: present
+    name: epel-release
+
+- name: Install packages
+  ansible.builtin.yum:
+    state: present
+    name: "{{ pkg_list }}"

ansible/roles/install_zsh/tasks/main.yaml

+---
+- name: Install zsh dependencies
+  ansible.builtin.yum:
+    name:
+      - autoconf
+      - bzip2
+      - coreutils
+      - epel-release
+      - gawk
+      - gcc
+      - gdbm-devel
+      - git
+      - grep
+      - gzip
+      - hostname
+      - libcap-devel
+      - make
+      - ncurses-devel
+      - pcre-devel
+      - rpm-build
+      - sed
+      - tar
+      - texi2html
+      - texinfo
+      - xz
+
+- name: Copy zsh rpm to /tmp
+  ansible.builtin.copy:
+    src: zsh-5.9-2.el7.x86_64.rpm
+    dest: /tmp/zsh-5.9-2.el7.x86_64.rpm
+    mode: '0755'
+
+- name: Install zsh
+  ansible.builtin.yum:
+    name:  /tmp/zsh-5.9-2.el7.x86_64.rpm

ansible/roles/ldap_config/tasks/debian.yml

+---
+- name: Install ldap packages
+  ansible.builtin.apt:
+    name: "{{ item }}"
+    state: present
+  loop:
+    - ldap-utils
+    - libnss-ldap
+    - libpam-ldap
+    - nslcd
+    - nslcd-utils
+    - pynslcd
+    - slapd
+    - sssd-ldap
+
+- name: Copy ldap cert(s) into place
+  ansible.builtin.copy:
+    src: "{{ item }}"
+    dest: "/etc/ldap/{{ item }}"
+    owner: ldap
+    group: ldap
+    mode: 0440
+  loop:
+    - ca.pem
+    - ldap.key
+    - ldap.pem
+
+- name: Copy ldap config into place
+  ansible.builtin.copy:
+    src: nslcd.conf
+    dest: /etc/nslcd.conf
+    owner: root
+    group: root
+    mode: 0600
+
+- name: Enable services
+  ansible.builtin.service:
+    name: "{{ item }}"
+    enabled: yes
+  loop:
+    - nslcd

ansible/roles/ldap_config/tasks/main.yml

+---
+- name: Include os specific tasks
+  ansible.builtin.include_tasks:
+    file: "{{ ansible_os_family|lower }}.yml"
+
+- name: Update nsswitch.conf to look for ldap
+  ansible.builtin.replace:
+    dest: /etc/nsswitch.conf
+    regexp: '^({{ item }}:(?!.*\bldap\b).*)$'
+    replace: '\1 ldap'
+  loop:
+    - passwd
+    - shadow
+    - group
+    - netgroup
+    - automount

ansible/roles/ldap_config/tasks/redhat.yml

+---
+- name: Install ldap packages
+  ansible.builtin.yum:
+    name: "{{ item }}"
+    state: present
+  loop:
+    - nss-pam-ldapd
+    - openldap
+    - openldap-clients
+    - openldap-servers
+    - sssd-ldap
+
+- name: Copy ldap cert(s) into place
+  ansible.builtin.copy:
+    src: "{{ item.src }}"
+    dest: "/cm/local/apps/openldap/etc/certs/{{ item.src }}"
+    owner: ldap
+    group: ldap
+    mode: 0440
+  loop:
+    - { src: ca.pem }
+    - { src: ldap.key }
+    - { src: ldap.pem }
+
+- name: Copy ldap config into place
+  ansible.builtin.copy:
+    src: nslcd.conf
+    dest: /etc/nslcd.conf
+    owner: root
+    group: root
+    mode: 0600
+
+- name: Enable services
+  ansible.builtin.service:
+    name: "{{ item }}"
+    enabled: yes
+  loop:
+    - nslcd

ansible/roles/lmod_user/tasks/main.yaml

+---
+- name: Setup rsyslog
+  template:
+    src: rsyslog.j2
+    dest: /etc/rsyslog.d/moduleTracking-client.conf
+
+- name: Restart rsyslog
+  service:
+    name: rsyslog
+    state: restarted

ansible/roles/lmod_user/templates/rsyslog.j2

+if $programname contains 'ModuleUsageTracking' then @{{ lmod_db_host_machine }}
+&~

ansible/roles/nfs_mounts/tasks/main.yml

+---
+- name: Create base directories
+  ansible.builtin.file:
+    path: "{{ item.dir }}"
+    state: directory
+    mode: "{{ item.mode }}"
+  loop:
+    - { dir: /local, mode: '0777' }
+    - { dir: /scratch, mode: '0755' }
+    - { dir: /share, mode: '0755' }
+    - { dir: /data/rc/apps, mode: '0755' } # this is only required for the symlink to be happy
+    - { dir: /data/user, mode: '0755' }
+    - { dir: /data/project, mode: '0755' }
+- name: Set up NFS GPFS mount point(s)
+  ansible.posix.mount:
+    path: "{{ item.path }}"
+    src: "{{ item.src }}"
+    fstype: "{{ item.fstype }}"
+    opts: "{{ item.opts }}"
+    state: present
+  loop:
+    - { path: /cm/shared, src: "gpfs.rc.uab.edu:/data/cm/shared-8.2", fstype: nfs, opts: "vers=3,_netdev,defaults" }
+    - { path: /data/project, src: "gpfs.rc.uab.edu:/data/project", fstype: nfs, opts: "vers=3,_netdev,defaults" }
+    - { path: /data/user, src: "gpfs.rc.uab.edu:/data/user", fstype: nfs, opts: "vers=3,_netdev,local_lock=posix,defaults" }
+    - { path: /data/rc/apps, src: "gpfs.rc.uab.edu:/data/rc/apps", fstype: nfs, opts: "vers=3,_netdev,defaults" }
+    - { path: /scratch, src: "gpfs.rc.uab.edu:/scratch", fstype: nfs, opts: "vers=3,_netdev,local_lock=posix,defaults" }
+    - { path: /home, src: "/data/user/home", fstype: none, opts: bind }
+- name: Create symbolic links
+  ansible.builtin.file:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    owner: root
+    group: root
+    force: yes
+    state: link
+  loop:
+    - { src: /data/rc/apps, dest: /share/apps }

ansible/roles/pam_slurm_adopt/tasks/main.yml

+---
+# restrict user access to nodes
+- name: Get version and release for slurm-pam pkg
+  shell: rpm -q --qf "%{VERSION}-%{RELEASE}\n" slurm-client
+  register: ver
+
+- debug:
+    var: ver
+
+- name: Install slurm-pam package
+  yum:
+    name: "slurm-pam-{{ ver.stdout }}"
+    state: present
+
+- name: Create new pam file wihtout pam_systemd.so
+  shell: grep -v pam_systemd.so < /etc/pam.d/password-auth > /etc/pam.d/password-auth-no-systemd
+
+- name: Update sshd pam file
+  blockinfile:
+    path: /etc/pam.d/sshd
+    insertbefore: "^account.+password-auth$"
+    backup: yes
+    marker: "# {mark} PAM config for Slurm"
+    marker_begin: "BEGIN account"
+    marker_end: "END account"
+    block: |
+      -account    sufficient     pam_slurm_adopt.so
+      account    required   pam_access.so
+
+- name: Update sshd pam file
+  blockinfile:
+    path: /etc/pam.d/sshd
+    insertbefore: "^session.+postlogin$"
+    backup: yes
+    marker: "# {mark} - PAM config for Slurm"
+    marker_begin: "BEGIN session"
+    marker_end: "END session"
+    block: |
+      session    include      password-auth-no-systemd
+
+- name: Update secure access config for the node
+  lineinfile:
+    dest: /etc/security/access.conf
+    line: "-:ALL EXCEPT root mhanby_ cmcclung_ (sysadmins) (atlab):ALL"
+    owner: root
+    mode: 0644

ansible/roles/slurm_client/tasks/main.yml

+---
+- name: Install slurm packages
+  ansible.builtin.yum:
+    name: "{{ item }}"
+    state: present
+  loop:
+    - munge
+    - "slurm-client-{{ slurm_version }}"
+
+- name: Create slurm group
+  ansible.builtin.group:
+    name: slurm
+    state: present
+    gid: 450
+
+- name: Create slurm user
+  ansible.builtin.user:
+    name: slurm
+    state: present
+    uid: 450
+    group: slurm
+
+- name: Copy munge key
+  ansible.builtin.copy:
+    src: munge.key
+    dest: /etc/munge/munge.key
+    owner: daemon
+    group: root
+    mode: 0400
+
+- name: Enable services
+  ansible.builtin.service:
+    name: "{{ item }}"
+    enabled: yes
+  loop:
+    - munge
+    - slurmd

openstack-compute/README.md

+This contains packer hcl files for creating images. For documentation on packer, see [here](https://www.packer.io/docs); for information about the openstack-specific builder, see [here](https://www.packer.io/plugins/builders/openstack)

openstack-compute/nodeimage.pkr.hcl

+locals {
+    local_image_name = "${var.image_name}${var.image_date_suffix ? formatdate("YYYYMMDDHHmm", timestamp()): ""}"
+}
+
+source "openstack" "image" {
+  skip_create_image = var.skip_create_image
+  image_name        = local.local_image_name
+  source_image_name = var.source_image
+  image_members     = var.image_membership
+  image_auto_accept_members = var.auto_accept_members
+  image_tags        = var.image_tags
+  flavor            = var.flavor
+  instance_name     = var.build_instance_name
+
+  floating_ip_network = var.floating_ip_network
+  networks = var.networks
+  security_groups = var.security_groups
+
+  ssh_username = var.ssh_username
+}
+
+build {
+  sources = ["source.openstack.image"]
+
+  provisioner "ansible" {
+    groups = ["compute"]
+    playbook_file = "./ansible/compute.yml"
+    roles_path = "./ansible/roles"
+    extra_arguments = [
+      "--extra-vars", "root_ssh_key='${var.root_ssh_key}'"
+    ]
+  }
+  provisioner "ansible" {
+    use_proxy =  false
+    ansible_env_vars = ["ANSIBLE_HOST_KEY_CHECKING=False"]
+    playbook_file = "../CRI_XCBC/ood-packer.yaml"
+    extra_arguments = [
+      "-t enable_lmod",
+      "-t lmod_user",
+    ]
+  }
+}

openstack-compute/variables.pkr.hcl

+variable "root_ssh_key" {
+  type = string
+  description = "The root key to use for ssh"
+}
+
+variable "image_name" {
+  type        = string
+  default     = "cluster-image"
+  description = "Name of the image in openstack"
+}
+
+variable "image_date_suffix" {
+  type        = bool
+  default     = false
+  description = "Append a date to the image name (in YYYYMMDDHHMMSS format)"
+}
+
+variable "image_tags" {
+  type        = list(string)
+  default     = []
+  description = "List of tags to be associated to the resulting image"
+}
+
+variable "image_membership" {
+  type        = list(string)
+  default     = []
+  description = "Projects/tenants to share the image in openstack with"
+}
+
+variable "auto_accept_members" {
+  type        = bool
+  default     = false
+  description = "A boolean value for auto accepting image in the projects/tenants defined in image_membership."
+}
+
+variable "skip_create_image" {
+  type = bool
+  default = false
+  description = "A boolean value for skipping image creation at the end of the build"
+}
+
+variable "source_image" {
+  type        = string
+  description = "The name of the source image to use"
+}
+
+variable "flavor" {
+  type        = string
+  description = "The name of the flavor to use"
+}
+
+variable "floating_ip_network" {
+  type = string
+  description = "floating ip network to use with (temporary) ip assignmnet to a vm"
+}
+
+variable "networks" {
+  type = list(string)
+  description = "List of network UUIDs to assign to the network"
+}
+
+variable "security_groups" {
+  type = list(string)
+  default = []
+  description = "A list of security groups to add - you should make sure ssh access is open to the machine"
+}
+
+variable "build_instance_name" {
+  type = string
+  default = "compute"
+  description = "A name of build instance used for image build"
+}
+
+variable "ssh_username" {
+  type = string
+  default = "centos"
+  description = "The default username to use for SSH"
+}

openstack-gpu/nodeimage.pkr.hcl

@@ -19,7 +19,8 @@ build {
   sources = ["source.openstack.image"]
 
   provisioner "ansible" {
-    playbook_file = "./ansible/node-gpu.yml"
+    groups = ["gpu"]
+    playbook_file = "./ansible/gpu.yml"
     roles_path = "./ansible/roles"
     extra_arguments = [
       "--extra-vars", "root_ssh_key='${var.root_ssh_key}'"

openstack-ood/README.md

+This contains packer hcl files for creating images. For documentation on packer, see [here](https://www.packer.io/docs); for information about the openstack-specific builder, see [here](https://www.packer.io/plugins/builders/openstack)

openstack-ood/nodeimage.pkr.hcl

+locals {
+    local_image_name = "${var.image_name}${var.image_date_suffix ? formatdate("-YYYYMMDDHHmm", timestamp()): ""}"
+}
+
+source "openstack" "image" {
+  skip_create_image = var.skip_create_image
+  image_name        = local.local_image_name
+  source_image_name = var.source_image
+  image_members     = var.image_membership
+  image_auto_accept_members = var.auto_accept_members
+  image_tags        = var.image_tags
+  image_disk_format = var.image_format
+  volume_size       = 10
+  flavor            = var.flavor
+  instance_name     = var.build_instance_name
+
+  use_blockstorage_volume = true
+  floating_ip_network = var.floating_ip_network
+  networks = var.networks
+  security_groups = var.security_groups
+
+  ssh_username = var.ssh_username
+}
+
+build {
+  sources = ["source.openstack.image"]
+
+  provisioner "ansible" {
+    user = var.ssh_username
+    groups = ["ood"]
+    playbook_file = "./ansible/ood.yml"
+    roles_path = "./ansible/roles"
+    extra_arguments = [
+      "--extra-vars", "root_ssh_key='${var.root_ssh_key}'"
+    ]
+  }
+  provisioner "ansible" {
+    user = var.ssh_username
+    groups = ["ood", "knightly"]
+    ansible_env_vars = ["ANSIBLE_HOST_KEY_CHECKING=False"]
+    playbook_file = "./CRI_XCBC/ood-packer.yaml"
+  }
+}

openstack-ood/variables.pkr.hcl

+variable "root_ssh_key" {
+  type = string
+  description = "The root key to use for ssh"
+}
+
+variable "image_name" {
+  type        = string
+  default     = "cluster-image"
+  description = "Name of the image in openstack"
+}
+
+variable "image_format" {
+  type        = string
+  default     = "qcow2"
+  description = "The format of the resulting image"
+}
+
+variable "image_date_suffix" {
+  type        = bool
+  default     = false
+  description = "Append a date to the image name (in YYYYMMDDHHMMSS format)"
+}
+
+variable "image_tags" {
+  type        = list(string)
+  default     = []
+  description = "List of tags to be associated to the resulting image"
+}
+
+variable "image_membership" {
+  type        = list(string)
+  default     = []
+  description = "Projects/tenants to share the image in openstack with"
+}
+
+variable "auto_accept_members" {
+  type        = bool
+  default     = false
+  description = "A boolean value for auto accepting image in the projects/tenants defined in image_membership."
+}
+
+variable "skip_create_image" {
+  type = bool
+  default = false
+  description = "A boolean value for skipping image creation at the end of the build"
+}
+
+variable "source_image" {
+  type        = string
+  description = "The name of the source image to use"
+}
+
+variable "flavor" {
+  type        = string
+  description = "The name of the flavor to use"
+}
+
+variable "floating_ip_network" {
+  type = string
+  description = "floating ip network to use with (temporary) ip assignmnet to a vm"
+}
+
+variable "networks" {
+  type = list(string)
+  description = "List of network UUIDs to assign to the network"
+}
+
+variable "security_groups" {
+  type = list(string)
+  default = []
+  description = "A list of security groups to add - you should make sure ssh access is open to the machine"
+}
+
+variable "build_instance_name" {
+  type = string
+  default = "ood"
+  description = "A name of build instance used for image build"
+}
+
+variable "ssh_username" {
+  type = string
+  default = "centos"
+  description = "The default username to use for SSH"
+}

openstack.pkrvars.hcl.example

@@ -12,3 +12,13 @@ security_groups = ["allow ssh"]
 image_date_suffix = false
 # Set to a public key that will be added as a root ssh key
 root_ssh_key = "ecdsa-sha2-nistp256 AAAAyourkeyhere comment string here"
+# Create image at the end of build. Set to true during a build test stage
+skip_create_image = false
+# Project/tenants to share the image with
+image_membership = []
+# Auto accept image on behalf of members. Need privileges both in build project and members provided
+auto_accept_members = false
+# Tags for the resulting image.
+image_tags = []
+# The username to connect to SSH with. Used by Packer build section
+ssh_username = "centos"

openstack/nodeimage.pkr.hcl

 locals {
-    local_image_name = "${var.image_name}${var.image_date_suffix ? formatdate("YYYYMMDDHHmm", timestamp()): ""}"
+    local_image_name = "${var.image_name}${var.image_date_suffix ? formatdate("-YYYYMMDDHHmm", timestamp()): ""}"
 }
 
 source "openstack" "image" {
+  skip_create_image = var.skip_create_image
   image_name        = local.local_image_name
   source_image_name = var.source_image
+  image_members     = var.image_membership
+  image_auto_accept_members = var.auto_accept_members
+  image_tags        = var.image_tags
   flavor            = var.flavor
+  instance_name     = var.build_instance_name
 
   floating_ip_network = var.floating_ip_network
   networks = var.networks
@@ -18,7 +23,9 @@ build {
   sources = ["source.openstack.image"]
 
   provisioner "ansible" {
-    playbook_file = "./ansible/node.yml"
+    user = var.ssh_username
+    groups = ["base"]
+    playbook_file = "./ansible/base.yml"
     roles_path = "./ansible/roles"
     extra_arguments = [
       "--extra-vars", "root_ssh_key='${var.root_ssh_key}'"