Commit 05c81fc7 authored by Ryan Jones's avatar Ryan Jones
Browse files

updated files

parent 5c5745b2
[Unit]
Description=Create_cuda_devices
#After=
Before=slurmd.service
#Requires=
[Service]
Type=oneshot
ExecStart=/bin/sh -c "/bin/nvidia-smi -c 1"
[Install]
WantedBy=multi-user.target
#DESC: A clone of Red Hat Enterprise Linux 7
# The general RHEL include has all of the necessary functions, but requires
# some basic variables specific to each chroot type to be defined.
# with additional procedure to add packages from variable EXTEND_COMPUTE
. include-rhel-xcbc
# Define the location of the YUM repository
# YUM_MIRROR="http://mirror.centos.org/centos-7/7/os/\$basearch/"
YUM_MIRROR="http://mirror.centos.org/centos-7/7/os/\$basearch/"
# Include the basic packages
. base_packages
# Additional packages to get closer to the definition of compute node I had in rocks.
. extend_compute_packages
# include additional packages necessary for development environment
. gpu_compute_packages
ADDITIONALPACKAGES=( "$EXTEND_COMPUTE" "$GPU_COMPUTE" )
# vim:filetype=sh:syntax=sh:expandtab:ts=4:sw=4:
GPU_COMPUTE="ElectricFence GConf2-devel PyQt4-devel SDL-devel alsa-lib-devel babel binutils-devel boost-devel \
boost-math byacc bzip2-devel bzr chrpath cmake compat-glibc cscope \
ctags cups-devel cvs-inetd cyrus-sasl-devel db4-devel dbus-devel dbus-glib-devel dejagnu \
diffstat doxygen e2fsprogs-devel elfutils-devel elfutils-libelf-devel evolution-data-server-devel expat-devel freeglut-devel \
freeipmi fuse-devel gcc-c++ gcc-gnat gcc-objc gcc-objc++ gdbm-devel gdk-pixbuf2-devel \
glib-networking gmp-devel gnutls-devel gstreamer-devel gstreamer-plugins-base-devel gvfs-devel hmaccalc hunspell-devel \
imake indent intltool java-1.6.0-openjdk-devel java-1.7.0-openjdk-devel junit kernel-abi-whitelists ldapjdk \
libarchive libgdata libgdata-devel libXaw-devel libXinerama-devel libXmu-devel libXpm-devel libXrandr-devel \
libacl-devel libaio-devel libattr-devel libblkid-devel libbonobo-devel libcanberra-devel libcap-devel libcap-ng-devel \
libcgroup-devel libcurl-devel libdrm-devel libgcrypt-devel libglade2-devel libgnome-devel libgnomeui-devel libgudev1-devel \
libhugetlbfs-devel libnl-devel libnotify-devel librsvg2-devel libselinux-devel libsoup-devel libstdc++-docs libtiff-devel \
libudev-devel libusb-devel libuuid-devel libxml2-devel libxslt-devel mesa-private-llvm mercurial mod_dav_svn \
nasm ncurses-devel net-snmp-devel nspr-devel nss-devel nss-softokn-devel nss-softokn-freebl-devel nss-util-devel \
numactl-devel openmotif-devel papi-devel patchutils pcre-devel perl-Convert-ASN1 perl-Crypt-SSLeay perl-XML-Dumper \
perl-XML-Grove perl-XML-NamespaceSupport perl-XML-SAX perl-XML-Twig perl-libxml-perl perltidy polkit-devel popt-devel \
postgresql-devel pulseaudio-libs-devel pygtk2-devel python-docs python-ldap readline-devel sane-backends-devel sqlite-devel \
startup-notification-devel subversion systemtap systemtap-sdt-devel systemtap-server tbb-devel tcl-devel tcp_wrappers-devel \
texinfo tk-devel unique-devel unixODBC-devel xz-devel ORBit2-devel PyQt4 ant-antlr \
ant-apache-bcel ant-apache-bsf ant-apache-log4j ant-apache-oro ant-apache-regexp ant-apache-resolver ant-commons-logging ant-commons-net \
ant-javamail ant-jdepend ant-jsch ant-junit ant-swing antlr apr-util apr-util-ldap \
atk-devel avahi-glib avalon-framework batik bcel boost-date-time boost-filesystem boost-graph \
boost-iostreams boost-program-options boost-python boost-regex boost-serialization boost-signals boost-test boost-wave \
bsf check-devel compat-glibc-headers db4-cxx docbook-dtds docbook-style-dsssl docbook-style-xsl docbook-utils \
ecj enchant evolution-data-server evolution-data-server-doc fakeroot fakeroot-libs file-devel fop \
fuse-libs geronimo-specs gettext-devel gnome-icon-theme gnome-keyring gnome-keyring-devel gnome-vfs2 gnome-vfs2-devel \
gtk-doc gtk2-devel gtk2-engines gvfs hamcrest hsqldb httpd httpd-tools \
jakarta-commons-discovery jakarta-commons-httpclient jakarta-commons-io jakarta-commons-lang jakarta-commons-logging jakarta-commons-net jakarta-oro java-1.6.0-openjdk-javadoc \
java_cup jdepend jdom jna jsch jss junit4 jzlib \
keyutils-libs-devel krb5-devel libICE-devel libIDL-devel libSM-devel libXcomposite-devel libXcursor-devel libXdamage-devel \
libXext-devel libXfixes-devel libXft-devel libXi-devel libXp-devel libXres libXt-devel libXxf86vm-devel \
libart_lgpl libart_lgpl-devel libatasmart libbonobo libbonoboui libbonoboui-devel libcanberra libcanberra-gtk2 \
libcdio libcgroup libcom_err-devel libcroco-devel liberation-fonts-common liberation-sans-fonts libexif libexif-devel \
libgnat libgnat-devel libgnome libgnomecanvas libgnomecanvas-devel libgnomeui libgpg-error-devel libgphoto2 \
libgphoto2-devel libgsf libgsf-devel libgweather libgweather-devel libical libical-devel libidn-devel \
libieee1284 libieee1284-devel libnotify librsvg2 libsepol-devel libsoup libstdc++-devel libtasn1-devel \
libtdb libv4l libwnck lm_sensors-devel lockdev log4j mesa-libGL-devel mesa-libGLU-devel \
mozilla-filesystem neon notification-daemon objectweb-asm openjade opensp openssl-devel pakchois \
pango-devel perl-Compress-Zlib perl-Error perl-Git perl-IO-Compress-Base perl-IO-Compress-Zlib perl-SGMLSpm perl-XML-LibXML \
perl-XML-Parser polkit-docs postgresql postgresql-libs pulseaudio-libs-glib2 pycairo-devel pygobject2-codegen pygobject2-devel \
pygobject2-doc pygtk2-codegen pygtk2-doc python-babel python-enchant python-magic qdox qt-devel \
rarian rarian-compat regexp rpm-devel rpmdevtools rpmlint sac sane-backends \
sane-backends-libs sip sip-devel slf4j smp_utils sound-theme-freedesktop startup-notification subversion-javahl \
systemtap-client systemtap-devel systemtap-runtime unique unixODBC w3m ws-commons-util ws-jaxme \
wsdl4j xalan-j2 xcb-util xerces-j2 xml-commons-apis xml-commons-resolver xmlgraphics-commons xmlrpc3-client \
xmlrpc3-common xulrunner libvdpau libvdpau-devel dkms"
GRUB_CMDLINE_LINUX='quiet modprobe.blacklist=nouveau'
[Unit]
Description=rebuild-grub
#After=
#Requires=
[Service]
Type=oneshot
ExecStart=/bin/sh -c "if [[ -z $(grep nouveau /boot/grub2/grub.cfg) ]]; then /usr/sbin/grub2-mkconfig -o /boot/grub2/grub.cfg; systemctl disable rebuild-grub; fi"
[Install]
WantedBy=multi-user.target
---
- name: copy gpu_template.tmpl for wwmkchroot
copy: src=gpu-nodes.tmpl dest="{{ template_path }}{{ gpu_template }}.tmpl"
- name: copy gpu_compute_packages for wwmkchroot
copy: src=gpu_compute_packages dest="{{ template_path }}gpu_compute_packages"
- name: make chroot
command: wwmkchroot "{{ gpu_template }}" "{{ gpu_chroot_loc }}"
- name: copy resolve.conf into image
copy: src=/etc/resolv.conf dest="{{ gpu_chroot_loc }}/etc/resolv.conf" #"
- name: yum install into the image chroot
command: yum -y --installroot={{ gpu_chroot_loc }} install chrony kernel lmod-ohpc grub2 glibc-devel glibc-headers kernel-devel kernel-headers dkms gcc
#" for vim
- name: yum install slurm client into image
command: yum -y --installroot={{ gpu_chroot_loc }} groupinstall ohpc-slurm-client
- name: create export dir
file: path="{{ gpu_chroot_loc }}/export" state=directory
#" for vim
#
# mount: src={{ item.src }} path={{ item.path }} opts=bind state=mounted fstype=none
# commented ^ out because ansible adds these entries to fstab!!!
- name: mount necessary bits
command: mount -o rw,bind "{{ item.src }}" "{{ item.path }}"
with_items:
- { src: /proc/, path: "{{ gpu_chroot_loc }}/proc/" }
- { src: /dev/, path: "{{ gpu_chroot_loc }}/dev/" }
- { src: /export/, path: "{{ gpu_chroot_loc }}/export/" }
- { src: /var/run/utmp, path: "{{ gpu_chroot_loc }}/var/run/utmp" }
- name: get kernel version
shell: chroot "{{ gpu_chroot_loc }}" rpm -q --qf '%{version}-%{release}.%{arch}\n' kernel | tail -1
register: kernel_ver
- name: create build dir
file: path="{{ gpu_chroot_loc }}/root/nvidia_build/" state=directory
- name: get nvidia installation files
copy: src={{ item }} dest={{ gpu_chroot_loc }}/root/nvidia_build mode=0700
with_items:
- "{{ nvidia_driver_installer }}"
- "{{ cuda_toolkit_installer }}"
- "{{ cuda_samples_installer }}"
- name: build nvidia kernel modules in the vnfs
shell: chroot {{ gpu_chroot_loc }} {{ nvidia_kernel_build_cmd }}
vars:
nvidia_kernel_build_cmd: "/root/nvidia_build/{{ nvidia_driver_installer }} -k {{ kernel_ver.stdout }} --kernel-install-path=/lib/modules/{{ kernel_ver.stdout }}/kernel/drivers/video -z -s -X"
- name: build cuda toolkit
shell: chroot {{ gpu_chroot_loc }} {{ cuda_toolkit_build_cmd }}
vars:
cuda_toolkit_build_cmd: "/root/nvidia_build/{{ cuda_toolkit_installer }} --noprompt --prefix=/export/cuda"
- name: build cuda samples
shell: chroot {{ gpu_chroot_loc }} {{ cuda_samples_build_cmd }}
vars:
cuda_samples_build_cmd: "/root/nvidia_build/{{ cuda_samples_installer }} --noprompt --prefix=/export/cuda/samples --cudaprefix=/export/cuda"
# mount: path={{ item.path }} state=unmounted
- name: unmount necessary bits
command: umount -l {{ item.path }}
with_items:
- { src: /proc/, path: "{{ gpu_chroot_loc }}/proc/" }
- { src: /dev/, path: "{{ gpu_chroot_loc }}/dev/" }
- { src: /export/, path: "{{ gpu_chroot_loc }}/export/" }
- { src: /var/run/utmp, path: "{{ gpu_chroot_loc }}/var/run/utmp" }
- name: blacklist nouveau in modprobe
copy: src=blacklist_modprobe dest={{ gpu_chroot_loc }}/etc/modprobe.d/blacklist.conf
- name: blacklist nouveau in grub
copy: src=grub_default dest={{ gpu_chroot_loc }}/etc/default/grub
- name: create rebuild-grub service
copy: src=rebuild-grub.service dest={{ gpu_chroot_loc }}/etc/systemd/system/rebuild-grub.service
- name: enable rebuild-grub.service
command: chroot {{ gpu_chroot_loc }} systemctl enable rebuild-grub.service
- name: create service create-cuda-devices
copy: src=create-cuda-devices.service dest={{ gpu_chroot_loc }}/etc/systemd/system/create-cuda-devices.service
- name: enable create-cuda-devices
command: chroot {{ gpu_chroot_loc }} systemctl enable create-cuda-devices
- name: copy ssh keys over
copy: src=cluster_root.pub dest={{ gpu_chroot_loc }}/root/.ssh/authorized_keys
- name: put NFS home mount info in image
lineinfile: line="{{ headnode_private_ip }}:/home /home nfs nfsvers=3,rsize=1024,wsize=1024,cto 0 0" dest={{ gpu_chroot_loc }}/etc/fstab state=present
- name: put NFS opt mount info in image
lineinfile: line="{{ headnode_private_ip }}:/opt/ohpc/pub /opt/ohpc/pub-master nfs nfsvers=3 0 0" dest={{ gpu_chroot_loc }}/etc/fstab state=present
- name: put NFS opt mount info in image
lineinfile: line="{{ headnode_private_ip }}:/export /share nfs nfsvers=3 0 0" dest={{ gpu_chroot_loc }}/etc/fstab state=present
- name: chronyd on compute image enabled
command: chroot '{{ gpu_chroot_loc }}' systemctl enable chronyd
- name: add headnode to compute chrony.conf
lineinfile: line="server {{ headnode_private_ip }}" dest={{ gpu_chroot_loc }}/etc/chrony.conf state=present
- name: wwimport file (gres.conf)
command: wwsh file import /etc/slurm/gres.conf
- name: wwimport file (passwd)
command: wwsh file import /etc/passwd
- name: wwimport file (group)
command: wwsh file import /etc/group
- name: wwimport file (shadow)
command: wwsh file import /etc/shadow
- name: wwimport file (slurm)
command: wwsh file import /etc/slurm/slurm.conf
- name: wwimport file (munge)
command: wwsh file import /etc/munge/munge.key
- name: build bootstrap image
shell: wwbootstrap {{ build_kernel_ver }}
- name: build the vnfs
command: wwvnfs -y --chroot "{{ gpu_chroot_loc }}/"
- name: fix export of usr/include
lineinfile: line="hybridize=''" path=/etc/warewulf/vnfs/{{ gpu_chroot }}.conf state=present insertafter=EOF
- name: set up provisioning interface
lineinfile: line="GATEWAYDEV={{ private_interface }}" dest=/tmp/network.ww create=yes
#" for vim
#
- name: add network file to import
command: wwsh -y file import /tmp/network.ww --name network
- name: set network file path
command: wwsh -y file set network --path /etc/sysconfig/network --mode=0644 --uid=0
- name: add slurm.conf file to import
command: wwsh -y file import /etc/slurm/slurm.conf --name slurm.conf
---
# - name: print single node info
# debug:
# var: item.mac
# with_items: "{{ login_nodes }}"
- name: add node to ww db
command: wwsh -y node new {{ item.name }} --ipaddr={{ item.ip }} --hwaddr={{ item.mac }} -D {{ private_interface }}
with_items: "{{ login_nodes }}"
- name: set nodes bootloader
command: wwsh -y object modify -s bootloader=sda -t node {{ item.name }}
with_items: "{{ login_nodes }}"
- name: set nodes partitions
command: wwsh -y object modify -s diskpartition=sda -t node {{ item.name }}
with_items: "{{ login_nodes }}"
- name: format partitions
command: wwsh -y object modify -s diskformat=sda1,sda2,sda3 -t node {{ item.name }}
with_items: "{{ login_nodes }}"
- name: define filesystems
command: wwsh -y object modify -s filesystems="{{ sda1 }},{{ sda2 }},{{ sda3 }}" -t node {{ item.name }}
with_items: "{{ login_nodes }}"
#" for vim
- name: remove node from slurm.conf if it exists already # to avoid duplication!
lineinfile:
dest: /etc/slurm/slurm.conf
regexp: "^NodeName={{ item.name }}"
state: absent
with_items: "{{ login_nodes }}"
- name: add node to slurm.conf
lineinfile:
dest: /etc/slurm/slurm.conf
line: "NodeName={{ item.name }} Sockets={{ item.sockets }} CoresPerSocket={{ item.corespersocket }} State=UNKNOWN"
insertbefore: "^# PARTITIONS"
state: present
with_items: "{{ login_nodes }}"
- name: set files to provision
command: wwsh -y provision set {{ item.name }} --vnfs={{ item.vnfs }} --bootstrap={{ build_kernel_ver }} --files=passwd,group,shadow,munge.key,slurm.conf,dynamic_hosts,network,gres.conf
with_items: "{{ login_nodes }}"
- name: restart dhcp
service: name=dhcpd state=restarted
- name: update pxeconfig to let node boot from pxe
command: wwsh -y object modify -D bootlocal -t node {{ item.name }}
with_items: "{{ login_nodes }}"
- name: wwsh pxe update
command: wwsh -v pxe update
register: command_result
failed_when: "'Building Pxelinux' not in command_result.stdout"
#DESC: A clone of Red Hat Enterprise Linux 7
# The general RHEL include has all of the necessary functions, but requires
# some basic variables specific to each chroot type to be defined.
# with additional procedure to add packages from variable EXTEND_COMPUTE
. include-rhel-xcbc
# Define the location of the YUM repository
# YUM_MIRROR="http://mirror.centos.org/centos-7/7/os/\$basearch/"
YUM_MIRROR="http://mirror.centos.org/centos-7/7/os/\$basearch/"
# Include the basic packages
. base_packages
# Additional packages to get closer to the definition of compute node I had in rocks.
. extend_compute_packages
# include additional packages necessary for development environment
. login_packages
ADDITIONALPACKAGES=( "$EXTEND_COMPUTE" "$LOGIN_PACKAGES" )
# vim:filetype=sh:syntax=sh:expandtab:ts=4:sw=4:
LOGIN_PACKAGES="ElectricFence GConf2-devel PyQt4-devel SDL-devel alsa-lib-devel babel binutils-devel boost-devel \
boost-math byacc bzip2-devel bzr chrpath cmake compat-glibc cscope \
ctags cups-devel cvs-inetd cyrus-sasl-devel db4-devel dbus-devel dbus-glib-devel dejagnu \
diffstat doxygen e2fsprogs-devel elfutils-devel elfutils-libelf-devel evolution-data-server-devel expat-devel freeglut-devel \
freeipmi fuse-devel gcc-c++ gcc-gnat gcc-objc gcc-objc++ gdbm-devel gdk-pixbuf2-devel \
glib-networking gmp-devel gnutls-devel gstreamer-devel gstreamer-plugins-base-devel gvfs-devel hmaccalc hunspell-devel \
imake indent intltool java-1.6.0-openjdk-devel java-1.7.0-openjdk-devel junit kernel-abi-whitelists ldapjdk \
libarchive libgdata libgdata-devel libXaw-devel libXinerama-devel libXmu-devel libXpm-devel libXrandr-devel \
libacl-devel libaio-devel libattr-devel libblkid-devel libbonobo-devel libcanberra-devel libcap-devel libcap-ng-devel \
libcgroup-devel libcurl-devel libdrm-devel libgcrypt-devel libglade2-devel libgnome-devel libgnomeui-devel libgudev1-devel \
libhugetlbfs-devel libnl-devel libnotify-devel librsvg2-devel libselinux-devel libsoup-devel libstdc++-docs libtiff-devel \
libudev-devel libusb-devel libuuid-devel libxml2-devel libxslt-devel mesa-private-llvm mercurial mod_dav_svn \
nasm ncurses-devel net-snmp-devel nspr-devel nss-devel nss-softokn-devel nss-softokn-freebl-devel nss-util-devel \
numactl-devel openmotif-devel papi-devel patchutils pcre-devel perl-Convert-ASN1 perl-Crypt-SSLeay perl-XML-Dumper \
perl-XML-Grove perl-XML-NamespaceSupport perl-XML-SAX perl-XML-Twig perl-libxml-perl perltidy polkit-devel popt-devel \
postgresql-devel pulseaudio-libs-devel pygtk2-devel python-docs python-ldap readline-devel sane-backends-devel sqlite-devel \
startup-notification-devel subversion systemtap systemtap-sdt-devel systemtap-server tbb-devel tcl-devel tcp_wrappers-devel \
texinfo tk-devel unique-devel unixODBC-devel xz-devel ORBit2-devel PyQt4 ant-antlr \
ant-apache-bcel ant-apache-bsf ant-apache-log4j ant-apache-oro ant-apache-regexp ant-apache-resolver ant-commons-logging ant-commons-net \
ant-javamail ant-jdepend ant-jsch ant-junit ant-swing antlr apr-util apr-util-ldap \
atk-devel avahi-glib avalon-framework batik bcel boost-date-time boost-filesystem boost-graph \
boost-iostreams boost-program-options boost-python boost-regex boost-serialization boost-signals boost-test boost-wave \
bsf check-devel compat-glibc-headers db4-cxx docbook-dtds docbook-style-dsssl docbook-style-xsl docbook-utils \
ecj enchant evolution-data-server evolution-data-server-doc fakeroot fakeroot-libs file-devel fop \
fuse-libs geronimo-specs gettext-devel gnome-icon-theme gnome-keyring gnome-keyring-devel gnome-vfs2 gnome-vfs2-devel \
gtk-doc gtk2-devel gtk2-engines gvfs hamcrest hsqldb httpd httpd-tools \
jakarta-commons-discovery jakarta-commons-httpclient jakarta-commons-io jakarta-commons-lang jakarta-commons-logging jakarta-commons-net jakarta-oro java-1.6.0-openjdk-javadoc \
java_cup jdepend jdom jna jsch jss junit4 jzlib \
keyutils-libs-devel krb5-devel libICE-devel libIDL-devel libSM-devel libXcomposite-devel libXcursor-devel libXdamage-devel \
libXext-devel libXfixes-devel libXft-devel libXi-devel libXp-devel libXres libXt-devel libXxf86vm-devel \
libart_lgpl libart_lgpl-devel libatasmart libbonobo libbonoboui libbonoboui-devel libcanberra libcanberra-gtk2 \
libcdio libcgroup libcom_err-devel libcroco-devel liberation-fonts-common liberation-sans-fonts libexif libexif-devel \
libgnat libgnat-devel libgnome libgnomecanvas libgnomecanvas-devel libgnomeui libgpg-error-devel libgphoto2 \
libgphoto2-devel libgsf libgsf-devel libgweather libgweather-devel libical libical-devel libidn-devel \
libieee1284 libieee1284-devel libnotify librsvg2 libsepol-devel libsoup libstdc++-devel libtasn1-devel \
libtdb libv4l libwnck lm_sensors-devel lockdev log4j mesa-libGL-devel mesa-libGLU-devel \
mozilla-filesystem neon notification-daemon objectweb-asm openjade opensp openssl-devel pakchois \
pango-devel perl-Compress-Zlib perl-Error perl-Git perl-IO-Compress-Base perl-IO-Compress-Zlib perl-SGMLSpm perl-XML-LibXML \
perl-XML-Parser polkit-docs postgresql postgresql-libs pulseaudio-libs-glib2 pycairo-devel pygobject2-codegen pygobject2-devel \
pygobject2-doc pygtk2-codegen pygtk2-doc python-babel python-enchant python-magic qdox qt-devel \
rarian rarian-compat regexp rpm-devel rpmdevtools rpmlint sac sane-backends \
sane-backends-libs sip sip-devel slf4j smp_utils sound-theme-freedesktop startup-notification subversion-javahl \
systemtap-client systemtap-devel systemtap-runtime unique unixODBC w3m ws-commons-util ws-jaxme \
wsdl4j xalan-j2 xcb-util xerces-j2 xml-commons-apis xml-commons-resolver xmlgraphics-commons xmlrpc3-client \
xmlrpc3-common xulrunner libvdpau libvdpau-devel dkms"
---
- name: copy login_template.tmpl for wwmkchroot
copy: src=login-nodes.tmpl dest="{{ template_path }}{{ login_template }}.tmpl"
- name: copy login_packages for wwmkchroot
copy: src=login_packages dest="{{ template_path }}login_packages"
- name: make chroot
command: wwmkchroot "{{ login_template }}" "{{ login_chroot_loc }}"
- name: copy resolve.conf into image
copy: src=/etc/resolv.conf dest="{{ login_chroot_loc }}/etc/resolv.conf" #"
- name: yum install into the image chroot
command: yum -y --installroot={{ login_chroot_loc }} install chrony kernel lmod-ohpc grub2 glibc-devel glibc-headers kernel-devel kernel-headers dkms
#" for vim
- name: yum install slurm client into image
command: yum -y --installroot={{ login_chroot_loc }} groupinstall ohpc-slurm-client
#" for vim
#
##FIREWALL STUFF - DON'T LEAVE COMMENTED OUT
# - template: src=firewalld.conf dest={{ login_chroot_loc }}/etc/firewalld/firewalld.conf
#
# - template: src=zones/firewall_public.xml.j2 dest={{ login_chroot_loc }}/etc/firewalld/zones/public.xml
#
# - template: src=services/slurm.xml dest={{ login_chroot_loc }}/etc/firewalld/services/slurm.xml
#
# - template: src=zones/firewall_internal.xml.j2 dest={{ login_chroot_loc }}/etc/firewalld/zones/internal.xml
- name: copy ssh keys over
copy: src=cluster_root.pub dest={{ login_chroot_loc }}/root/.ssh/authorized_keys
- name: put NFS home mount info in image
lineinfile: line="{{ headnode_private_ip }}:/home /home nfs nfsvers=3,rsize=1024,wsize=1024,cto 0 0" dest={{ login_chroot_loc }}/etc/fstab state=present
- name: put NFS opt mount info in image
lineinfile: line="{{ headnode_private_ip }}:/opt/ohpc/pub /opt/ohpc/pub-master nfs nfsvers=3 0 0" dest={{ login_chroot_loc }}/etc/fstab state=present
- name: put NFS opt mount info in image
lineinfile: line="{{ headnode_private_ip }}:/export /share nfs nfsvers=3 0 0" dest={{ login_chroot_loc }}/etc/fstab state=present
- name: chronyd on login image enabled
command: chroot '{{ login_chroot_loc }}' systemctl enable chronyd
- name: add headnode to compute chrony.conf
lineinfile: line="server {{ headnode_private_ip }}" dest={{ login_chroot_loc }}/etc/chrony.conf state=present
- name: wwimport file (gres.conf)
command: wwsh file import /etc/slurm/gres.conf
- name: wwimport file (passwd)
command: wwsh file import /etc/passwd
- name: wwimport file (group)
command: wwsh file import /etc/group
- name: wwimport file (shadow)
command: wwsh file import /etc/shadow
- name: wwimport file (slurm)
command: wwsh file import /etc/slurm/slurm.conf
- name: wwimport file (munge)
command: wwsh file import /etc/munge/munge.key
- name: build bootstrap image
shell: wwbootstrap {{ build_kernel_ver }}
- name: fix export of usr/include
lineinfile: line="hybridize=''" dest=/etc/warewulf/vnfs/{{ login_chroot }}.conf state=present insertafter=EOF create=yes
- name: build the vnfs
command: wwvnfs -y --chroot "{{ login_chroot_loc }}/"
- name: set up provisioning interface
lineinfile: line="GATEWAYDEV={{ private_interface }}" dest=/tmp/network.ww create=yes
#" for vim
#
- name: add network file to import
command: wwsh -y file import /tmp/network.ww --name network
- name: set network file path
command: wwsh -y file set network --path /etc/sysconfig/network --mode=0644 --uid=0
- name: add slurm.conf file to import
command: wwsh -y file import /etc/slurm/slurm.conf --name slurm.conf
# firewalld config file
# default zone
# The default zone used if an empty zone string is used.
# Default: public
DefaultZone=public
# Minimal mark
# Marks up to this minimum are free for use for example in the direct
# interface. If more free marks are needed, increase the minimum
# Default: 100
MinimalMark=100
# Clean up on exit
# If set to no or false the firewall configuration will not get cleaned up
# on exit or stop of firewalld
# Default: yes
CleanupOnExit=yes
# Lockdown
# If set to enabled, firewall changes with the D-Bus interface will be limited
# to applications that are listed in the lockdown whitelist.
# The lockdown whitelist file is lockdown-whitelist.xml
# Default: no
Lockdown=no
# IPv6_rpfilter
# Performs a reverse path filter test on a packet for IPv6. If a reply to the
# packet would be sent via the same interface that the packet arrived on, the
# packet will match and be accepted, otherwise dropped.
# The rp_filter for IPv4 is controlled using sysctl.
# Default: yes
IPv6_rpfilter=yes
<?xml version="1.0" encoding="utf-8"?>
<service>
<short>Slurmctld</short>
<description> Slurmctld is the controller daemon for the Slurm
scheduler, which communicates with slurmd on controlled nodes.
</description>
<port protocol="tcp" port="6817"/>
<port protocol="udp" port="6817"/>
<port protocol="tcp" port="6818"/>
<port protocol="udp" port="6818"/>
<port protocol="tcp" port="7321"/>
<port protocol="udp" port="7321"/>
</service>
<?xml version="1.0" encoding="utf-8"?>
<zone>
<short>External</short>
<description>For use on external networks. You do not trust the other computers on networks to not harm your computer. Only selected incoming connections are accepted.</description>
<service name="ssh"/>
<masquerade/>
</zone>
<?xml version="1.0" encoding="utf-8"?>
<zone>
<short>Internal</short>
<description>For use on internal networks. You mostly trust the other computers on the networks to not harm your computer. Only selected incoming connections are accepted.</description>
<interface name="{{ login_private_interface }}"/>
<service name="mdns"/>
<service name="ipp-client"/>
<service name="dhcpv6-client"/>
<service name="ssh"/>
<service name="nfs"/>
<service name="mountd"/>
<service name="rpc-bind"/>
<service name="http"/>
<service name="https"/>
<service name="tftp"/>
<service name="samba-client"/>
</zone>
<?xml version="1.0" encoding="utf-8"?>
<zone>
<short>Public</short>
<description>For use in public areas. You do not trust the other computers on networks to not harm your computer. Only selected incoming connections are accepted.</description>
<interface name="{{ login_public_interface }}"/>
<service name="ssh"/>
</zone>
---
- name: Waiting for the compute node to bootup
pause:
seconds: 150
- name: tell nodes to get files
command: pdsh -w $(wwsh node list | tail --lines=+3 | cut -f 1 -d' '| tr '\n' ',') /warewulf/bin/wwgetfiles
- name: start munge
service: name=munge state=restarted enabled=yes
- name: start slurmctld
service: name=slurmctld state=restarted enabled=yes
- name: start munge on compute nodes
shell: pdsh -w $(wwsh node list | tail --lines=+3 | cut -f 1 -d' '| tr '\n' ',') systemctl start munge
- name: enable munge on compute nodes
shell: pdsh -w $(wwsh node list | tail --lines=+3 | cut -f 1 -d' '| tr '\n' ',') systemctl enable munge
- name: start slurmd on compute nodes
shell: pdsh -w $(wwsh node list | tail --lines=+3 | cut -f 1 -d' '| tr '\n' ',') systemctl start slurmd
- name: enable slurmd on compute nodes
shell: pdsh -w $(wwsh node list | tail --lines=+3 | cut -f 1 -d' '| tr '\n' ',') systemctl enable slurmd
- name: Waiting for slurmd to enable on the compute nodes
pause:
seconds: 30