mirror of
https://github.com/ClusterCockpit/cc-docker.git
synced 2025-07-23 21:31:42 +02:00
Start converting to Rocky Linux base image
This commit is contained in:
@@ -1,31 +1,35 @@
|
||||
FROM scidas/slurm.base:19.05.1
|
||||
MAINTAINER Michael J. Stealey <stealey@renci.org>
|
||||
FROM rockylinux:8
|
||||
MAINTAINER Jan Eitzinger <jan.eitzinger@fau.de>
|
||||
|
||||
# install openmpi 3.0.1
|
||||
RUN yum -y install \
|
||||
gcc-c++ \
|
||||
gcc-gfortran \
|
||||
&& yum -y localinstall \
|
||||
/packages/openmpi-*.rpm
|
||||
ENV SLURM_VERSION=19.05.1 \
|
||||
MUNGE_UID=981 \
|
||||
SLURM_UID=982 \
|
||||
WORKER_UID=1000
|
||||
|
||||
# install Lmod 7.7
|
||||
RUN yum -y install \
|
||||
lua-posix \
|
||||
lua \
|
||||
lua-filesystem \
|
||||
lua-devel \
|
||||
wget \
|
||||
bzip2 \
|
||||
expectk \
|
||||
make \
|
||||
&& wget https://sourceforge.net/projects/lmod/files/Lmod-7.7.tar.bz2 \
|
||||
&& tar -xjvf Lmod-7.7.tar.bz2
|
||||
WORKDIR /Lmod-7.7
|
||||
RUN ./configure --prefix=/opt/apps \
|
||||
&& make install \
|
||||
&& ln -s /opt/apps/lmod/lmod/init/profile /etc/profile.d/z00_lmod.sh \
|
||||
&& ln -s /opt/apps/lmod/lmod/init/cshrc /etc/profile.d/z00_lmod.csh
|
||||
WORKDIR /
|
||||
RUN yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm -y
|
||||
|
||||
RUN groupadd -g $MUNGE_UID munge \
|
||||
&& useradd -m -c "MUNGE Uid 'N' Gid Emporium" -d /var/lib/munge -u $MUNGE_UID -g munge -s /sbin/nologin munge \
|
||||
&& groupadd -g $SLURM_UID slurm \
|
||||
&& useradd -m -c "Slurm workload manager" -d /var/lib/slurm -u $SLURM_UID -g slurm -s /bin/bash slurm \
|
||||
&& groupadd -g $WORKER_UID worker \
|
||||
&& useradd -m -c "Workflow user" -d /home/worker -u $WORKER_UID -g worker -s /bin/bash worker
|
||||
|
||||
RUN yum install -y munge munge-libs
|
||||
RUN dnf --enablerepo=powertools install munge-devel -y
|
||||
RUN yum install rng-tools -y
|
||||
|
||||
RUN yum install -y python3 gcc openssl openssl-devel \
|
||||
pam-devel numactl numactl-devel hwloc sudo \
|
||||
lua readline-devel ncurses-devel man2html \
|
||||
libibmad libibumad rpm-build perl-ExtUtils-MakeMaker.noarch rpm-build make wget
|
||||
|
||||
RUN dnf --enablerepo=powertools install rrdtool-devel lua-devel hwloc-devel rpm-build -y
|
||||
RUN dnf install mariadb-server mariadb-devel -y
|
||||
RUN mkdir /usr/local/slurm-tmp
|
||||
RUN cd /usr/local/slurm-tmp
|
||||
RUN wget https://download.schedmd.com/slurm/slurm-22.05.6.tar.bz2
|
||||
RUN rpmbuild -ta slurm-22.05.6.tar.bz2
|
||||
|
||||
ENV USE_SLURMDBD=true \
|
||||
CLUSTER_NAME=snowflake \
|
||||
@@ -36,12 +40,12 @@ ENV USE_SLURMDBD=true \
|
||||
ACCOUNTING_STORAGE_PORT=6819 \
|
||||
PARTITION_NAME=docker
|
||||
|
||||
# clean up
|
||||
RUN rm -f /packages/slurm-*.rpm /packages/openmpi-*.rpm \
|
||||
&& yum clean all \
|
||||
&& rm -rf /var/cache/yum \
|
||||
&& rm -f /Lmod-7.7.tar.bz2
|
||||
|
||||
COPY docker-entrypoint.sh /docker-entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/tini", "--", "/docker-entrypoint.sh"]
|
||||
VOLUME ["/home", "/.secret"]
|
||||
# 22: SSH
|
||||
# 3306: MariaDB
|
||||
# 6817: SlurmCtlD
|
||||
# 6818: SlurmD
|
||||
# 6819: SlurmDBD
|
||||
EXPOSE 22 3306 6817 6818 6819
|
||||
ENTRYPOINT ["/docker-entrypoint.sh"]
|
||||
|
||||
@@ -12,44 +12,49 @@ _sshd_host() {
|
||||
|
||||
# setup worker ssh to be passwordless
|
||||
_ssh_worker() {
|
||||
if [[ ! -d /home/worker ]]; then
|
||||
mkdir -p /home/worker
|
||||
chown -R worker:worker /home/worker
|
||||
fi
|
||||
cat > /home/worker/setup-worker-ssh.sh <<'EOF2'
|
||||
mkdir -p ~/.ssh
|
||||
chmod 0700 ~/.ssh
|
||||
ssh-keygen -b 2048 -t rsa -f ~/.ssh/id_rsa -q -N "" -C "$(whoami)@$(hostname)-$(date -I)"
|
||||
cat ~/.ssh/id_rsa.pub > ~/.ssh/authorized_keys
|
||||
chmod 0640 ~/.ssh/authorized_keys
|
||||
cat >> ~/.ssh/config <<EOF
|
||||
if [[ ! -d /home/worker ]]; then
|
||||
mkdir -p /home/worker
|
||||
chown -R worker:worker /home/worker
|
||||
fi
|
||||
cat > /home/worker/setup-worker-ssh.sh <<EOF2
|
||||
mkdir -p ~/.ssh
|
||||
chmod 0700 ~/.ssh
|
||||
ssh-keygen -b 2048 -t rsa -f ~/.ssh/id_rsa -q -N "" -C "$(whoami)@$(hostname)-$(date -I)"
|
||||
cat ~/.ssh/id_rsa.pub > ~/.ssh/authorized_keys
|
||||
chmod 0640 ~/.ssh/authorized_keys
|
||||
cat >> ~/.ssh/config <<EOF
|
||||
Host *
|
||||
StrictHostKeyChecking no
|
||||
UserKnownHostsFile /dev/null
|
||||
LogLevel QUIET
|
||||
EOF
|
||||
chmod 0644 ~/.ssh/config
|
||||
cd ~/
|
||||
tar -czvf ~/worker-secret.tar.gz .ssh
|
||||
cd -
|
||||
StrictHostKeyChecking no
|
||||
UserKnownHostsFile /dev/null
|
||||
LogLevel QUIET
|
||||
EOF2
|
||||
chmod +x /home/worker/setup-worker-ssh.sh
|
||||
chown worker: /home/worker/setup-worker-ssh.sh
|
||||
sudo -u worker /home/worker/setup-worker-ssh.sh
|
||||
chmod 0644 ~/.ssh/config
|
||||
cd ~/
|
||||
tar -czvf ~/worker-secret.tar.gz .ssh
|
||||
cd -
|
||||
EOF2
|
||||
chmod +x /home/worker/setup-worker-ssh.sh
|
||||
chown worker: /home/worker/setup-worker-ssh.sh
|
||||
sudo -u worker /home/worker/setup-worker-ssh.sh
|
||||
}
|
||||
|
||||
# start munge and generate key
|
||||
_munge_start() {
|
||||
chown -R munge: /etc/munge /var/lib/munge /var/log/munge /var/run/munge
|
||||
chmod 0700 /etc/munge
|
||||
chmod 0711 /var/lib/munge
|
||||
chmod 0700 /var/log/munge
|
||||
chmod 0755 /var/run/munge
|
||||
/sbin/create-munge-key -f
|
||||
sudo -u munge /sbin/munged
|
||||
munge -n
|
||||
munge -n | unmunge
|
||||
remunge
|
||||
chown -R munge: /etc/munge /var/lib/munge /var/log/munge /var/run/munge
|
||||
chmod 0700 /etc/munge
|
||||
chmod 0711 /var/lib/munge
|
||||
chmod 0700 /var/log/munge
|
||||
chmod 0755 /var/run/munge
|
||||
/sbin/create-munge-key -f
|
||||
rngd -r /dev/urandom
|
||||
/usr/sbin/create-munge-key -r -f
|
||||
sh -c "dd if=/dev/urandom bs=1 count=1024 > /etc/munge/munge.key"
|
||||
chown munge: /etc/munge/munge.key
|
||||
chmod 400 /etc/munge/munge.key
|
||||
sudo -u munge /sbin/munged
|
||||
munge -n
|
||||
munge -n | unmunge
|
||||
remunge
|
||||
}
|
||||
|
||||
# copy secrets to /.secret directory for other nodes
|
||||
@@ -76,15 +81,10 @@ _generate_slurm_conf() {
|
||||
#
|
||||
ClusterName=$CLUSTER_NAME
|
||||
SlurmctldHost=$CONTROL_MACHINE
|
||||
#SlurmctldHostr=
|
||||
#
|
||||
SlurmUser=slurm
|
||||
#SlurmdUser=root
|
||||
SlurmctldPort=$SLURMCTLD_PORT
|
||||
SlurmdPort=$SLURMD_PORT
|
||||
AuthType=auth/munge
|
||||
#JobCredentialPrivateKey=
|
||||
#JobCredentialPublicCertificate=
|
||||
StateSaveLocation=/var/spool/slurm/ctld
|
||||
SlurmdSpoolDir=/var/spool/slurm/d
|
||||
SwitchType=switch/none
|
||||
@@ -92,25 +92,7 @@ MpiDefault=none
|
||||
SlurmctldPidFile=/var/run/slurmctld.pid
|
||||
SlurmdPidFile=/var/run/slurmd.pid
|
||||
ProctrackType=proctrack/pgid
|
||||
#PluginDir=
|
||||
#FirstJobId=
|
||||
ReturnToService=0
|
||||
#MaxJobCount=
|
||||
#PlugStackConfig=
|
||||
#PropagatePrioProcess=
|
||||
#PropagateResourceLimits=
|
||||
#PropagateResourceLimitsExcept=
|
||||
#Prolog=
|
||||
#Epilog=
|
||||
#SrunProlog=
|
||||
#SrunEpilog=
|
||||
#TaskProlog=
|
||||
#TaskEpilog=
|
||||
#TaskPlugin=
|
||||
#TrackWCKey=no
|
||||
#TreeWidth=50
|
||||
#TmpFS=
|
||||
#UsePAM=
|
||||
#
|
||||
# TIMERS
|
||||
SlurmctldTimeout=300
|
||||
@@ -122,17 +104,7 @@ Waittime=0
|
||||
#
|
||||
# SCHEDULING
|
||||
SchedulerType=sched/backfill
|
||||
#SchedulerAuth=
|
||||
#SelectType=select/linear
|
||||
FastSchedule=1
|
||||
#PriorityType=priority/multifactor
|
||||
#PriorityDecayHalfLife=14-0
|
||||
#PriorityUsageResetPeriod=14-0
|
||||
#PriorityWeightFairshare=100000
|
||||
#PriorityWeightAge=1000
|
||||
#PriorityWeightPartition=10000
|
||||
#PriorityWeightJobSize=1000
|
||||
#PriorityMaxAge=1-0
|
||||
#
|
||||
# LOGGING
|
||||
SlurmctldDebug=3
|
||||
@@ -161,33 +133,31 @@ EOF
|
||||
|
||||
# run slurmctld
|
||||
_slurmctld() {
|
||||
if $USE_SLURMDBD; then
|
||||
echo -n "cheking for slurmdbd.conf"
|
||||
while [ ! -f /.secret/slurmdbd.conf ]; do
|
||||
echo -n "."
|
||||
sleep 1
|
||||
done
|
||||
echo ""
|
||||
fi
|
||||
mkdir -p /var/spool/slurm/ctld \
|
||||
/var/spool/slurm/d \
|
||||
/var/log/slurm
|
||||
chown -R slurm: /var/spool/slurm/ctld \
|
||||
/var/spool/slurm/d \
|
||||
/var/log/slurm
|
||||
touch /var/log/slurmctld.log
|
||||
chown slurm: /var/log/slurmctld.log
|
||||
if [[ ! -f /home/config/slurm.conf ]]; then
|
||||
echo "### generate slurm.conf ###"
|
||||
_generate_slurm_conf
|
||||
else
|
||||
echo "### use provided slurm.conf ###"
|
||||
cp /home/config/slurm.conf /etc/slurm/slurm.conf
|
||||
fi
|
||||
sacctmgr -i add cluster "${CLUSTER_NAME}"
|
||||
sleep 2s
|
||||
/usr/sbin/slurmctld
|
||||
cp -f /etc/slurm/slurm.conf /.secret/
|
||||
cd /root/rpmbuild/RPMS/aarch64
|
||||
yum -y --nogpgcheck localinstall slurm-22.05.6-1.el8.aarch64.rpm slurm-perlapi-22.05.6-1.el8.aarch64.rpm slurm-slurmctld-22.05.6-1.el8.aarch64.rpm
|
||||
if $USE_SLURMDBD; then
|
||||
echo -n "checking for slurmdbd.conf"
|
||||
while [ ! -f /.secret/slurmdbd.conf ]; do
|
||||
echo -n "."
|
||||
sleep 1
|
||||
done
|
||||
echo ""
|
||||
fi
|
||||
mkdir -p /var/spool/slurm/ctld /var/spool/slurmd /var/log/slurm /etc/slurm
|
||||
chown -R slurm: /var/spool/slurm/ctld /var/spool/slurmd /var/log/slurm
|
||||
touch /var/log/slurmctld.log
|
||||
chown slurm: /var/log/slurmctld.log
|
||||
if [[ ! -f /home/config/slurm.conf ]]; then
|
||||
echo "### generate slurm.conf ###"
|
||||
_generate_slurm_conf
|
||||
else
|
||||
echo "### use provided slurm.conf ###"
|
||||
cp /home/config/slurm.conf /etc/slurm/slurm.conf
|
||||
fi
|
||||
sacctmgr -i add cluster "${CLUSTER_NAME}"
|
||||
sleep 2s
|
||||
/usr/sbin/slurmctld
|
||||
cp -f /etc/slurm/slurm.conf /.secret/
|
||||
}
|
||||
|
||||
### main ###
|
||||
|
||||
Reference in New Issue
Block a user