Unix port based slurmrest service+

2025-07-23 05:11:41 +02:00 · 2024-10-24 14:54:40 +02:00
parent f574568d76
commit 0c1f6b446e
11 changed files with 219 additions and 41 deletions
--- a/slurm/base/Dockerfile
+++ b/slurm/base/Dockerfile
@@ -19,7 +19,7 @@ RUN yum install -y munge munge-libs rng-tools \
    openssh-server openssh-clients dbus-devel \
    pam-devel numactl numactl-devel hwloc sudo \
    lua readline-devel ncurses-devel man2html \
-    autoconf automake json-c-devel \
+    autoconf automake json-c-devel libjwt-devel \
    libibmad libibumad rpm-build perl-ExtUtils-MakeMaker.noarch rpm-build make wget

 RUN dnf --enablerepo=powertools install -y munge-devel rrdtool-devel lua-devel hwloc-devel mariadb-server mariadb-devel
--- a/slurm/controller/docker-entrypoint.sh
+++ b/slurm/controller/docker-entrypoint.sh
@@ -4,6 +4,8 @@ set -e
 # Determine the system architecture dynamically
 ARCH=$(uname -m)
 SLURM_VERSION="24.05.3"
+SLURM_JWT=daemon
+SLURMRESTD_SECURITY=disable_user_check

 _delete_secrets() {
    if [ -f /.secret/munge.key ]; then
@@ -11,6 +13,9 @@ _delete_secrets() {
        sudo rm -rf /.secret/munge.key
        sudo rm -rf /.secret/worker-secret.tar.gz
        sudo rm -rf /.secret/setup-worker-ssh.sh
+        sudo rm -rf /.secret/jwt.key
+        sudo rm -rf /.secret/jwt_public.key
+        sudo rm -rf /.secret/jwt_token.key

        echo "Done removing secrets"
        ls /.secret/
@@ -88,6 +93,31 @@ _copy_secrets() {
    rm -f /home/worker/setup-worker-ssh.sh
 }

+_openssl_jwt_key() {
+    cd /.secret
+    openssl rand -base64 32 > jwt.key
+    # openssl genpkey -algorithm RSA -out jwt.key -pkeyopt rsa_keygen_bits:2048
+    # openssl rsa -pubout -in jwt.key -out jwt_public.key
+    cd ..
+}
+
+_generate_jwt_token() {
+    PEM=$(cat /etc/config/jwt.key)
+    USER=\"slurm\"
+    NOW=$(date +%s)
+    IAT="${NOW}"
+    EXP=$((${NOW} + 3600000))
+    HEADER_RAW='{"alg":"HS256", "typ":"JWT"}'
+    HEADER=$(echo -n "${HEADER_RAW}" | openssl base64 | tr -d '=' | tr '/+' '_-' | tr -d '\n')
+    PAYLOAD_RAW='{"iss":'${USER}'}'
+    PAYLOAD=$(echo -n "${PAYLOAD_RAW}" | openssl base64 | tr -d '=' | tr '/+' '_-' | tr -d '\n')
+    HEADER_PAYLOAD="${HEADER}"."${PAYLOAD}"
+    SIGNATURE=$(openssl dgst -sha256 -sign <(echo -n "${PEM}") <(echo -n "${HEADER_PAYLOAD}") | openssl base64 | tr -d '=' | tr '/+' '_-' | tr -d '\n')
+    JWT="${HEADER_PAYLOAD}"."${SIGNATURE}"
+    echo $JWT | cat >/.secret/jwt_token.txt
+    chmod 777 /.secret/jwt_token.txt
+}
+
 # run slurmctld
 _slurmctld() {
    cd /root/rpmbuild/RPMS/$ARCH
@@ -105,19 +135,22 @@ _slurmctld() {
    echo ""
    mkdir -p /var/spool/slurm/ctld /var/spool/slurm/d /var/log/slurm /etc/slurm /var/run/slurm/d /var/run/slurm/ctld /var/lib/slurm/d /var/lib/slurm/ctld
    chown -R slurm: /var/spool/slurm/ctld /var/spool/slurm/d /var/log/slurm /var/spool /var/lib /var/run/slurm/d /var/run/slurm/ctld /var/lib/slurm/d /var/lib/slurm/ctld
+    mkdir -p /etc/config
+    chown -R slurm: /etc/config
+
    touch /var/log/slurmctld.log
-    chown slurm: /var/log/slurmctld.log
+    chown -R slurm: /var/log/slurmctld.log
    touch /var/log/slurmd.log
-    chown slurm: /var/log/slurmd.log
+    chown -R slurm: /var/log/slurmd.log

    touch /var/lib/slurm/d/job_state
-    chown slurm: /var/lib/slurm/d/job_state
+    chown -R slurm: /var/lib/slurm/d/job_state
    touch /var/lib/slurm/d/fed_mgr_state
-    chown slurm: /var/lib/slurm/d/fed_mgr_state
+    chown -R slurm: /var/lib/slurm/d/fed_mgr_state
    touch /var/run/slurm/d/slurmctld.pid
-    chown slurm: /var/run/slurm/d/slurmctld.pid
+    chown -R slurm: /var/run/slurm/d/slurmctld.pid
    touch /var/run/slurm/d/slurmd.pid
-    chown slurm: /var/run/slurm/d/slurmd.pid
+    chown -R slurm: /var/run/slurm/d/slurmd.pid

    if [[ ! -f /home/config/slurm.conf ]]; then
        echo "### Missing slurm.conf ###"
@@ -129,6 +162,19 @@ _slurmctld() {
        chmod 600 /etc/slurm/slurm.conf
    fi

+    _openssl_jwt_key
+
+    if [ ! -f /.secret/jwt.key ]; then
+        echo "### Missing jwt.key ###"
+        exit 1
+    else
+        cp /.secret/jwt.key /etc/config/jwt.key
+        chown slurm: /etc/config/jwt.key
+        chmod 0400 /etc/config/jwt.key
+    fi
+
+    _generate_jwt_token
+
    sudo yum install -y nc
    sudo yum install -y procps
    sudo yum install -y iputils
@@ -149,6 +195,7 @@ _slurmctld() {
 ### main ###
 _delete_secrets
 _sshd_host
+
 _ssh_worker
 _munge_start
 _copy_secrets
--- a/slurm/controller/slurm.conf
+++ b/slurm/controller/slurm.conf
@@ -22,6 +22,8 @@ MpiDefault=none
 SlurmctldPidFile=/var/run/slurm/d/slurmctld.pid
 SlurmdPidFile=/var/run/slurm/d/slurmd.pid
 ProctrackType=proctrack/linuxproc
+AuthAltTypes=auth/jwt
+AuthAltParameters=jwt_key=/etc/config/jwt.key
 #PluginDir=
 #CacheGroups=0
 #FirstJobId=
--- a/slurm/database/docker-entrypoint.sh
+++ b/slurm/database/docker-entrypoint.sh
@@ -4,7 +4,7 @@ set -e
 # Determine the system architecture dynamically
 ARCH=$(uname -m)
 SLURM_VERSION="24.05.3"
-
+SLURM_JWT=daemon
 SLURM_ACCT_DB_SQL=/slurm_acct_db.sql

 # start sshd server
@@ -52,12 +52,16 @@ _wait_for_worker() {

 # run slurmdbd
 _slurmdbd() {
-   cd /root/rpmbuild/RPMS/$ARCH
-   yum -y --nogpgcheck localinstall slurm-$SLURM_VERSION*.$ARCH.rpm \
-       slurm-perlapi-$SLURM_VERSION*.$ARCH.rpm \
-       slurm-slurmdbd-$SLURM_VERSION*.$ARCH.rpm
+  cd /root/rpmbuild/RPMS/$ARCH
+  yum -y --nogpgcheck localinstall slurm-$SLURM_VERSION*.$ARCH.rpm \
+    slurm-perlapi-$SLURM_VERSION*.$ARCH.rpm \
+    slurm-slurmdbd-$SLURM_VERSION*.$ARCH.rpm
  mkdir -p /var/spool/slurm/d /var/log/slurm /etc/slurm
-  chown slurm: /var/spool/slurm/d /var/log/slurm
+  chown -R slurm: /var/spool/slurm/d /var/log/slurm
+
+  mkdir -p /etc/config
+  chown -R slurm: /etc/config
+
  if [[ ! -f /home/config/slurmdbd.conf ]]; then
    echo "### Missing slurmdbd.conf ###"
    exit
@@ -67,8 +71,26 @@ _slurmdbd() {
    chown slurm: /etc/slurm/slurmdbd.conf
    chmod 600 /etc/slurm/slurmdbd.conf
  fi
-  echo "Starting slurmdbd"
+
+  echo -n "checking for jwt.key"
+  while [ ! -f /.secret/jwt.key ]; do
+    echo -n "."
+    sleep 1
+  done
+
+  cp /.secret/jwt.key /etc/config/jwt.key
+  chown slurm: /etc/config/jwt.key
+  chmod 0400 /etc/config/jwt.key
+
+  echo ""
+
+  sudo yum install -y nc
+  sudo yum install -y procps
+  sudo yum install -y iputils
+  
  cp /etc/slurm/slurmdbd.conf /.secret/slurmdbd.conf
+
+  echo "Starting slurmdbd"
  /usr/sbin/slurmdbd -Dvv
  echo "Started slurmdbd"
 }
--- a/slurm/database/slurmdbd.conf
+++ b/slurm/database/slurmdbd.conf
@@ -14,7 +14,8 @@
 # Authentication info
 AuthType=auth/munge
 #AuthInfo=/var/run/munge/munge.socket.2
-#
+AuthAltTypes=auth/jwt
+AuthAltParameters=jwt_key=/etc/config/jwt.key
 # slurmDBD info
 DbdAddr=slurmdbd
 DbdHost=slurmdbd
--- a/slurm/rest/Dockerfile
+++ b/slurm/rest/Dockerfile
@@ -1,10 +1,15 @@
 FROM clustercockpit/slurm.base:24.05.3
 LABEL org.opencontainers.image.authors="jan.eitzinger@fau.de"

+ARG uid_u
+ARG gid_g
+ENV uid_u=${uid_u}
+ENV gid_g=${gid_g}
+
 # clean up
 RUN rm -f /root/rpmbuild/RPMS/slurm-*.rpm \
  && yum clean all \
  && rm -rf /var/cache/yum

 COPY docker-entrypoint.sh /docker-entrypoint.sh
-ENTRYPOINT ["/docker-entrypoint.sh"]
+ENTRYPOINT /docker-entrypoint.sh $uid_u $gid_g
--- a/slurm/rest/docker-entrypoint.sh
+++ b/slurm/rest/docker-entrypoint.sh
@@ -4,6 +4,18 @@ set -e
 # Determine the system architecture dynamically
 ARCH=$(uname -m)
 SLURM_VERSION="24.05.3"
+SLURMRESTD="/tmp/slurmrestd.socket"
+# SLURM_JWT=daemon
+
+uid_u="${1:-}"
+gid_g="${2:-}"
+
+echo Your container args are: "$@"
+
+# Change the uid
+# usermod -u "${uid_u}" slurm
+# Change the gid
+# groupmod -g "${gid_g}" slurm

 # start sshd server
 _sshd_host() {
@@ -14,7 +26,6 @@ _sshd_host() {
    /usr/sbin/sshd
 }

-# start munge and generate key
 # start munge using existing key
 _munge_start_using_key() {
    if [ ! -f /.secret/munge.key ]; then
@@ -37,6 +48,48 @@ _munge_start_using_key() {
    remunge
 }

+_enable_slurmrestd() {
+
+    cd /tmp
+    mkdir statesave
+    dd if=/dev/random of=/tmp/statesave/jwt_hs256.key bs=32 count=1
+    chown slurm:slurm /tmp/statesave/jwt_hs256.key
+    chmod 0600 /tmp/statesave/jwt_hs256.key
+    chown slurm:slurm /tmp/statesave
+    chmod 0755 /tmp/statesave
+
+    cat >/usr/lib/systemd/system/slurmrestd.service <<EOF
+[Unit]
+Description=Slurm REST daemon
+After=network-online.target slurmctld.service
+Wants=network-online.target
+ConditionPathExists=/etc/slurm/slurm.conf
+
+[Service]
+Type=simple
+EnvironmentFile=-/etc/sysconfig/slurmrestd
+EnvironmentFile=-/etc/default/slurmrestd
+# slurmrestd should not run as root or the slurm user.
+# Please either use the -u and -g options in /etc/sysconfig/slurmrestd or
+# /etc/default/slurmrestd, or explicitly set the User and Group in this file
+# an unpriviledged user to run as.
+User=slurm
+Restart=always
+RestartSec=5
+# Group=
+# Default to listen on both socket and slurmrestd port
+ExecStart=/usr/sbin/slurmrestd -f /etc/config/slurmrestd.conf -a rest_auth/jwt $SLURMRESTD_OPTIONS -vvvvvv -s dbv0.0.39,v0.0.39 unix:$SLURMRESTD 0.0.0.0:6820
+# /usr/sbin/slurmrestd -f /etc/config/slurmrestd.conf -vvvvvv -a rest_auth/jwt -s dbv0.0.39,v0.0.39 -u slurm unix:$SLURMRESTD 0.0.0.0:6820
+# Enable auth/jwt be default, comment out the line to disable it for slurmrestd
+Environment="SLURM_JWT=daemon"
+ExecReload=/bin/kill -HUP $MAINPID
+
+[Install]
+WantedBy=multi-user.target
+
+EOF
+}
+
 # run slurmrestd
 _slurmrestd() {
    cd /root/rpmbuild/RPMS/$ARCH
@@ -46,20 +99,23 @@ _slurmrestd() {
        slurm-torque-$SLURM_VERSION*.$ARCH.rpm \
        slurm-slurmctld-$SLURM_VERSION*.$ARCH.rpm \
        slurm-slurmrestd-$SLURM_VERSION*.$ARCH.rpm
+
    echo -n "checking for slurmdbd.conf"
    while [ ! -f /.secret/slurmdbd.conf ]; do
        echo -n "."
        sleep 1
    done
    echo ""
-    # mkdir -p /var/spool/slurm/ctld /var/spool/slurm/d /var/log/slurm /etc/slurm
-    # chown -R slurm: /var/spool/slurm/ctld /var/spool/slurm/d /var/log/slurm

-    mkdir -p /etc/config /var/spool/slurm /var/spool/slurm/restd /var/spool/slurm/restd/rest
-    chown -R slurm: /etc/config /var/spool/slurm /var/spool/slurm/restd /var/spool/slurm/restd/rest
+    mkdir -p /etc/config /var/spool/slurm /var/spool/slurm/restd /var/spool/slurm/restd/rest /var/run/slurm
+    chown -R slurm: /etc/config /var/spool/slurm /var/spool/slurm/restd /var/spool/slurm/restd/rest /var/run/slurm
+    chmod 755 /var/run/slurm

    touch /var/log/slurmrestd.log
    chown slurm: /var/log/slurmrestd.log
+    chown slurm: /tmp
+    chmod 777 /tmp
+
    if [[ ! -f /home/config/slurmrestd.conf ]]; then
        echo "### Missing slurm.conf ###"
        exit
@@ -67,11 +123,31 @@ _slurmrestd() {
        echo "### use provided slurmrestd.conf ###"
        cp /home/config/slurmrestd.conf /etc/config/slurmrestd.conf
        cp /home/config/slurm.conf /etc/config/slurm.conf
-
    fi
+
+    echo -n "checking for jwt.key"
+    while [ ! -f /.secret/jwt.key ]; do
+        echo -n "."
+        sleep 1
+    done
+
+    sudo yum install -y nc
+    sudo yum install -y procps
+    sudo yum install -y iputils
+
+    cp /.secret/jwt.key /etc/config/jwt.key
+    chown slurm: /etc/config/jwt.key
+    chmod 0400 /etc/config/jwt.key
+
+    echo ""
+
    sleep 2s
-    export SLURMRESTD=/var/spool/slurm/restd/rest
-    /usr/sbin/slurmrestd -f /etc/config/slurmrestd.conf -s dbv0.0.39,v0.0.39 -vv -u slurm 0.0.0.0:6820
+    echo "Starting slurmrestd"
+    # _enable_slurmrestd
+    # sudo ln -s /usr/lib/systemd/system/slurmrestd.service /etc/systemd/system/multi-user.target.wants/slurmrestd.service
+
+    /usr/sbin/slurmrestd -f /etc/config/slurmrestd.conf -vvvvvv -s dbv0.0.39,v0.0.39 -u slurm unix:$SLURMRESTD 0.0.0.0:6820
+    echo "Started slurmrestd"
 }

 ### main ###
--- a/slurm/worker/docker-entrypoint.sh
+++ b/slurm/worker/docker-entrypoint.sh
@@ -78,6 +78,10 @@ _slurmd() {
  fi
  echo "found slurm.conf"

+  sudo yum install -y nc
+  sudo yum install -y procps
+  sudo yum install -y iputils
+
  mkdir -p /var/spool/slurm/d /etc/slurm /var/run/slurm/d /var/log/slurm
  chown slurm: /var/spool/slurm/d /var/run/slurm/d /var/log/slurm
  cp /home/config/cgroup.conf /etc/slurm/cgroup.conf