fix + feat: working JWT auth for slurm restd and other daemons

curl_slurmrestd.sh

@@ -1,4 +1,3 @@
-JWT="eyJhbGciOiJSUzI1NiIsICJ0eXAiOiJKV1QifQ.eyJpc3MiOiJzbHVybSJ9.dzAHf1Ojoa149uRCCWY1eP3vDyCIZCOZ3h554R-KJJ8-OP0CJ0ymvSkFISLcYcyd9vVKmaYdSN3tWEF6bNZEmyX7G560i1MbkNFvhkhNVSPLKEKNPs38h5ra3ZlTlLlxAlDzXRAAn6UEEgKdm5vx4Jhec7ptaRL_zeSFpTS5fJPc0QE1Cm7e7nU39-9e8l4WU4KpRMxT6ANFm22_G4-mSA-AgCAvKQFzj2FInKsXDUTGlliNJuAgFxf-9LQxoeAknOQhEqcTXii_yBy9DNcT03pdNcAu5Ru4_qlX62vroInU_eh5mWQyiUdXN9Wj_OfMmfLoYFkJeUFYexBMZnSBgg"
+SLURM_JWT=$(cat data/slurm/secret/jwt_token.txt)
-
+curl -X 'GET' -v 'http://localhost:6820/slurm/v0.0.39/ping' --location --silent --show-error -H "X-SLURM-USER-NAME: root" -H "X-SLURM-USER-TOKEN: $SLURM_JWT"
-# curl -X 'GET' -v 'http://localhost:6820/slurm/v0.0.39/ping' -H "X-SLURM-USER-NAME:slurm" -H "X-SLURM-USER-TOKEN:$SLURM_JWT"
+# curl -v --unix-socket data/slurm/tmp/slurmrestd.socket 'http://localhost:6820/slurm/v0.0.39/ping'
-curl -v --unix-socket data/slurm/tmp/slurmrestd.socket 'http://localhost:6820/slurm/v0.0.39/ping'

docker-compose.yml

@@ -72,7 +72,6 @@ services:
     volumes:
       - ${DATADIR}/slurm/home:/home
       - ${DATADIR}/slurm/secret:/.secret
-      - ${DATADIR}/slurm/tmp:/tmp:rw
       - ./slurm/controller/slurm.conf:/home/config/slurm.conf
       - /etc/timezone:/etc/timezone:ro
       - /etc/localtime:/etc/localtime:ro
@@ -92,11 +91,9 @@ services:
     volumes:
       - ${DATADIR}/slurm/home:/home
       - ${DATADIR}/slurm/secret:/.secret
-      - ${DATADIR}/slurm/tmp:/tmp:rw
       - ./slurm/database/slurmdbd.conf:/home/config/slurmdbd.conf
       - /etc/timezone:/etc/timezone:ro
       - /etc/localtime:/etc/localtime:ro
-      - ${DATADIR}/slurm/state:/var/lib/slurm/d
     ports:
       - "6819:6819"
 
@@ -111,7 +108,6 @@ services:
     volumes:
       - ${DATADIR}/slurm/home:/home
       - ${DATADIR}/slurm/secret:/.secret
-      - ${DATADIR}/slurm/tmp:/tmp:rw
       - ./slurm/worker/cgroup.conf:/home/config/cgroup.conf
       - ./slurm/controller/slurm.conf:/home/config/slurm.conf
       - /etc/timezone:/etc/timezone:ro
@@ -124,16 +120,15 @@ services:
     hostname: slurmrestd
     build:
       context: ./slurm/rest
-      args:
+    environment:
-        uid_u: ${UID_U}
+      - SLURM_JWT=daemon
-        gid_g: ${GID_G}
+      - SLURMRESTD_DEBUG=9
     depends_on:
       - slurmctld
     privileged: true
     volumes:
       - ${DATADIR}/slurm/home:/home
       - ${DATADIR}/slurm/secret:/.secret
-      - ${DATADIR}/slurm/tmp:/tmp:rw
       - ./slurm/controller/slurm.conf:/home/config/slurm.conf
       - ./slurm/rest/slurmrestd.conf:/home/config/slurmrestd.conf
       - /etc/timezone:/etc/timezone:ro

jwt_verifier.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+import sys
+import os
+import pprint
+import json
+import time
+from datetime import datetime, timedelta, timezone
+
+from jwt import JWT
+from jwt.jwa import HS256
+from jwt.jwk import jwk_from_dict
+from jwt.utils import b64decode,b64encode
+
+if len(sys.argv) != 2:
+    sys.exit("verify_jwt.py [JWT Token]");
+
+with open("data/slurm/secret/jwt_hs256.key", "rb") as f:
+    priv_key = f.read()
+
+signing_key = jwk_from_dict({
+    'kty': 'oct',
+    'k': b64encode(priv_key)
+})
+
+a = JWT()
+b = a.decode(sys.argv[1], signing_key, algorithms=["HS256"])
+print(b)

slurm/base/Dockerfile

@@ -9,10 +9,10 @@ RUN ARCH=$(uname -m) && yum install -y https://rpmfind.net/linux/almalinux/8.10/
 
 RUN groupadd -g 981 munge \
   && useradd  -m -c "MUNGE Uid 'N' Gid Emporium" -d /var/lib/munge -u 981 -g munge  -s /sbin/nologin munge \
-  && groupadd -g 982 slurm \
+  && groupadd -g 1000 slurm \
-  && useradd  -m -c "Slurm workload manager" -d /var/lib/slurm -u 982 -g slurm  -s /bin/bash slurm \
+  && useradd  -m -c "Slurm workload manager" -d /var/lib/slurm -u 1000 -g slurm  -s /bin/bash slurm \
-  && groupadd -g 1000 worker \
+  && groupadd -g 982 worker \
-  && useradd  -m -c "Workflow user" -d /home/worker -u 1000 -g worker  -s /bin/bash worker
+  && useradd  -m -c "Workflow user" -d /home/worker -u 982 -g worker  -s /bin/bash worker
 
 RUN yum install -y munge munge-libs rng-tools \
     python3 gcc openssl openssl-devel \

slurm/controller/docker-entrypoint.sh

@@ -13,9 +13,8 @@ _delete_secrets() {
         sudo rm -rf /.secret/munge.key
         sudo rm -rf /.secret/worker-secret.tar.gz
         sudo rm -rf /.secret/setup-worker-ssh.sh
-        sudo rm -rf /.secret/jwt.key
+        sudo rm -rf /.secret/jwt_hs256.key
-        sudo rm -rf /.secret/jwt_public.key
+        sudo rm -rf /.secret/jwt_token.txt
-        sudo rm -rf /.secret/jwt_token.key
 
         echo "Done removing secrets"
         ls /.secret/
@@ -94,27 +93,48 @@ _copy_secrets() {
 }
 
 _openssl_jwt_key() {
-    cd /.secret
+
-    openssl rand -base64 32 > jwt.key
+    mkdir -p /var/spool/slurm/statesave
-    # openssl genpkey -algorithm RSA -out jwt.key -pkeyopt rsa_keygen_bits:2048
+    dd if=/dev/random of=/var/spool/slurm/statesave/jwt_hs256.key bs=32 count=1
-    # openssl rsa -pubout -in jwt.key -out jwt_public.key
+    chown slurm:slurm /var/spool/slurm/statesave/jwt_hs256.key
-    cd ..
+    chmod 0600 /var/spool/slurm/statesave/jwt_hs256.key
+    chown slurm:slurm /var/spool/slurm/statesave
+    chmod 0755 /var/spool/slurm/statesave
+    cp /var/spool/slurm/statesave/jwt_hs256.key /.secret/jwt_hs256.key
+    chmod 777 /.secret/jwt_hs256.key
 }
 
 _generate_jwt_token() {
-    PEM=$(cat /etc/config/jwt.key)
+
-    USER=\"slurm\"
+    secret_key=$(cat /var/spool/slurm/statesave/jwt_hs256.key)
-    NOW=$(date +%s)
+    start_time=$(date +%s)
-    IAT="${NOW}"
+    exp_time=$((start_time + 100000000))
-    EXP=$((${NOW} + 3600000))
+    base64url() {
-    HEADER_RAW='{"alg":"HS256", "typ":"JWT"}'
+        # Don't wrap, make URL-safe, delete trailer.
-    HEADER=$(echo -n "${HEADER_RAW}" | openssl base64 | tr -d '=' | tr '/+' '_-' | tr -d '\n')
+        base64 -w 0 | tr '+/' '-_' | tr -d '='
-    PAYLOAD_RAW='{"iss":'${USER}'}'
+    }
-    PAYLOAD=$(echo -n "${PAYLOAD_RAW}" | openssl base64 | tr -d '=' | tr '/+' '_-' | tr -d '\n')
+
-    HEADER_PAYLOAD="${HEADER}"."${PAYLOAD}"
+    jwt_header=$(echo -n '{"alg":"HS256","typ":"JWT"}' | base64url)
-    SIGNATURE=$(openssl dgst -sha256 -sign <(echo -n "${PEM}") <(echo -n "${HEADER_PAYLOAD}") | openssl base64 | tr -d '=' | tr '/+' '_-' | tr -d '\n')
+
-    JWT="${HEADER_PAYLOAD}"."${SIGNATURE}"
+    jwt_claims=$(cat <<EOF |
-    echo $JWT | cat >/.secret/jwt_token.txt
+{
+  "sun": "root",
+  "exp": $exp_time,
+  "iat": $start_time
+}
+EOF
+        jq -Mcj '.' | base64url)
+    # jq -Mcj => Monochrome output, compact output, join lines
+
+    jwt_signature=$(echo -n "${jwt_header}.${jwt_claims}" |
+        openssl dgst -sha256 -hmac "$secret_key" -binary | base64url)
+
+    # Use the same colours as jwt.io, more-or-less.
+    echo "$(tput setaf 1)${jwt_header}$(tput sgr0).$(tput setaf 5)${jwt_claims}$(tput sgr0).$(tput setaf 6)${jwt_signature}$(tput sgr0)"
+
+    jwt="${jwt_header}.${jwt_claims}.${jwt_signature}"
+
+    echo $jwt | cat >/.secret/jwt_token.txt
     chmod 777 /.secret/jwt_token.txt
 }
 
@@ -162,23 +182,24 @@ _slurmctld() {
         chmod 600 /etc/slurm/slurm.conf
     fi
 
-    _openssl_jwt_key
-
-    if [ ! -f /.secret/jwt.key ]; then
-        echo "### Missing jwt.key ###"
-        exit 1
-    else
-        cp /.secret/jwt.key /etc/config/jwt.key
-        chown slurm: /etc/config/jwt.key
-        chmod 0600 /etc/config/jwt.key
-    fi
-
-    _generate_jwt_token
-
     sudo yum install -y nc
     sudo yum install -y procps
     sudo yum install -y iputils
     sudo yum install -y lsof
+    sudo yum install -y jq
+
+    _openssl_jwt_key
+
+    if [ ! -f /.secret/jwt_hs256.key ]; then
+        echo "### Missing jwt.key ###"
+        exit 1
+    else
+        cp /.secret/jwt_hs256.key /etc/config/jwt_hs256.key
+        chown slurm: /etc/config/jwt_hs256.key
+        chmod 0600 /etc/config/jwt_hs256.key
+    fi
+
+    _generate_jwt_token
 
     while ! nc -z slurmdbd 6819; do
         echo "Waiting for slurmdbd to be ready..."

slurm/controller/slurm.conf

@@ -23,7 +23,7 @@ SlurmctldPidFile=/var/run/slurm/d/slurmctld.pid
 SlurmdPidFile=/var/run/slurm/d/slurmd.pid
 ProctrackType=proctrack/linuxproc
 AuthAltTypes=auth/jwt
-AuthAltParameters=jwt_key=/etc/config/jwt.key
+AuthAltParameters=jwt_key=/var/spool/slurm/statesave/jwt_hs256.key
 #PluginDir=
 #CacheGroups=0
 #FirstJobId=
@@ -71,9 +71,9 @@ SelectTypeParameters=CR_CPU_Memory
 #PriorityMaxAge=1-0
 #
 # LOGGING
-SlurmctldDebug=3
+SlurmctldDebug=6
 SlurmctldLogFile=/var/log/slurm/slurmctld.log
-SlurmdDebug=3
+SlurmdDebug=6
 SlurmdLogFile=/var/log/slurm/slurmd.log
 JobCompType=jobcomp/filetxt
 JobCompLoc=/var/log/slurm/jobcomp.log

slurm/database/docker-entrypoint.sh

@@ -74,14 +74,17 @@ _slurmdbd() {
   fi
 
   echo "checking for jwt.key"
-  while [ ! -f /.secret/jwt.key ]; do
+  while [ ! -f /.secret/jwt_hs256.key ]; do
     echo "."
     sleep 1
   done
 
-  cp /.secret/jwt.key /etc/config/jwt.key
+  mkdir -p /var/spool/slurm/statesave
-  chown slurm: /etc/config/jwt.key
+  chown slurm:slurm /var/spool/slurm/statesave
-  chmod 0400 /etc/config/jwt.key
+  chmod 0755 /var/spool/slurm/statesave
+  cp /.secret/jwt_hs256.key /var/spool/slurm/statesave/jwt_hs256.key
+  chown slurm: /var/spool/slurm/statesave/jwt_hs256.key
+  chmod 0600 /var/spool/slurm/statesave/jwt_hs256.key
 
   echo ""
 

slurm/database/slurmdbd.conf

@@ -15,7 +15,7 @@
 AuthType=auth/munge
 #AuthInfo=/var/run/munge/munge.socket.2
 AuthAltTypes=auth/jwt
-AuthAltParameters=jwt_key=/etc/config/jwt.key
+AuthAltParameters=jwt_key=/var/spool/slurm/statesave/jwt_hs256.key
 # slurmDBD info
 DbdAddr=slurmdbd
 DbdHost=slurmdbd

slurm/rest/Dockerfile

@@ -1,15 +1,10 @@
 FROM clustercockpit/slurm.base:24.05.3
 LABEL org.opencontainers.image.authors="jan.eitzinger@fau.de"
 
-ARG uid_u
-ARG gid_g
-ENV uid_u=${uid_u}
-ENV gid_g=${gid_g}
-
 # clean up
 RUN rm -f /root/rpmbuild/RPMS/slurm-*.rpm \
   && yum clean all \
   && rm -rf /var/cache/yum
 
 COPY docker-entrypoint.sh /docker-entrypoint.sh
-ENTRYPOINT /docker-entrypoint.sh $uid_u $gid_g
+ENTRYPOINT ["/docker-entrypoint.sh"]

slurm/rest/docker-entrypoint.sh

@@ -4,18 +4,8 @@ set -e
 # Determine the system architecture dynamically
 ARCH=$(uname -m)
 SLURM_VERSION="24.05.3"
-SLURMRESTD="/tmp/slurmrestd.socket"
+# SLURMRESTD="/tmp/slurmrestd.socket"
-# SLURM_JWT=daemon
+SLURM_JWT=daemon
-
-uid_u="${1:-}"
-gid_g="${2:-}"
-
-echo Your container args are: "$@"
-
-# Change the uid
-# usermod -u "${uid_u}" slurm
-# Change the gid
-# groupmod -g "${gid_g}" slurm
 
 # start sshd server
 _sshd_host() {
@@ -50,14 +40,6 @@ _munge_start_using_key() {
 
 _enable_slurmrestd() {
 
-    cd /tmp
-    mkdir statesave
-    dd if=/dev/random of=/tmp/statesave/jwt_hs256.key bs=32 count=1
-    chown slurm:slurm /tmp/statesave/jwt_hs256.key
-    chmod 0600 /tmp/statesave/jwt_hs256.key
-    chown slurm:slurm /tmp/statesave
-    chmod 0755 /tmp/statesave
-
     cat >/usr/lib/systemd/system/slurmrestd.service <<EOF
 [Unit]
 Description=Slurm REST daemon
@@ -78,8 +60,7 @@ Restart=always
 RestartSec=5
 # Group=
 # Default to listen on both socket and slurmrestd port
-ExecStart=/usr/sbin/slurmrestd -f /etc/config/slurmrestd.conf -a rest_auth/jwt $SLURMRESTD_OPTIONS -vvvvvv -s dbv0.0.39,v0.0.39 unix:$SLURMRESTD 0.0.0.0:6820
+ExecStart=/usr/sbin/slurmrestd -f /etc/config/slurmrestd.conf -a rest_auth/jwt $SLURMRESTD_OPTIONS -vvvvvv -s dbv0.0.39,v0.0.39 0.0.0.0:6820
-# /usr/sbin/slurmrestd -f /etc/config/slurmrestd.conf -vvvvvv -a rest_auth/jwt -s dbv0.0.39,v0.0.39 -u slurm unix:$SLURMRESTD 0.0.0.0:6820
 # Enable auth/jwt be default, comment out the line to disable it for slurmrestd
 Environment="SLURM_JWT=daemon"
 ExecReload=/bin/kill -HUP $MAINPID
@@ -113,8 +94,6 @@ _slurmrestd() {
 
     touch /var/log/slurmrestd.log
     chown slurm: /var/log/slurmrestd.log
-    chown worker: /tmp
-    chmod 770 /tmp
 
     if [[ ! -f /home/config/slurmrestd.conf ]]; then
         echo "### Missing slurm.conf ###"
@@ -126,7 +105,7 @@ _slurmrestd() {
     fi
 
     echo "checking for jwt.key"
-    while [ ! -f /.secret/jwt.key ]; do
+    while [ ! -f /.secret/jwt_hs256.key ]; do
         echo "."
         sleep 1
     done
@@ -137,9 +116,12 @@ _slurmrestd() {
     sudo yum install -y lsof
     sudo yum install -y socat
 
-    cp /.secret/jwt.key /etc/config/jwt.key
+    mkdir -p /var/spool/slurm/statesave
-    chown slurm: /etc/config/jwt.key
+    chown slurm:slurm /var/spool/slurm/statesave
-    chmod 0400 /etc/config/jwt.key
+    chmod 0755 /var/spool/slurm/statesave
+    cp /.secret/jwt_hs256.key /var/spool/slurm/statesave/jwt_hs256.key
+    chown slurm: /var/spool/slurm/statesave/jwt_hs256.key
+    chmod 0400 /var/spool/slurm/statesave/jwt_hs256.key
 
     echo ""
 
@@ -148,7 +130,7 @@ _slurmrestd() {
     # _enable_slurmrestd
     # sudo ln -s /usr/lib/systemd/system/slurmrestd.service /etc/systemd/system/multi-user.target.wants/slurmrestd.service
 
-    /usr/sbin/slurmrestd -f /etc/config/slurmrestd.conf -vvvvvv -s dbv0.0.39,v0.0.39 -u worker unix:$SLURMRESTD 0.0.0.0:6820
+    SLURMRESTD_SECURITY=disable_user_check SLURMRESTD_DEBUG=9 /usr/sbin/slurmrestd -f /etc/config/slurmrestd.conf -a rest_auth/jwt -s dbv0.0.39,v0.0.39 -u slurm 0.0.0.0:6820
     echo "Started slurmrestd"
 }