fix + feat: working JWT auth for slurm restd and other daemons

This commit is contained in:
Aditya Ujeniya 2024-10-27 23:41:19 +01:00
parent 07b09a82bb
commit 255f05bee7
10 changed files with 115 additions and 93 deletions

View File

@ -1,4 +1,3 @@
JWT="eyJhbGciOiJSUzI1NiIsICJ0eXAiOiJKV1QifQ.eyJpc3MiOiJzbHVybSJ9.dzAHf1Ojoa149uRCCWY1eP3vDyCIZCOZ3h554R-KJJ8-OP0CJ0ymvSkFISLcYcyd9vVKmaYdSN3tWEF6bNZEmyX7G560i1MbkNFvhkhNVSPLKEKNPs38h5ra3ZlTlLlxAlDzXRAAn6UEEgKdm5vx4Jhec7ptaRL_zeSFpTS5fJPc0QE1Cm7e7nU39-9e8l4WU4KpRMxT6ANFm22_G4-mSA-AgCAvKQFzj2FInKsXDUTGlliNJuAgFxf-9LQxoeAknOQhEqcTXii_yBy9DNcT03pdNcAu5Ru4_qlX62vroInU_eh5mWQyiUdXN9Wj_OfMmfLoYFkJeUFYexBMZnSBgg" SLURM_JWT=$(cat data/slurm/secret/jwt_token.txt)
curl -X 'GET' -v 'http://localhost:6820/slurm/v0.0.39/ping' --location --silent --show-error -H "X-SLURM-USER-NAME: root" -H "X-SLURM-USER-TOKEN: $SLURM_JWT"
# curl -X 'GET' -v 'http://localhost:6820/slurm/v0.0.39/ping' -H "X-SLURM-USER-NAME:slurm" -H "X-SLURM-USER-TOKEN:$SLURM_JWT" # curl -v --unix-socket data/slurm/tmp/slurmrestd.socket 'http://localhost:6820/slurm/v0.0.39/ping'
curl -v --unix-socket data/slurm/tmp/slurmrestd.socket 'http://localhost:6820/slurm/v0.0.39/ping'

View File

@ -72,7 +72,6 @@ services:
volumes: volumes:
- ${DATADIR}/slurm/home:/home - ${DATADIR}/slurm/home:/home
- ${DATADIR}/slurm/secret:/.secret - ${DATADIR}/slurm/secret:/.secret
- ${DATADIR}/slurm/tmp:/tmp:rw
- ./slurm/controller/slurm.conf:/home/config/slurm.conf - ./slurm/controller/slurm.conf:/home/config/slurm.conf
- /etc/timezone:/etc/timezone:ro - /etc/timezone:/etc/timezone:ro
- /etc/localtime:/etc/localtime:ro - /etc/localtime:/etc/localtime:ro
@ -92,11 +91,9 @@ services:
volumes: volumes:
- ${DATADIR}/slurm/home:/home - ${DATADIR}/slurm/home:/home
- ${DATADIR}/slurm/secret:/.secret - ${DATADIR}/slurm/secret:/.secret
- ${DATADIR}/slurm/tmp:/tmp:rw
- ./slurm/database/slurmdbd.conf:/home/config/slurmdbd.conf - ./slurm/database/slurmdbd.conf:/home/config/slurmdbd.conf
- /etc/timezone:/etc/timezone:ro - /etc/timezone:/etc/timezone:ro
- /etc/localtime:/etc/localtime:ro - /etc/localtime:/etc/localtime:ro
- ${DATADIR}/slurm/state:/var/lib/slurm/d
ports: ports:
- "6819:6819" - "6819:6819"
@ -111,7 +108,6 @@ services:
volumes: volumes:
- ${DATADIR}/slurm/home:/home - ${DATADIR}/slurm/home:/home
- ${DATADIR}/slurm/secret:/.secret - ${DATADIR}/slurm/secret:/.secret
- ${DATADIR}/slurm/tmp:/tmp:rw
- ./slurm/worker/cgroup.conf:/home/config/cgroup.conf - ./slurm/worker/cgroup.conf:/home/config/cgroup.conf
- ./slurm/controller/slurm.conf:/home/config/slurm.conf - ./slurm/controller/slurm.conf:/home/config/slurm.conf
- /etc/timezone:/etc/timezone:ro - /etc/timezone:/etc/timezone:ro
@ -124,16 +120,15 @@ services:
hostname: slurmrestd hostname: slurmrestd
build: build:
context: ./slurm/rest context: ./slurm/rest
args: environment:
uid_u: ${UID_U} - SLURM_JWT=daemon
gid_g: ${GID_G} - SLURMRESTD_DEBUG=9
depends_on: depends_on:
- slurmctld - slurmctld
privileged: true privileged: true
volumes: volumes:
- ${DATADIR}/slurm/home:/home - ${DATADIR}/slurm/home:/home
- ${DATADIR}/slurm/secret:/.secret - ${DATADIR}/slurm/secret:/.secret
- ${DATADIR}/slurm/tmp:/tmp:rw
- ./slurm/controller/slurm.conf:/home/config/slurm.conf - ./slurm/controller/slurm.conf:/home/config/slurm.conf
- ./slurm/rest/slurmrestd.conf:/home/config/slurmrestd.conf - ./slurm/rest/slurmrestd.conf:/home/config/slurmrestd.conf
- /etc/timezone:/etc/timezone:ro - /etc/timezone:/etc/timezone:ro

27
jwt_verifier.py Normal file
View File

@ -0,0 +1,27 @@
#!/usr/bin/env python3
import sys
import os
import pprint
import json
import time
from datetime import datetime, timedelta, timezone
from jwt import JWT
from jwt.jwa import HS256
from jwt.jwk import jwk_from_dict
from jwt.utils import b64decode,b64encode
if len(sys.argv) != 2:
sys.exit("verify_jwt.py [JWT Token]");
with open("data/slurm/secret/jwt_hs256.key", "rb") as f:
priv_key = f.read()
signing_key = jwk_from_dict({
'kty': 'oct',
'k': b64encode(priv_key)
})
a = JWT()
b = a.decode(sys.argv[1], signing_key, algorithms=["HS256"])
print(b)

View File

@ -9,10 +9,10 @@ RUN ARCH=$(uname -m) && yum install -y https://rpmfind.net/linux/almalinux/8.10/
RUN groupadd -g 981 munge \ RUN groupadd -g 981 munge \
&& useradd -m -c "MUNGE Uid 'N' Gid Emporium" -d /var/lib/munge -u 981 -g munge -s /sbin/nologin munge \ && useradd -m -c "MUNGE Uid 'N' Gid Emporium" -d /var/lib/munge -u 981 -g munge -s /sbin/nologin munge \
&& groupadd -g 982 slurm \ && groupadd -g 1000 slurm \
&& useradd -m -c "Slurm workload manager" -d /var/lib/slurm -u 982 -g slurm -s /bin/bash slurm \ && useradd -m -c "Slurm workload manager" -d /var/lib/slurm -u 1000 -g slurm -s /bin/bash slurm \
&& groupadd -g 1000 worker \ && groupadd -g 982 worker \
&& useradd -m -c "Workflow user" -d /home/worker -u 1000 -g worker -s /bin/bash worker && useradd -m -c "Workflow user" -d /home/worker -u 982 -g worker -s /bin/bash worker
RUN yum install -y munge munge-libs rng-tools \ RUN yum install -y munge munge-libs rng-tools \
python3 gcc openssl openssl-devel \ python3 gcc openssl openssl-devel \

View File

@ -13,9 +13,8 @@ _delete_secrets() {
sudo rm -rf /.secret/munge.key sudo rm -rf /.secret/munge.key
sudo rm -rf /.secret/worker-secret.tar.gz sudo rm -rf /.secret/worker-secret.tar.gz
sudo rm -rf /.secret/setup-worker-ssh.sh sudo rm -rf /.secret/setup-worker-ssh.sh
sudo rm -rf /.secret/jwt.key sudo rm -rf /.secret/jwt_hs256.key
sudo rm -rf /.secret/jwt_public.key sudo rm -rf /.secret/jwt_token.txt
sudo rm -rf /.secret/jwt_token.key
echo "Done removing secrets" echo "Done removing secrets"
ls /.secret/ ls /.secret/
@ -94,27 +93,48 @@ _copy_secrets() {
} }
_openssl_jwt_key() { _openssl_jwt_key() {
cd /.secret
openssl rand -base64 32 > jwt.key mkdir -p /var/spool/slurm/statesave
# openssl genpkey -algorithm RSA -out jwt.key -pkeyopt rsa_keygen_bits:2048 dd if=/dev/random of=/var/spool/slurm/statesave/jwt_hs256.key bs=32 count=1
# openssl rsa -pubout -in jwt.key -out jwt_public.key chown slurm:slurm /var/spool/slurm/statesave/jwt_hs256.key
cd .. chmod 0600 /var/spool/slurm/statesave/jwt_hs256.key
chown slurm:slurm /var/spool/slurm/statesave
chmod 0755 /var/spool/slurm/statesave
cp /var/spool/slurm/statesave/jwt_hs256.key /.secret/jwt_hs256.key
chmod 777 /.secret/jwt_hs256.key
} }
_generate_jwt_token() { _generate_jwt_token() {
PEM=$(cat /etc/config/jwt.key)
USER=\"slurm\" secret_key=$(cat /var/spool/slurm/statesave/jwt_hs256.key)
NOW=$(date +%s) start_time=$(date +%s)
IAT="${NOW}" exp_time=$((start_time + 100000000))
EXP=$((${NOW} + 3600000)) base64url() {
HEADER_RAW='{"alg":"HS256", "typ":"JWT"}' # Don't wrap, make URL-safe, delete trailer.
HEADER=$(echo -n "${HEADER_RAW}" | openssl base64 | tr -d '=' | tr '/+' '_-' | tr -d '\n') base64 -w 0 | tr '+/' '-_' | tr -d '='
PAYLOAD_RAW='{"iss":'${USER}'}' }
PAYLOAD=$(echo -n "${PAYLOAD_RAW}" | openssl base64 | tr -d '=' | tr '/+' '_-' | tr -d '\n')
HEADER_PAYLOAD="${HEADER}"."${PAYLOAD}" jwt_header=$(echo -n '{"alg":"HS256","typ":"JWT"}' | base64url)
SIGNATURE=$(openssl dgst -sha256 -sign <(echo -n "${PEM}") <(echo -n "${HEADER_PAYLOAD}") | openssl base64 | tr -d '=' | tr '/+' '_-' | tr -d '\n')
JWT="${HEADER_PAYLOAD}"."${SIGNATURE}" jwt_claims=$(cat <<EOF |
echo $JWT | cat >/.secret/jwt_token.txt {
"sun": "root",
"exp": $exp_time,
"iat": $start_time
}
EOF
jq -Mcj '.' | base64url)
# jq -Mcj => Monochrome output, compact output, join lines
jwt_signature=$(echo -n "${jwt_header}.${jwt_claims}" |
openssl dgst -sha256 -hmac "$secret_key" -binary | base64url)
# Use the same colours as jwt.io, more-or-less.
echo "$(tput setaf 1)${jwt_header}$(tput sgr0).$(tput setaf 5)${jwt_claims}$(tput sgr0).$(tput setaf 6)${jwt_signature}$(tput sgr0)"
jwt="${jwt_header}.${jwt_claims}.${jwt_signature}"
echo $jwt | cat >/.secret/jwt_token.txt
chmod 777 /.secret/jwt_token.txt chmod 777 /.secret/jwt_token.txt
} }
@ -162,23 +182,24 @@ _slurmctld() {
chmod 600 /etc/slurm/slurm.conf chmod 600 /etc/slurm/slurm.conf
fi fi
_openssl_jwt_key
if [ ! -f /.secret/jwt.key ]; then
echo "### Missing jwt.key ###"
exit 1
else
cp /.secret/jwt.key /etc/config/jwt.key
chown slurm: /etc/config/jwt.key
chmod 0600 /etc/config/jwt.key
fi
_generate_jwt_token
sudo yum install -y nc sudo yum install -y nc
sudo yum install -y procps sudo yum install -y procps
sudo yum install -y iputils sudo yum install -y iputils
sudo yum install -y lsof sudo yum install -y lsof
sudo yum install -y jq
_openssl_jwt_key
if [ ! -f /.secret/jwt_hs256.key ]; then
echo "### Missing jwt.key ###"
exit 1
else
cp /.secret/jwt_hs256.key /etc/config/jwt_hs256.key
chown slurm: /etc/config/jwt_hs256.key
chmod 0600 /etc/config/jwt_hs256.key
fi
_generate_jwt_token
while ! nc -z slurmdbd 6819; do while ! nc -z slurmdbd 6819; do
echo "Waiting for slurmdbd to be ready..." echo "Waiting for slurmdbd to be ready..."

View File

@ -23,7 +23,7 @@ SlurmctldPidFile=/var/run/slurm/d/slurmctld.pid
SlurmdPidFile=/var/run/slurm/d/slurmd.pid SlurmdPidFile=/var/run/slurm/d/slurmd.pid
ProctrackType=proctrack/linuxproc ProctrackType=proctrack/linuxproc
AuthAltTypes=auth/jwt AuthAltTypes=auth/jwt
AuthAltParameters=jwt_key=/etc/config/jwt.key AuthAltParameters=jwt_key=/var/spool/slurm/statesave/jwt_hs256.key
#PluginDir= #PluginDir=
#CacheGroups=0 #CacheGroups=0
#FirstJobId= #FirstJobId=
@ -71,9 +71,9 @@ SelectTypeParameters=CR_CPU_Memory
#PriorityMaxAge=1-0 #PriorityMaxAge=1-0
# #
# LOGGING # LOGGING
SlurmctldDebug=3 SlurmctldDebug=6
SlurmctldLogFile=/var/log/slurm/slurmctld.log SlurmctldLogFile=/var/log/slurm/slurmctld.log
SlurmdDebug=3 SlurmdDebug=6
SlurmdLogFile=/var/log/slurm/slurmd.log SlurmdLogFile=/var/log/slurm/slurmd.log
JobCompType=jobcomp/filetxt JobCompType=jobcomp/filetxt
JobCompLoc=/var/log/slurm/jobcomp.log JobCompLoc=/var/log/slurm/jobcomp.log

View File

@ -74,14 +74,17 @@ _slurmdbd() {
fi fi
echo "checking for jwt.key" echo "checking for jwt.key"
while [ ! -f /.secret/jwt.key ]; do while [ ! -f /.secret/jwt_hs256.key ]; do
echo "." echo "."
sleep 1 sleep 1
done done
cp /.secret/jwt.key /etc/config/jwt.key mkdir -p /var/spool/slurm/statesave
chown slurm: /etc/config/jwt.key chown slurm:slurm /var/spool/slurm/statesave
chmod 0400 /etc/config/jwt.key chmod 0755 /var/spool/slurm/statesave
cp /.secret/jwt_hs256.key /var/spool/slurm/statesave/jwt_hs256.key
chown slurm: /var/spool/slurm/statesave/jwt_hs256.key
chmod 0600 /var/spool/slurm/statesave/jwt_hs256.key
echo "" echo ""

View File

@ -15,7 +15,7 @@
AuthType=auth/munge AuthType=auth/munge
#AuthInfo=/var/run/munge/munge.socket.2 #AuthInfo=/var/run/munge/munge.socket.2
AuthAltTypes=auth/jwt AuthAltTypes=auth/jwt
AuthAltParameters=jwt_key=/etc/config/jwt.key AuthAltParameters=jwt_key=/var/spool/slurm/statesave/jwt_hs256.key
# slurmDBD info # slurmDBD info
DbdAddr=slurmdbd DbdAddr=slurmdbd
DbdHost=slurmdbd DbdHost=slurmdbd

View File

@ -1,15 +1,10 @@
FROM clustercockpit/slurm.base:24.05.3 FROM clustercockpit/slurm.base:24.05.3
LABEL org.opencontainers.image.authors="jan.eitzinger@fau.de" LABEL org.opencontainers.image.authors="jan.eitzinger@fau.de"
ARG uid_u
ARG gid_g
ENV uid_u=${uid_u}
ENV gid_g=${gid_g}
# clean up # clean up
RUN rm -f /root/rpmbuild/RPMS/slurm-*.rpm \ RUN rm -f /root/rpmbuild/RPMS/slurm-*.rpm \
&& yum clean all \ && yum clean all \
&& rm -rf /var/cache/yum && rm -rf /var/cache/yum
COPY docker-entrypoint.sh /docker-entrypoint.sh COPY docker-entrypoint.sh /docker-entrypoint.sh
ENTRYPOINT /docker-entrypoint.sh $uid_u $gid_g ENTRYPOINT ["/docker-entrypoint.sh"]

View File

@ -4,18 +4,8 @@ set -e
# Determine the system architecture dynamically # Determine the system architecture dynamically
ARCH=$(uname -m) ARCH=$(uname -m)
SLURM_VERSION="24.05.3" SLURM_VERSION="24.05.3"
SLURMRESTD="/tmp/slurmrestd.socket" # SLURMRESTD="/tmp/slurmrestd.socket"
# SLURM_JWT=daemon SLURM_JWT=daemon
uid_u="${1:-}"
gid_g="${2:-}"
echo Your container args are: "$@"
# Change the uid
# usermod -u "${uid_u}" slurm
# Change the gid
# groupmod -g "${gid_g}" slurm
# start sshd server # start sshd server
_sshd_host() { _sshd_host() {
@ -50,14 +40,6 @@ _munge_start_using_key() {
_enable_slurmrestd() { _enable_slurmrestd() {
cd /tmp
mkdir statesave
dd if=/dev/random of=/tmp/statesave/jwt_hs256.key bs=32 count=1
chown slurm:slurm /tmp/statesave/jwt_hs256.key
chmod 0600 /tmp/statesave/jwt_hs256.key
chown slurm:slurm /tmp/statesave
chmod 0755 /tmp/statesave
cat >/usr/lib/systemd/system/slurmrestd.service <<EOF cat >/usr/lib/systemd/system/slurmrestd.service <<EOF
[Unit] [Unit]
Description=Slurm REST daemon Description=Slurm REST daemon
@ -78,8 +60,7 @@ Restart=always
RestartSec=5 RestartSec=5
# Group= # Group=
# Default to listen on both socket and slurmrestd port # Default to listen on both socket and slurmrestd port
ExecStart=/usr/sbin/slurmrestd -f /etc/config/slurmrestd.conf -a rest_auth/jwt $SLURMRESTD_OPTIONS -vvvvvv -s dbv0.0.39,v0.0.39 unix:$SLURMRESTD 0.0.0.0:6820 ExecStart=/usr/sbin/slurmrestd -f /etc/config/slurmrestd.conf -a rest_auth/jwt $SLURMRESTD_OPTIONS -vvvvvv -s dbv0.0.39,v0.0.39 0.0.0.0:6820
# /usr/sbin/slurmrestd -f /etc/config/slurmrestd.conf -vvvvvv -a rest_auth/jwt -s dbv0.0.39,v0.0.39 -u slurm unix:$SLURMRESTD 0.0.0.0:6820
# Enable auth/jwt be default, comment out the line to disable it for slurmrestd # Enable auth/jwt be default, comment out the line to disable it for slurmrestd
Environment="SLURM_JWT=daemon" Environment="SLURM_JWT=daemon"
ExecReload=/bin/kill -HUP $MAINPID ExecReload=/bin/kill -HUP $MAINPID
@ -113,8 +94,6 @@ _slurmrestd() {
touch /var/log/slurmrestd.log touch /var/log/slurmrestd.log
chown slurm: /var/log/slurmrestd.log chown slurm: /var/log/slurmrestd.log
chown worker: /tmp
chmod 770 /tmp
if [[ ! -f /home/config/slurmrestd.conf ]]; then if [[ ! -f /home/config/slurmrestd.conf ]]; then
echo "### Missing slurm.conf ###" echo "### Missing slurm.conf ###"
@ -126,7 +105,7 @@ _slurmrestd() {
fi fi
echo "checking for jwt.key" echo "checking for jwt.key"
while [ ! -f /.secret/jwt.key ]; do while [ ! -f /.secret/jwt_hs256.key ]; do
echo "." echo "."
sleep 1 sleep 1
done done
@ -137,9 +116,12 @@ _slurmrestd() {
sudo yum install -y lsof sudo yum install -y lsof
sudo yum install -y socat sudo yum install -y socat
cp /.secret/jwt.key /etc/config/jwt.key mkdir -p /var/spool/slurm/statesave
chown slurm: /etc/config/jwt.key chown slurm:slurm /var/spool/slurm/statesave
chmod 0400 /etc/config/jwt.key chmod 0755 /var/spool/slurm/statesave
cp /.secret/jwt_hs256.key /var/spool/slurm/statesave/jwt_hs256.key
chown slurm: /var/spool/slurm/statesave/jwt_hs256.key
chmod 0400 /var/spool/slurm/statesave/jwt_hs256.key
echo "" echo ""
@ -148,7 +130,7 @@ _slurmrestd() {
# _enable_slurmrestd # _enable_slurmrestd
# sudo ln -s /usr/lib/systemd/system/slurmrestd.service /etc/systemd/system/multi-user.target.wants/slurmrestd.service # sudo ln -s /usr/lib/systemd/system/slurmrestd.service /etc/systemd/system/multi-user.target.wants/slurmrestd.service
/usr/sbin/slurmrestd -f /etc/config/slurmrestd.conf -vvvvvv -s dbv0.0.39,v0.0.39 -u worker unix:$SLURMRESTD 0.0.0.0:6820 SLURMRESTD_SECURITY=disable_user_check SLURMRESTD_DEBUG=9 /usr/sbin/slurmrestd -f /etc/config/slurmrestd.conf -a rest_auth/jwt -s dbv0.0.39,v0.0.39 -u slurm 0.0.0.0:6820
echo "Started slurmrestd" echo "Started slurmrestd"
} }