From 255f05bee7156087b8c7b8d99ac4dd09377a19eb Mon Sep 17 00:00:00 2001 From: Aditya Ujeniya Date: Sun, 27 Oct 2024 23:41:19 +0100 Subject: [PATCH] fix + feat: working JWT auth for slurm restd and other daemons --- curl_slurmrestd.sh | 7 +-- docker-compose.yml | 11 +--- jwt_verifier.py | 27 ++++++++ slurm/base/Dockerfile | 8 +-- slurm/controller/docker-entrypoint.sh | 89 +++++++++++++++++---------- slurm/controller/slurm.conf | 6 +- slurm/database/docker-entrypoint.sh | 11 ++-- slurm/database/slurmdbd.conf | 2 +- slurm/rest/Dockerfile | 7 +-- slurm/rest/docker-entrypoint.sh | 40 ++++-------- 10 files changed, 115 insertions(+), 93 deletions(-) create mode 100644 jwt_verifier.py diff --git a/curl_slurmrestd.sh b/curl_slurmrestd.sh index a52cd70..dc506ff 100755 --- a/curl_slurmrestd.sh +++ b/curl_slurmrestd.sh @@ -1,4 +1,3 @@ -JWT="eyJhbGciOiJSUzI1NiIsICJ0eXAiOiJKV1QifQ.eyJpc3MiOiJzbHVybSJ9.dzAHf1Ojoa149uRCCWY1eP3vDyCIZCOZ3h554R-KJJ8-OP0CJ0ymvSkFISLcYcyd9vVKmaYdSN3tWEF6bNZEmyX7G560i1MbkNFvhkhNVSPLKEKNPs38h5ra3ZlTlLlxAlDzXRAAn6UEEgKdm5vx4Jhec7ptaRL_zeSFpTS5fJPc0QE1Cm7e7nU39-9e8l4WU4KpRMxT6ANFm22_G4-mSA-AgCAvKQFzj2FInKsXDUTGlliNJuAgFxf-9LQxoeAknOQhEqcTXii_yBy9DNcT03pdNcAu5Ru4_qlX62vroInU_eh5mWQyiUdXN9Wj_OfMmfLoYFkJeUFYexBMZnSBgg" - -# curl -X 'GET' -v 'http://localhost:6820/slurm/v0.0.39/ping' -H "X-SLURM-USER-NAME:slurm" -H "X-SLURM-USER-TOKEN:$SLURM_JWT" -curl -v --unix-socket data/slurm/tmp/slurmrestd.socket 'http://localhost:6820/slurm/v0.0.39/ping' \ No newline at end of file +SLURM_JWT=$(cat data/slurm/secret/jwt_token.txt) +curl -X 'GET' -v 'http://localhost:6820/slurm/v0.0.39/ping' --location --silent --show-error -H "X-SLURM-USER-NAME: root" -H "X-SLURM-USER-TOKEN: $SLURM_JWT" +# curl -v --unix-socket data/slurm/tmp/slurmrestd.socket 'http://localhost:6820/slurm/v0.0.39/ping' \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 51e3354..f10d9ed 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,7 +72,6 @@ services: volumes: - ${DATADIR}/slurm/home:/home - ${DATADIR}/slurm/secret:/.secret - - ${DATADIR}/slurm/tmp:/tmp:rw - ./slurm/controller/slurm.conf:/home/config/slurm.conf - /etc/timezone:/etc/timezone:ro - /etc/localtime:/etc/localtime:ro @@ -92,11 +91,9 @@ services: volumes: - ${DATADIR}/slurm/home:/home - ${DATADIR}/slurm/secret:/.secret - - ${DATADIR}/slurm/tmp:/tmp:rw - ./slurm/database/slurmdbd.conf:/home/config/slurmdbd.conf - /etc/timezone:/etc/timezone:ro - /etc/localtime:/etc/localtime:ro - - ${DATADIR}/slurm/state:/var/lib/slurm/d ports: - "6819:6819" @@ -111,7 +108,6 @@ services: volumes: - ${DATADIR}/slurm/home:/home - ${DATADIR}/slurm/secret:/.secret - - ${DATADIR}/slurm/tmp:/tmp:rw - ./slurm/worker/cgroup.conf:/home/config/cgroup.conf - ./slurm/controller/slurm.conf:/home/config/slurm.conf - /etc/timezone:/etc/timezone:ro @@ -124,16 +120,15 @@ services: hostname: slurmrestd build: context: ./slurm/rest - args: - uid_u: ${UID_U} - gid_g: ${GID_G} + environment: + - SLURM_JWT=daemon + - SLURMRESTD_DEBUG=9 depends_on: - slurmctld privileged: true volumes: - ${DATADIR}/slurm/home:/home - ${DATADIR}/slurm/secret:/.secret - - ${DATADIR}/slurm/tmp:/tmp:rw - ./slurm/controller/slurm.conf:/home/config/slurm.conf - ./slurm/rest/slurmrestd.conf:/home/config/slurmrestd.conf - /etc/timezone:/etc/timezone:ro diff --git a/jwt_verifier.py b/jwt_verifier.py new file mode 100644 index 0000000..e9ec78e --- /dev/null +++ b/jwt_verifier.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +import sys +import os +import pprint +import json +import time +from datetime import datetime, timedelta, timezone + +from jwt import JWT +from jwt.jwa import HS256 +from jwt.jwk import jwk_from_dict +from jwt.utils import b64decode,b64encode + +if len(sys.argv) != 2: + sys.exit("verify_jwt.py [JWT Token]"); + +with open("data/slurm/secret/jwt_hs256.key", "rb") as f: + priv_key = f.read() + +signing_key = jwk_from_dict({ + 'kty': 'oct', + 'k': b64encode(priv_key) +}) + +a = JWT() +b = a.decode(sys.argv[1], signing_key, algorithms=["HS256"]) +print(b) \ No newline at end of file diff --git a/slurm/base/Dockerfile b/slurm/base/Dockerfile index f47588e..ca6b27f 100644 --- a/slurm/base/Dockerfile +++ b/slurm/base/Dockerfile @@ -9,10 +9,10 @@ RUN ARCH=$(uname -m) && yum install -y https://rpmfind.net/linux/almalinux/8.10/ RUN groupadd -g 981 munge \ && useradd -m -c "MUNGE Uid 'N' Gid Emporium" -d /var/lib/munge -u 981 -g munge -s /sbin/nologin munge \ - && groupadd -g 982 slurm \ - && useradd -m -c "Slurm workload manager" -d /var/lib/slurm -u 982 -g slurm -s /bin/bash slurm \ - && groupadd -g 1000 worker \ - && useradd -m -c "Workflow user" -d /home/worker -u 1000 -g worker -s /bin/bash worker + && groupadd -g 1000 slurm \ + && useradd -m -c "Slurm workload manager" -d /var/lib/slurm -u 1000 -g slurm -s /bin/bash slurm \ + && groupadd -g 982 worker \ + && useradd -m -c "Workflow user" -d /home/worker -u 982 -g worker -s /bin/bash worker RUN yum install -y munge munge-libs rng-tools \ python3 gcc openssl openssl-devel \ diff --git a/slurm/controller/docker-entrypoint.sh b/slurm/controller/docker-entrypoint.sh index 6fca56d..72ac3c9 100755 --- a/slurm/controller/docker-entrypoint.sh +++ b/slurm/controller/docker-entrypoint.sh @@ -13,9 +13,8 @@ _delete_secrets() { sudo rm -rf /.secret/munge.key sudo rm -rf /.secret/worker-secret.tar.gz sudo rm -rf /.secret/setup-worker-ssh.sh - sudo rm -rf /.secret/jwt.key - sudo rm -rf /.secret/jwt_public.key - sudo rm -rf /.secret/jwt_token.key + sudo rm -rf /.secret/jwt_hs256.key + sudo rm -rf /.secret/jwt_token.txt echo "Done removing secrets" ls /.secret/ @@ -94,27 +93,48 @@ _copy_secrets() { } _openssl_jwt_key() { - cd /.secret - openssl rand -base64 32 > jwt.key - # openssl genpkey -algorithm RSA -out jwt.key -pkeyopt rsa_keygen_bits:2048 - # openssl rsa -pubout -in jwt.key -out jwt_public.key - cd .. + + mkdir -p /var/spool/slurm/statesave + dd if=/dev/random of=/var/spool/slurm/statesave/jwt_hs256.key bs=32 count=1 + chown slurm:slurm /var/spool/slurm/statesave/jwt_hs256.key + chmod 0600 /var/spool/slurm/statesave/jwt_hs256.key + chown slurm:slurm /var/spool/slurm/statesave + chmod 0755 /var/spool/slurm/statesave + cp /var/spool/slurm/statesave/jwt_hs256.key /.secret/jwt_hs256.key + chmod 777 /.secret/jwt_hs256.key } _generate_jwt_token() { - PEM=$(cat /etc/config/jwt.key) - USER=\"slurm\" - NOW=$(date +%s) - IAT="${NOW}" - EXP=$((${NOW} + 3600000)) - HEADER_RAW='{"alg":"HS256", "typ":"JWT"}' - HEADER=$(echo -n "${HEADER_RAW}" | openssl base64 | tr -d '=' | tr '/+' '_-' | tr -d '\n') - PAYLOAD_RAW='{"iss":'${USER}'}' - PAYLOAD=$(echo -n "${PAYLOAD_RAW}" | openssl base64 | tr -d '=' | tr '/+' '_-' | tr -d '\n') - HEADER_PAYLOAD="${HEADER}"."${PAYLOAD}" - SIGNATURE=$(openssl dgst -sha256 -sign <(echo -n "${PEM}") <(echo -n "${HEADER_PAYLOAD}") | openssl base64 | tr -d '=' | tr '/+' '_-' | tr -d '\n') - JWT="${HEADER_PAYLOAD}"."${SIGNATURE}" - echo $JWT | cat >/.secret/jwt_token.txt + + secret_key=$(cat /var/spool/slurm/statesave/jwt_hs256.key) + start_time=$(date +%s) + exp_time=$((start_time + 100000000)) + base64url() { + # Don't wrap, make URL-safe, delete trailer. + base64 -w 0 | tr '+/' '-_' | tr -d '=' + } + + jwt_header=$(echo -n '{"alg":"HS256","typ":"JWT"}' | base64url) + + jwt_claims=$(cat < Monochrome output, compact output, join lines + + jwt_signature=$(echo -n "${jwt_header}.${jwt_claims}" | + openssl dgst -sha256 -hmac "$secret_key" -binary | base64url) + + # Use the same colours as jwt.io, more-or-less. + echo "$(tput setaf 1)${jwt_header}$(tput sgr0).$(tput setaf 5)${jwt_claims}$(tput sgr0).$(tput setaf 6)${jwt_signature}$(tput sgr0)" + + jwt="${jwt_header}.${jwt_claims}.${jwt_signature}" + + echo $jwt | cat >/.secret/jwt_token.txt chmod 777 /.secret/jwt_token.txt } @@ -162,23 +182,24 @@ _slurmctld() { chmod 600 /etc/slurm/slurm.conf fi - _openssl_jwt_key - - if [ ! -f /.secret/jwt.key ]; then - echo "### Missing jwt.key ###" - exit 1 - else - cp /.secret/jwt.key /etc/config/jwt.key - chown slurm: /etc/config/jwt.key - chmod 0600 /etc/config/jwt.key - fi - - _generate_jwt_token - sudo yum install -y nc sudo yum install -y procps sudo yum install -y iputils sudo yum install -y lsof + sudo yum install -y jq + + _openssl_jwt_key + + if [ ! -f /.secret/jwt_hs256.key ]; then + echo "### Missing jwt.key ###" + exit 1 + else + cp /.secret/jwt_hs256.key /etc/config/jwt_hs256.key + chown slurm: /etc/config/jwt_hs256.key + chmod 0600 /etc/config/jwt_hs256.key + fi + + _generate_jwt_token while ! nc -z slurmdbd 6819; do echo "Waiting for slurmdbd to be ready..." diff --git a/slurm/controller/slurm.conf b/slurm/controller/slurm.conf index 83c9f24..8fbaee4 100644 --- a/slurm/controller/slurm.conf +++ b/slurm/controller/slurm.conf @@ -23,7 +23,7 @@ SlurmctldPidFile=/var/run/slurm/d/slurmctld.pid SlurmdPidFile=/var/run/slurm/d/slurmd.pid ProctrackType=proctrack/linuxproc AuthAltTypes=auth/jwt -AuthAltParameters=jwt_key=/etc/config/jwt.key +AuthAltParameters=jwt_key=/var/spool/slurm/statesave/jwt_hs256.key #PluginDir= #CacheGroups=0 #FirstJobId= @@ -71,9 +71,9 @@ SelectTypeParameters=CR_CPU_Memory #PriorityMaxAge=1-0 # # LOGGING -SlurmctldDebug=3 +SlurmctldDebug=6 SlurmctldLogFile=/var/log/slurm/slurmctld.log -SlurmdDebug=3 +SlurmdDebug=6 SlurmdLogFile=/var/log/slurm/slurmd.log JobCompType=jobcomp/filetxt JobCompLoc=/var/log/slurm/jobcomp.log diff --git a/slurm/database/docker-entrypoint.sh b/slurm/database/docker-entrypoint.sh index c9227ae..62b967d 100755 --- a/slurm/database/docker-entrypoint.sh +++ b/slurm/database/docker-entrypoint.sh @@ -74,14 +74,17 @@ _slurmdbd() { fi echo "checking for jwt.key" - while [ ! -f /.secret/jwt.key ]; do + while [ ! -f /.secret/jwt_hs256.key ]; do echo "." sleep 1 done - cp /.secret/jwt.key /etc/config/jwt.key - chown slurm: /etc/config/jwt.key - chmod 0400 /etc/config/jwt.key + mkdir -p /var/spool/slurm/statesave + chown slurm:slurm /var/spool/slurm/statesave + chmod 0755 /var/spool/slurm/statesave + cp /.secret/jwt_hs256.key /var/spool/slurm/statesave/jwt_hs256.key + chown slurm: /var/spool/slurm/statesave/jwt_hs256.key + chmod 0600 /var/spool/slurm/statesave/jwt_hs256.key echo "" diff --git a/slurm/database/slurmdbd.conf b/slurm/database/slurmdbd.conf index 1be920c..884a988 100644 --- a/slurm/database/slurmdbd.conf +++ b/slurm/database/slurmdbd.conf @@ -15,7 +15,7 @@ AuthType=auth/munge #AuthInfo=/var/run/munge/munge.socket.2 AuthAltTypes=auth/jwt -AuthAltParameters=jwt_key=/etc/config/jwt.key +AuthAltParameters=jwt_key=/var/spool/slurm/statesave/jwt_hs256.key # slurmDBD info DbdAddr=slurmdbd DbdHost=slurmdbd diff --git a/slurm/rest/Dockerfile b/slurm/rest/Dockerfile index 664921d..a111d6b 100644 --- a/slurm/rest/Dockerfile +++ b/slurm/rest/Dockerfile @@ -1,15 +1,10 @@ FROM clustercockpit/slurm.base:24.05.3 LABEL org.opencontainers.image.authors="jan.eitzinger@fau.de" -ARG uid_u -ARG gid_g -ENV uid_u=${uid_u} -ENV gid_g=${gid_g} - # clean up RUN rm -f /root/rpmbuild/RPMS/slurm-*.rpm \ && yum clean all \ && rm -rf /var/cache/yum COPY docker-entrypoint.sh /docker-entrypoint.sh -ENTRYPOINT /docker-entrypoint.sh $uid_u $gid_g +ENTRYPOINT ["/docker-entrypoint.sh"] diff --git a/slurm/rest/docker-entrypoint.sh b/slurm/rest/docker-entrypoint.sh index 3c0cddd..146ceff 100755 --- a/slurm/rest/docker-entrypoint.sh +++ b/slurm/rest/docker-entrypoint.sh @@ -4,18 +4,8 @@ set -e # Determine the system architecture dynamically ARCH=$(uname -m) SLURM_VERSION="24.05.3" -SLURMRESTD="/tmp/slurmrestd.socket" -# SLURM_JWT=daemon - -uid_u="${1:-}" -gid_g="${2:-}" - -echo Your container args are: "$@" - -# Change the uid -# usermod -u "${uid_u}" slurm -# Change the gid -# groupmod -g "${gid_g}" slurm +# SLURMRESTD="/tmp/slurmrestd.socket" +SLURM_JWT=daemon # start sshd server _sshd_host() { @@ -50,14 +40,6 @@ _munge_start_using_key() { _enable_slurmrestd() { - cd /tmp - mkdir statesave - dd if=/dev/random of=/tmp/statesave/jwt_hs256.key bs=32 count=1 - chown slurm:slurm /tmp/statesave/jwt_hs256.key - chmod 0600 /tmp/statesave/jwt_hs256.key - chown slurm:slurm /tmp/statesave - chmod 0755 /tmp/statesave - cat >/usr/lib/systemd/system/slurmrestd.service <