ClusterCockpit/cc-docker
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-docker.git synced 2025-03-14 19:05:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

Update to nats auto generation script

Browse Source
This commit is contained in:
Aditya Ujeniya 2025-01-29 23:29:41 +01:00
parent 0edb94da94
commit 43ca8e35dd
10 changed files with 152 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
.env
View File

@ -2,15 +2,6 @@
# CCBACKEND DEVEL DOCKER SETTINGS
########################################################################
########################################################################
# SLURM
########################################################################
SLURM_VERSION=22.05.6
ARCH=aarch64
MUNGE_UID=981
SLURM_UID=982
WORKER_UID=1000
########################################################################
# INFLUXDB
########################################################################
@ -22,27 +13,6 @@ INFLUXDB_BUCKET=ClusterCockpit
# Whether or not to check SSL Cert in Symfony Client, Default: false
INFLUXDB_SSL=false
########################################################################
# MARIADB
########################################################################
MARIADB_ROOT_PASSWORD=root
MARIADB_DATABASE=ClusterCockpit
MARIADB_USER=clustercockpit
MARIADB_PASSWORD=clustercockpit
MARIADB_PORT=3306
#########################################
# LDAP
########################################################################
LDAP_ADMIN_PASSWORD=mashup
LDAP_ORGANISATION=NHR@FAU
LDAP_DOMAIN=rrze.uni-erlangen.de
########################################################################
# PHPMyAdmin
########################################################################
PHPMYADMIN_PORT=8081
########################################################################
# INTERNAL SETTINGS
########################################################################

18
cc-metric-store/config.json
View File

@ -180,6 +180,22 @@
"https-key-file": null
},
"retention-in-memory": "48h",
"nats": null,
"nats": [
{
"address": "nats://nats:4222",
"username": "root",
"password": "root",
"subscriptions": [
{
"subscribe-to": "hpc-nats",
"cluster-tag": "fritz"
},
{
"subscribe-to": "hpc-nats",
"cluster-tag": "alex"
}
]
}
],
"jwt-public-key": "kzfYrYy+TzpanWZHJ5qSdMj5uKUWgq74BWhQG6copP0="
}

114
dataGenerationScript.sh Executable file
View File

@ -0,0 +1,114 @@
#!/bin/bash
echo ""
echo "|--------------------------------------------------------------------------------------|"
echo "| This is Data generation script for docker services |"
echo "| Starting file required by docker services in data/ |"
echo "|--------------------------------------------------------------------------------------|"
# Download unedited checkpoint files to ./data/cc-metric-store-source/checkpoints
if [ ! -d data/cc-metric-store-source ]; then
mkdir -p data/cc-metric-store-source/checkpoints
cd data/cc-metric-store-source/checkpoints
wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/cc-metric-store-checkpoints.tar.xz
tar xf cc-metric-store-checkpoints.tar.xz
rm cc-metric-store-checkpoints.tar.xz
cd ../../../
else
echo "'data/cc-metric-store-source' already exists!"
fi
if [ ! -d data/mariadb ]; then
mkdir -p data/mariadb
cat > data/mariadb/01.databases.sql <<EOF
CREATE DATABASE IF NOT EXISTS \`ccbackend\`;
EOF
else
echo "'data/mariadb' already exists!"
fi
if [ ! -d data/ldap ]; then
mkdir -p data/ldap
cat > data/ldap/add_users.ldif <<EOF
dn: ou=users,dc=example,dc=com
objectClass: organizationalUnit
ou: users
dn: uid=ldapuser,ou=users,dc=example,dc=com
objectClass: inetOrgPerson
objectClass: posixAccount
objectClass: top
cn: Ldap User
sn: User
uid: ldapuser
uidNumber: 1
gidNumber: 1
homeDirectory: /home/ldapuser
userPassword: {SSHA}sQRqFQtuiupej7J/rbrQrTwYEHDduV+N
EOF
else
echo "'data/ldap' already exists!"
fi
if [ ! -d data/nats ]; then
mkdir -p data/nats
cat > data/nats/docker-entrypoint.sh <<EOF
#!/bin/sh
set -e
# Start NATS server in the background
nats-server --user root --pass root --http_port 8222 &
# Wait for NATS to be ready
until nc -z 0.0.0.0 4222; do
echo "Waiting for NATS to start..."
sleep 1
done
echo "NATS is up and running. Executing custom script..."
apk add curl
curl -sf https://binaries.nats.dev/nats-io/natscli/nats@latest | sh
# Run your custom script
while true; do
timestamp="\$(date '+%s')"
for metric in cpu_irq cpu_load mem_cached net_bytes_in cpu_user cpu_idle nfs4_read mem_used nfs4_write nfs4_total ib_xmit ib_xmit_pkts net_bytes_out cpu_iowait ib_recv cpu_system ib_recv_pkts; do
for hostname in a0603 a0903 a0832 a0329 a0702 a0122 a1624 a0731 a0224 a0704 a0631 a0225 a0222 a0427 a0603 a0429 a0833 a0705 a0901 a0601 a0227 a0804 a0322 a0226 a0126 a0129 a0605 a0801 a0934; do
echo "\$metric,cluster=alex,hostname=\$hostname,type=node value=$((1 + RANDOM % 100)).0 \$timestamp" >sample_alex.txt
done
done
./nats pub hpc-nats "\$(cat sample_alex.txt)" -s nats://0.0.0.0:4222 --user root --password root
for metric in cpu_irq cpu_load mem_cached net_bytes_in cpu_user cpu_idle nfs4_read mem_used nfs4_write nfs4_total ib_xmit ib_xmit_pkts net_bytes_out cpu_iowait ib_recv cpu_system ib_recv_pkts; do
for hostname in f0201 f0202 f0203 f0204 f0205 f0206 f0207 f0208 f0209 f0210 f0211 f0212 f0213 f0214 f0215 f0217 f0218 f0219 f0220 f0221 f0222 f0223 f0224 f0225 f0226 f0227 f0228 f0229; do
echo "\$metric,cluster=fritz,hostname=\$hostname,type=node value=$((1 + RANDOM % 100)).0 \$timestamp" >sample_fritz.txt
done
done
./nats pub hpc-nats "\$(cat sample_fritz.txt)" -s nats://0.0.0.0:4222 --user root --password root
sleep 1m
done
EOF
else
echo "'data/nats' already exists!"
fi
# prepare folders for influxdb2
if [ ! -d data/influxdb ]; then
mkdir -p data/influxdb/data
mkdir -p data/influxdb/config
else
echo "'data/influxdb' already exists!"
fi
echo ""
echo "|--------------------------------------------------------------------------------------|"
echo "| Finished generating relevant files for docker services in data/ |"
echo "|--------------------------------------------------------------------------------------|"

14
docker-compose.yml
View File

@ -3,15 +3,19 @@ services:
container_name: nats
image: nats:alpine
ports:
- "4222:4222"
- "8222:8222"
- "0.0.0.0:4222:4222"
- "0.0.0.0:8222:8222"
- "0.0.0.0:6222:6222"
volumes:
- ${DATADIR}/nats:/data
entrypoint: ["/bin/sh", "/data/docker-entrypoint.sh"]
cc-metric-store:
container_name: cc-metric-store
build:
context: ./cc-metric-store
ports:
- "8084:8084"
- "0.0.0.0:8084:8084"
volumes:
- ${DATADIR}/cc-metric-store:/data
depends_on:
@ -30,7 +34,7 @@ services:
DOCKER_INFLUXDB_INIT_RETENTION: 100w
DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: ${INFLUXDB_ADMIN_TOKEN}
ports:
- "127.0.0.1:${INFLUXDB_PORT}:8086"
- "0.0.0.0:8086:8086"
volumes:
- ${DATADIR}/influxdb/data:/var/lib/influxdb2
- ${DATADIR}/influxdb/config:/etc/influxdb2
@ -62,7 +66,7 @@ services:
MARIADB_USER: slurm
MARIADB_PASSWORD: demo
ports:
- "0.0.0.0:${MARIADB_PORT}:3306"
- "0.0.0.0:3306:3306"
volumes:
- ${DATADIR}/mariadb:/docker-entrypoint-initdb.d
cap_add:

5
env-template.txt
View File

@ -1,5 +0,0 @@
SLURM_VERSION=22.05.6
ARCH=aarch64
MUNGE_UID=981
SLURM_UID=982
WORKER_UID=1000

0
config.json → misc/config.json
View File

0
curl_slurmrestd.sh → misc/curl_slurmrestd.sh
View File

0
jwt_verifier.py → misc/jwt_verifier.py
View File

0
prerequisite_installation_script.sh → scripts/prerequisite_installation_script.sh
View File

85
setupDev.sh
View File

@ -16,9 +16,6 @@ echo "| 'sudo' keyword.
echo "|--------------------------------------------------------------------------------------|"
echo ""
export UID_U=$(id -u $USER)
export GID_G=$(id -g $USER)
# Check cc-backend, touch job.db if exists
if [ ! -d cc-backend ]; then
echo "'cc-backend' not yet prepared! Please clone cc-backend repository before starting this script."
@ -32,35 +29,28 @@ else
rm ./job-archive-demo.tar
cp ./configs/env-template.txt .env
cp ./configs/config-demo.json config.json
cp -f ../misc/config.json config.json
make
./cc-backend -migrate-db
./cc-backend --init-db --add-user demo:admin:AdminDev
./cc-backend --init-db --add-user demo:admin:demo
cd ..
else
cd ..
# echo "'cc-backend/var' exists. Cautiously exiting."
# echo -n "Stopped."
# exit
echo "'cc-backend/var' exists. Cautiously exiting."
echo -n "Stopped."
exit
fi
fi
mkdir -m777 data
# Download unedited checkpoint files to ./data/cc-metric-store-source/checkpoints
if [ ! -d data/cc-metric-store-source ]; then
mkdir -p data/cc-metric-store-source/checkpoints
cd data/cc-metric-store-source/checkpoints
wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/cc-metric-store-checkpoints.tar.xz
tar xf cc-metric-store-checkpoints.tar.xz
rm cc-metric-store-checkpoints.tar.xz
cd ../../../
else
echo "'data/cc-metric-store-source' already exists!"
if [ ! -d data ]; then
mkdir -m777 data
fi
chmod u+x dataGenerationScript.sh
./dataGenerationScript.sh
# Update timestamps
perl ./migrateTimestamps.pl
@ -70,57 +60,8 @@ if [ ! -d data/cc-metric-store/archive ]; then
fi
# cleanup sources
# rm -r ./data/job-archive-source
# rm -r ./data/cc-metric-store-source
if [ ! -d data/mariadb ]; then
mkdir -p data/mariadb
cat > data/mariadb/01.databases.sql <<EOF
CREATE DATABASE IF NOT EXISTS \`ccbackend\`;
EOF
else
echo "'data/mariadb' already exists!"
fi
if [ ! -d data/ldap ]; then
mkdir -p data/ldap
cat > data/ldap/add_users.ldif <<EOF
dn: ou=users,dc=example,dc=com
objectClass: organizationalUnit
ou: users
dn: uid=ldapuser,ou=users,dc=example,dc=com
objectClass: inetOrgPerson
objectClass: posixAccount
objectClass: top
cn: Ldap User
sn: User
uid: ldapuser
uidNumber: 1
gidNumber: 1
homeDirectory: /home/ldapuser
userPassword: {SSHA}sQRqFQtuiupej7J/rbrQrTwYEHDduV+N
EOF
else
echo "'data/ldap' already exists!"
fi
# prepare folders for influxdb2
if [ ! -d data/influxdb ]; then
mkdir -p data/influxdb/data
mkdir -p data/influxdb/config
else
echo "'data/influxdb' already exists!"
fi
# Check dotenv-file and docker-compose-yml, copy accordingly if not present and build docker services
if [ ! -d .env ]; then
cp templates/env.default ./.env
fi
if [ ! -f docker-compose.yml ]; then
cp templates/docker-compose.yml.default ./docker-compose.yml
if [ -d data/cc-metric-store-source ]; then
rm -r data/cc-metric-store-source
fi
docker-compose down
@ -133,8 +74,6 @@ cd ../..
docker-compose build
docker-compose up -d
cp -f config.json cc-backend/config.json
echo ""
echo "|--------------------------------------------------------------------------------------|"
echo "| Check logs for each slurm service by using these commands: |"