diff --git a/.env b/.env index 4e9aa63..04f530a 100644 --- a/.env +++ b/.env @@ -2,15 +2,6 @@ # CCBACKEND DEVEL DOCKER SETTINGS ######################################################################## -######################################################################## -# SLURM -######################################################################## -SLURM_VERSION=22.05.6 -ARCH=aarch64 -MUNGE_UID=981 -SLURM_UID=982 -WORKER_UID=1000 - ######################################################################## # INFLUXDB ######################################################################## @@ -22,27 +13,6 @@ INFLUXDB_BUCKET=ClusterCockpit # Whether or not to check SSL Cert in Symfony Client, Default: false INFLUXDB_SSL=false -######################################################################## -# MARIADB -######################################################################## -MARIADB_ROOT_PASSWORD=root -MARIADB_DATABASE=ClusterCockpit -MARIADB_USER=clustercockpit -MARIADB_PASSWORD=clustercockpit -MARIADB_PORT=3306 - -######################################### -# LDAP -######################################################################## -LDAP_ADMIN_PASSWORD=mashup -LDAP_ORGANISATION=NHR@FAU -LDAP_DOMAIN=rrze.uni-erlangen.de - -######################################################################## -# PHPMyAdmin -######################################################################## -PHPMYADMIN_PORT=8081 - ######################################################################## # INTERNAL SETTINGS ######################################################################## diff --git a/cc-metric-store/config.json b/cc-metric-store/config.json index 2dc0b1b..a7173b2 100644 --- a/cc-metric-store/config.json +++ b/cc-metric-store/config.json @@ -180,6 +180,22 @@ "https-key-file": null }, "retention-in-memory": "48h", - "nats": null, + "nats": [ + { + "address": "nats://nats:4222", + "username": "root", + "password": "root", + "subscriptions": [ + { + "subscribe-to": "hpc-nats", + "cluster-tag": "fritz" + }, + { + "subscribe-to": "hpc-nats", + "cluster-tag": "alex" + } + ] + } + ], "jwt-public-key": "kzfYrYy+TzpanWZHJ5qSdMj5uKUWgq74BWhQG6copP0=" } \ No newline at end of file diff --git a/dataGenerationScript.sh b/dataGenerationScript.sh new file mode 100755 index 0000000..b049bce --- /dev/null +++ b/dataGenerationScript.sh @@ -0,0 +1,114 @@ +#!/bin/bash +echo "" +echo "|--------------------------------------------------------------------------------------|" +echo "| This is Data generation script for docker services |" +echo "| Starting file required by docker services in data/ |" +echo "|--------------------------------------------------------------------------------------|" + +# Download unedited checkpoint files to ./data/cc-metric-store-source/checkpoints +if [ ! -d data/cc-metric-store-source ]; then + mkdir -p data/cc-metric-store-source/checkpoints + cd data/cc-metric-store-source/checkpoints + wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/cc-metric-store-checkpoints.tar.xz + tar xf cc-metric-store-checkpoints.tar.xz + rm cc-metric-store-checkpoints.tar.xz + cd ../../../ +else + echo "'data/cc-metric-store-source' already exists!" +fi + +if [ ! -d data/mariadb ]; then + mkdir -p data/mariadb + cat > data/mariadb/01.databases.sql < data/ldap/add_users.ldif < data/nats/docker-entrypoint.sh <sample_alex.txt + done + done + + ./nats pub hpc-nats "\$(cat sample_alex.txt)" -s nats://0.0.0.0:4222 --user root --password root + + for metric in cpu_irq cpu_load mem_cached net_bytes_in cpu_user cpu_idle nfs4_read mem_used nfs4_write nfs4_total ib_xmit ib_xmit_pkts net_bytes_out cpu_iowait ib_recv cpu_system ib_recv_pkts; do + for hostname in f0201 f0202 f0203 f0204 f0205 f0206 f0207 f0208 f0209 f0210 f0211 f0212 f0213 f0214 f0215 f0217 f0218 f0219 f0220 f0221 f0222 f0223 f0224 f0225 f0226 f0227 f0228 f0229; do + echo "\$metric,cluster=fritz,hostname=\$hostname,type=node value=$((1 + RANDOM % 100)).0 \$timestamp" >sample_fritz.txt + done + done + + ./nats pub hpc-nats "\$(cat sample_fritz.txt)" -s nats://0.0.0.0:4222 --user root --password root + + sleep 1m + +done +EOF + +else + echo "'data/nats' already exists!" +fi + +# prepare folders for influxdb2 +if [ ! -d data/influxdb ]; then + mkdir -p data/influxdb/data + mkdir -p data/influxdb/config +else + echo "'data/influxdb' already exists!" +fi + +echo "" +echo "|--------------------------------------------------------------------------------------|" +echo "| Finished generating relevant files for docker services in data/ |" +echo "|--------------------------------------------------------------------------------------|" \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 7227dda..59f5891 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,15 +3,19 @@ services: container_name: nats image: nats:alpine ports: - - "4222:4222" - - "8222:8222" + - "0.0.0.0:4222:4222" + - "0.0.0.0:8222:8222" + - "0.0.0.0:6222:6222" + volumes: + - ${DATADIR}/nats:/data + entrypoint: ["/bin/sh", "/data/docker-entrypoint.sh"] cc-metric-store: container_name: cc-metric-store build: context: ./cc-metric-store ports: - - "8084:8084" + - "0.0.0.0:8084:8084" volumes: - ${DATADIR}/cc-metric-store:/data depends_on: @@ -30,7 +34,7 @@ services: DOCKER_INFLUXDB_INIT_RETENTION: 100w DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: ${INFLUXDB_ADMIN_TOKEN} ports: - - "127.0.0.1:${INFLUXDB_PORT}:8086" + - "0.0.0.0:8086:8086" volumes: - ${DATADIR}/influxdb/data:/var/lib/influxdb2 - ${DATADIR}/influxdb/config:/etc/influxdb2 @@ -62,7 +66,7 @@ services: MARIADB_USER: slurm MARIADB_PASSWORD: demo ports: - - "0.0.0.0:${MARIADB_PORT}:3306" + - "0.0.0.0:3306:3306" volumes: - ${DATADIR}/mariadb:/docker-entrypoint-initdb.d cap_add: diff --git a/env-template.txt b/env-template.txt deleted file mode 100755 index 3bdeb8f..0000000 --- a/env-template.txt +++ /dev/null @@ -1,5 +0,0 @@ -SLURM_VERSION=22.05.6 -ARCH=aarch64 -MUNGE_UID=981 -SLURM_UID=982 -WORKER_UID=1000 diff --git a/config.json b/misc/config.json similarity index 100% rename from config.json rename to misc/config.json diff --git a/curl_slurmrestd.sh b/misc/curl_slurmrestd.sh similarity index 100% rename from curl_slurmrestd.sh rename to misc/curl_slurmrestd.sh diff --git a/jwt_verifier.py b/misc/jwt_verifier.py similarity index 100% rename from jwt_verifier.py rename to misc/jwt_verifier.py diff --git a/prerequisite_installation_script.sh b/scripts/prerequisite_installation_script.sh similarity index 100% rename from prerequisite_installation_script.sh rename to scripts/prerequisite_installation_script.sh diff --git a/setupDev.sh b/setupDev.sh index 616ba89..0408c85 100755 --- a/setupDev.sh +++ b/setupDev.sh @@ -16,9 +16,6 @@ echo "| 'sudo' keyword. echo "|--------------------------------------------------------------------------------------|" echo "" -export UID_U=$(id -u $USER) -export GID_G=$(id -g $USER) - # Check cc-backend, touch job.db if exists if [ ! -d cc-backend ]; then echo "'cc-backend' not yet prepared! Please clone cc-backend repository before starting this script." @@ -32,35 +29,28 @@ else rm ./job-archive-demo.tar cp ./configs/env-template.txt .env - cp ./configs/config-demo.json config.json + cp -f ../misc/config.json config.json make ./cc-backend -migrate-db - ./cc-backend --init-db --add-user demo:admin:AdminDev + ./cc-backend --init-db --add-user demo:admin:demo cd .. else cd .. - # echo "'cc-backend/var' exists. Cautiously exiting." - # echo -n "Stopped." - # exit + echo "'cc-backend/var' exists. Cautiously exiting." + echo -n "Stopped." + exit fi fi -mkdir -m777 data - -# Download unedited checkpoint files to ./data/cc-metric-store-source/checkpoints -if [ ! -d data/cc-metric-store-source ]; then - mkdir -p data/cc-metric-store-source/checkpoints - cd data/cc-metric-store-source/checkpoints - wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/cc-metric-store-checkpoints.tar.xz - tar xf cc-metric-store-checkpoints.tar.xz - rm cc-metric-store-checkpoints.tar.xz - cd ../../../ -else - echo "'data/cc-metric-store-source' already exists!" +if [ ! -d data ]; then + mkdir -m777 data fi +chmod u+x dataGenerationScript.sh +./dataGenerationScript.sh + # Update timestamps perl ./migrateTimestamps.pl @@ -70,57 +60,8 @@ if [ ! -d data/cc-metric-store/archive ]; then fi # cleanup sources -# rm -r ./data/job-archive-source -# rm -r ./data/cc-metric-store-source - -if [ ! -d data/mariadb ]; then - mkdir -p data/mariadb - cat > data/mariadb/01.databases.sql < data/ldap/add_users.ldif <