Update to nats auto generation script

.env

@@ -2,15 +2,6 @@
 # CCBACKEND DEVEL DOCKER SETTINGS
 ########################################################################
 
-########################################################################
-# SLURM
-########################################################################
-SLURM_VERSION=22.05.6
-ARCH=aarch64
-MUNGE_UID=981 
-SLURM_UID=982 
-WORKER_UID=1000
-
 ########################################################################
 # INFLUXDB
 ########################################################################
@@ -22,27 +13,6 @@ INFLUXDB_BUCKET=ClusterCockpit
 # Whether or not to check SSL Cert in Symfony Client, Default: false
 INFLUXDB_SSL=false
 
-########################################################################
-# MARIADB
-########################################################################
-MARIADB_ROOT_PASSWORD=root
-MARIADB_DATABASE=ClusterCockpit
-MARIADB_USER=clustercockpit
-MARIADB_PASSWORD=clustercockpit
-MARIADB_PORT=3306
-
-#########################################
-# LDAP
-########################################################################
-LDAP_ADMIN_PASSWORD=mashup
-LDAP_ORGANISATION=NHR@FAU
-LDAP_DOMAIN=rrze.uni-erlangen.de
-
-########################################################################
-# PHPMyAdmin
-########################################################################
-PHPMYADMIN_PORT=8081
-
 ########################################################################
 # INTERNAL SETTINGS
 ########################################################################

cc-metric-store/config.json

@@ -180,6 +180,22 @@
         "https-key-file": null
     },
     "retention-in-memory": "48h",
-    "nats": null,
+    "nats": [
+        {
+            "address": "nats://nats:4222",
+            "username": "root",
+            "password": "root",
+            "subscriptions": [
+                {
+                    "subscribe-to": "hpc-nats",
+                    "cluster-tag": "fritz"
+                },
+                {
+                    "subscribe-to": "hpc-nats",
+                    "cluster-tag": "alex"
+                }
+            ]
+        }
+    ],
     "jwt-public-key": "kzfYrYy+TzpanWZHJ5qSdMj5uKUWgq74BWhQG6copP0="
 }

dataGenerationScript.sh

@@ -0,0 +1,114 @@
+#!/bin/bash
+echo ""
+echo "|--------------------------------------------------------------------------------------|"
+echo "| This is Data generation script for docker services                                   |"
+echo "| Starting file required by docker services in data/                                   |"
+echo "|--------------------------------------------------------------------------------------|"
+
+# Download unedited checkpoint files to ./data/cc-metric-store-source/checkpoints
+if [ ! -d data/cc-metric-store-source ]; then
+    mkdir -p data/cc-metric-store-source/checkpoints
+    cd data/cc-metric-store-source/checkpoints
+    wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/cc-metric-store-checkpoints.tar.xz
+    tar xf cc-metric-store-checkpoints.tar.xz
+    rm cc-metric-store-checkpoints.tar.xz
+    cd ../../../
+else
+    echo "'data/cc-metric-store-source' already exists!"
+fi
+
+if [ ! -d data/mariadb ]; then
+    mkdir -p data/mariadb
+    cat > data/mariadb/01.databases.sql <<EOF
+CREATE DATABASE IF NOT EXISTS \`ccbackend\`;
+EOF
+else
+    echo "'data/mariadb' already exists!"
+fi
+
+if [ ! -d data/ldap ]; then
+    mkdir -p data/ldap
+    cat > data/ldap/add_users.ldif <<EOF
+dn: ou=users,dc=example,dc=com
+objectClass: organizationalUnit
+ou: users
+
+dn: uid=ldapuser,ou=users,dc=example,dc=com
+objectClass: inetOrgPerson
+objectClass: posixAccount
+objectClass: top
+cn: Ldap User
+sn: User
+uid: ldapuser
+uidNumber: 1
+gidNumber: 1
+homeDirectory: /home/ldapuser
+userPassword: {SSHA}sQRqFQtuiupej7J/rbrQrTwYEHDduV+N
+EOF
+
+else
+    echo "'data/ldap' already exists!"
+fi
+
+if [ ! -d data/nats ]; then
+    mkdir -p data/nats
+    cat > data/nats/docker-entrypoint.sh <<EOF
+#!/bin/sh
+set -e
+
+# Start NATS server in the background
+nats-server --user root --pass root --http_port 8222 &
+
+# Wait for NATS to be ready
+until nc -z 0.0.0.0 4222; do
+    echo "Waiting for NATS to start..."
+    sleep 1
+done
+
+echo "NATS is up and running. Executing custom script..."
+
+apk add curl
+curl -sf https://binaries.nats.dev/nats-io/natscli/nats@latest | sh
+
+# Run your custom script
+while true; do
+
+    timestamp="\$(date '+%s')"
+
+    for metric in cpu_irq cpu_load mem_cached net_bytes_in cpu_user cpu_idle nfs4_read mem_used nfs4_write nfs4_total ib_xmit ib_xmit_pkts net_bytes_out cpu_iowait ib_recv cpu_system ib_recv_pkts; do
+        for hostname in a0603 a0903 a0832 a0329 a0702 a0122 a1624 a0731 a0224 a0704 a0631 a0225 a0222 a0427 a0603 a0429 a0833 a0705 a0901 a0601 a0227 a0804 a0322 a0226 a0126 a0129 a0605 a0801 a0934; do
+            echo "\$metric,cluster=alex,hostname=\$hostname,type=node value=$((1 + RANDOM % 100)).0 \$timestamp" >sample_alex.txt
+        done
+    done
+
+    ./nats pub hpc-nats "\$(cat sample_alex.txt)" -s nats://0.0.0.0:4222 --user root --password root
+
+    for metric in cpu_irq cpu_load mem_cached net_bytes_in cpu_user cpu_idle nfs4_read mem_used nfs4_write nfs4_total ib_xmit ib_xmit_pkts net_bytes_out cpu_iowait ib_recv cpu_system ib_recv_pkts; do
+        for hostname in f0201 f0202 f0203 f0204 f0205 f0206 f0207 f0208 f0209 f0210 f0211 f0212 f0213 f0214 f0215 f0217 f0218 f0219 f0220 f0221 f0222 f0223 f0224 f0225 f0226 f0227 f0228 f0229; do
+            echo "\$metric,cluster=fritz,hostname=\$hostname,type=node value=$((1 + RANDOM % 100)).0 \$timestamp" >sample_fritz.txt
+        done
+    done
+
+    ./nats pub hpc-nats "\$(cat sample_fritz.txt)" -s nats://0.0.0.0:4222 --user root --password root
+
+    sleep 1m
+
+done
+EOF
+
+else
+    echo "'data/nats' already exists!"
+fi
+
+# prepare folders for influxdb2
+if [ ! -d data/influxdb ]; then
+    mkdir -p data/influxdb/data
+    mkdir -p data/influxdb/config
+else
+    echo "'data/influxdb' already exists!"
+fi
+
+echo ""
+echo "|--------------------------------------------------------------------------------------|"
+echo "| Finished generating relevant files for docker services in data/                      |"
+echo "|--------------------------------------------------------------------------------------|"

docker-compose.yml

@@ -3,15 +3,19 @@ services:
     container_name: nats
     image: nats:alpine
     ports:
-      - "4222:4222"
+      - "0.0.0.0:4222:4222"
-      - "8222:8222"
+      - "0.0.0.0:8222:8222"
+      - "0.0.0.0:6222:6222"
+    volumes:
+      - ${DATADIR}/nats:/data
+    entrypoint: ["/bin/sh", "/data/docker-entrypoint.sh"]
 
   cc-metric-store:
     container_name: cc-metric-store
     build:
       context: ./cc-metric-store
     ports:
-      - "8084:8084"
+      - "0.0.0.0:8084:8084"
     volumes:
       - ${DATADIR}/cc-metric-store:/data
     depends_on:
@@ -30,7 +34,7 @@ services:
       DOCKER_INFLUXDB_INIT_RETENTION: 100w
       DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: ${INFLUXDB_ADMIN_TOKEN}
     ports:
-      - "127.0.0.1:${INFLUXDB_PORT}:8086"
+      - "0.0.0.0:8086:8086"
     volumes:
       - ${DATADIR}/influxdb/data:/var/lib/influxdb2
       - ${DATADIR}/influxdb/config:/etc/influxdb2
@@ -62,7 +66,7 @@ services:
       MARIADB_USER: slurm
       MARIADB_PASSWORD: demo
     ports:
-      - "0.0.0.0:${MARIADB_PORT}:3306"
+      - "0.0.0.0:3306:3306"
     volumes:
       - ${DATADIR}/mariadb:/docker-entrypoint-initdb.d
     cap_add:

env-template.txt

@@ -1,5 +0,0 @@
-SLURM_VERSION=22.05.6
-ARCH=aarch64
-MUNGE_UID=981 
-SLURM_UID=982 
-WORKER_UID=1000

setupDev.sh

@@ -16,9 +16,6 @@ echo "| 'sudo' keyword.
 echo "|--------------------------------------------------------------------------------------|"
 echo ""
 
-export UID_U=$(id -u $USER)
-export GID_G=$(id -g $USER)
-
 # Check cc-backend, touch job.db if exists
 if [ ! -d cc-backend ]; then
     echo "'cc-backend' not yet prepared! Please clone cc-backend repository before starting this script."
@@ -32,35 +29,28 @@ else
         rm ./job-archive-demo.tar
 
         cp ./configs/env-template.txt .env
-        cp ./configs/config-demo.json config.json
+        cp -f ../misc/config.json config.json
 
         make
 
         ./cc-backend -migrate-db
-        ./cc-backend --init-db --add-user demo:admin:AdminDev
+        ./cc-backend --init-db --add-user demo:admin:demo
         cd ..
     else
         cd ..
-    #     echo "'cc-backend/var' exists. Cautiously exiting."
+        echo "'cc-backend/var' exists. Cautiously exiting."
-    #     echo -n "Stopped."
+        echo -n "Stopped."
-    #     exit
+        exit
     fi
 fi
 
-mkdir -m777 data
+if [ ! -d data ]; then
-
+    mkdir -m777 data
-# Download unedited checkpoint files to ./data/cc-metric-store-source/checkpoints
-if [ ! -d data/cc-metric-store-source ]; then
-    mkdir -p data/cc-metric-store-source/checkpoints
-    cd data/cc-metric-store-source/checkpoints
-    wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/cc-metric-store-checkpoints.tar.xz
-    tar xf cc-metric-store-checkpoints.tar.xz
-    rm cc-metric-store-checkpoints.tar.xz
-    cd ../../../
-else
-    echo "'data/cc-metric-store-source' already exists!"
 fi
 
+chmod u+x dataGenerationScript.sh
+./dataGenerationScript.sh
+
 # Update timestamps
 perl ./migrateTimestamps.pl
 
@@ -70,57 +60,8 @@ if [ ! -d data/cc-metric-store/archive ]; then
 fi
 
 # cleanup sources
-# rm -r ./data/job-archive-source
+if [ -d data/cc-metric-store-source ]; then
-# rm -r ./data/cc-metric-store-source
+    rm -r data/cc-metric-store-source
-
-if [ ! -d data/mariadb ]; then
-    mkdir -p data/mariadb
-    cat > data/mariadb/01.databases.sql <<EOF
-CREATE DATABASE IF NOT EXISTS \`ccbackend\`;
-EOF
-else
-    echo "'data/mariadb' already exists!"
-fi
-
-if [ ! -d data/ldap ]; then
-    mkdir -p data/ldap
-    cat > data/ldap/add_users.ldif <<EOF
-dn: ou=users,dc=example,dc=com
-objectClass: organizationalUnit
-ou: users
-
-dn: uid=ldapuser,ou=users,dc=example,dc=com
-objectClass: inetOrgPerson
-objectClass: posixAccount
-objectClass: top
-cn: Ldap User
-sn: User
-uid: ldapuser
-uidNumber: 1
-gidNumber: 1
-homeDirectory: /home/ldapuser
-userPassword: {SSHA}sQRqFQtuiupej7J/rbrQrTwYEHDduV+N
-EOF
-
-else
-    echo "'data/ldap' already exists!"
-fi
-
-# prepare folders for influxdb2
-if [ ! -d data/influxdb ]; then
-    mkdir -p data/influxdb/data
-    mkdir -p data/influxdb/config
-else
-    echo "'data/influxdb' already exists!"
-fi
-
-# Check dotenv-file and docker-compose-yml, copy accordingly if not present and build docker services
-if [ ! -d .env ]; then
-    cp templates/env.default ./.env
-fi
-
-if [ ! -f docker-compose.yml ]; then
-    cp templates/docker-compose.yml.default ./docker-compose.yml
 fi
 
 docker-compose down
@@ -133,8 +74,6 @@ cd ../..
 docker-compose build
 docker-compose up -d
 
-cp -f config.json cc-backend/config.json
-
 echo ""
 echo "|--------------------------------------------------------------------------------------|"
 echo "| Check logs for each slurm service by using these commands:                           |"