From 2761102adf3e6b7c38cc1048a0514de07ae6a303 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 9 Jun 2022 11:38:26 +0200 Subject: [PATCH] Remove ccbackend autobuild, remove ccms/influx splitting, add mariadb --- README.md | 22 ++- setupDev.sh | 163 +++++------------- startDev.sh | 3 - templates/docker-compose.yml.ccms | 29 ---- ...ml.influxdb => docker-compose.yml.default} | 18 ++ templates/docker-compose.yml.mariadb | 28 +++ templates/{env.influxdb => env.default} | 4 +- templates/{env.ccms => env.mariadb} | 17 +- templates/env.mysql | 5 +- 9 files changed, 110 insertions(+), 179 deletions(-) delete mode 100644 templates/docker-compose.yml.ccms rename templates/{docker-compose.yml.influxdb => docker-compose.yml.default} (75%) create mode 100644 templates/docker-compose.yml.mariadb rename templates/{env.influxdb => env.default} (87%) rename templates/{env.ccms => env.mariadb} (54%) diff --git a/README.md b/README.md index aaea96d..658159f 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,11 @@ This is a `docker-compose` setup which provides a quickly started environment fo It includes the following containers: * nats (Default) * cc-metric-store (Default) -* influxdb (Optional) +* influxdb (Default) * openldap (Default) -* mysql (Optional, Manual) -* phpmyadmin (Optional, Manual) +* mysql (Optional) +* mariadb (Optional) +* phpmyadmin (Optional) The setup comes with fixture data for a Job archive, cc-metric-store checkpoints, InfluxDB, MySQL, and a LDAP user directory. @@ -23,19 +24,16 @@ The setup comes with fixture data for a Job archive, cc-metric-store checkpoints ## Configuration Templates Located in `./templates` -* `docker-compose.yml.ccms`: Docker-Compose file to setup cc-metric-store metric database and LDAP containers (Default). Used in `setupDev.sh`. -* `docker-compose.yml.influxdb`: Docker-Compose file to setup influxDB metric database and LDAP containers. Used in `setupDev.sh`. +* `docker-compose.yml.default`: Docker-Compose file to setup cc-metric-store, InfluxDB, and LDAP containers (Default). Used in `setupDev.sh`. * `docker-compose.yml.mysql`: Docker-Compose configuration template if additional MySQL and phpmyadmin containers are desired. -* `env.ccms`: Environment variables for setup with cc-metric-store metric database and LDAP containers (Default). Used in `setupDev.sh`. -* `env.influxdb`: Environment variables for setup with influxDB metric database and LDAP containers. Used in `setupDev.sh`. +* `docker-compose.yml.mariadb`: Docker-Compose configuration template if additional MariaDB and phpmyadmin containers are desired. +* `env.default`: Environment variables for setup with cc-metric-store, InfluxDB, and LDAP containers (Default). Used in `setupDev.sh`. * `env.mysql`: Additional environment variables required if additional MySQL and phpmyadmin containers are desired. +* `env.mariadb`: Additional environment variables required if additional MariaDB and phpmyadmin containers are desired. ## Setup -1. `$ ./setupDev.sh [help|ccms|influxdb]`: **NOTICE** The script will download files of a total size of 338MB (mostly for the InfluxDB data). - * `help`: Displays help. - * `ccms`: Copies according docker-compose.yml and env-file to root directory, downloads cc-metric-store checkpoint data, and builds containers. - * `influxdb`: Copies according docker-compose.yml and env-file to root directory, downloads influxDB data, and builds containers. +1. `$ ./setupDev.sh`: **NOTICE** The script will download files of a total size of 338MB (mostly for the InfluxDB data). 2. After that from the root of the cc-docker sandbox you can start up the containers and launch cc-backend with: `$ ./startDev.sh` @@ -43,7 +41,7 @@ Located in `./templates` ## Post-Setup Adjustment for using `influxdb` -When using `influxdb` containers, one must adjust the following files: +When using `influxdb` as a metric database, one must adjust the following files: * `cc-backend/var/job-archive/emmy/cluster.json` * `cc-backend/var/job-archive/woody/cluster.json` diff --git a/setupDev.sh b/setupDev.sh index fbf4dc9..854b625 100755 --- a/setupDev.sh +++ b/setupDev.sh @@ -1,152 +1,71 @@ #!/bin/bash -if [ -z "$1" ]; then - echo "No argument supplied. Use 'help', 'ccms' (Default), or 'influxdb'." - exit -elif [ "$1" == "help" ]; then - echo "Script to setup cc-backend devel environment. Use 'help', 'ccms' (Default), or 'influxdb' as argument." - echo "'help' : This help." - echo "'ccms' : Setup cc-metric-store example data and build docker container." - echo "'influxdb' : Setup influxdb example data and build docker container. Requires additional configuration afterwards." - exit -else - echo "Starting setup for '$1' ..." -fi - # Download data for influxdb2 -if [ "$1" == "influxdb" ]; then - if [ ! -d data/influxdb ]; then +if [ ! -d data/influxdb ]; then + mkdir -p data/influxdb/data + cd data/influxdb/data + wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/influxdbv2-data.tar.xz + tar xJf influxdbv2-data.tar.xz + rm influxdbv2-data.tar.xz + cd ../../../ +else + echo "'data/influxdb' already exists!" + echo -n "Remove existing folder and redownload? [yes to redownload / no to continue] " + read -r answer + if [ "$answer" == "yes" ]; then + echo "Removing 'data/influxdb' ..." + rm -rf data/influxdb + echo "Reinstall 'data/influxdb'..." mkdir -p data/influxdb/data cd data/influxdb/data wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/influxdbv2-data.tar.xz tar xJf influxdbv2-data.tar.xz rm influxdbv2-data.tar.xz cd ../../../ + echo "done." else - echo "'data/influxdb' already exists!" - echo -n "Remove existing folder and redownload? [yes to redownload / no to continue] " - read -r answer - if [ "$answer" == "yes" ]; then - echo "Removing 'data/influxdb' ..." - rm -rf data/influxdb - echo "Reinstall 'data/influxdb'..." - mkdir -p data/influxdb/data - cd data/influxdb/data - wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/influxdbv2-data.tar.xz - tar xJf influxdbv2-data.tar.xz - rm influxdbv2-data.tar.xz - cd ../../../ - echo "done." - else - echo "'data/influxdb' unchanged." - fi + echo "'data/influxdb' unchanged." fi fi # Download checkpoint files for cc-metric-store -if [ "$1" == "ccms" ]; then - if [ ! -d data/cc-metric-store ]; then - mkdir -p data/cc-metric-store/checkpoints - mkdir -p data/cc-metric-store/archive - cd data/cc-metric-store/checkpoints - wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/cc-metric-store-checkpoints.tar.xz - tar xf cc-metric-store-checkpoints.tar.xz - rm cc-metric-store-checkpoints.tar.xz - cd ../../../ - else - echo "'data/cc-metric-store' already exists!" - echo -n "Remove existing folder and redownload? [yes to redownload / no to continue] " - read -r answer - if [ "$answer" == "yes" ]; then - echo "Removing 'data/cc-metric-store' ..." - rm -rf data/cc-metric-store - echo "Reinstall 'data/cc-metric-store'..." - mkdir -p data/cc-metric-store/checkpoints - mkdir -p data/cc-metric-store/archive - cd data/cc-metric-store/checkpoints - wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/cc-metric-store-checkpoints.tar.xz - tar xf cc-metric-store-checkpoints.tar.xz - rm cc-metric-store-checkpoints.tar.xz - cd ../../../ - echo "done." - else - echo "'data/cc-metric-store' unchanged." - fi - fi -fi - -# Download && Setup cc-backend -if [ ! -d cc-backend ]; then - ## Get backend git [can use --recursive to load frontend via ssh directly] - git clone https://github.com/ClusterCockpit/cc-backend.git - cd cc-backend - ## Get frontend git [http variant] - cd frontend - ### Comment ths if --recursive - git clone https://github.com/ClusterCockpit/cc-frontend.git . - yarn install - yarn build - cd .. - ## Download Demo-Archive and prepare SQLite DB - mkdir ./var - cd ./var - wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive.tar.xz - tar xJf job-archive.tar.xz - rm ./job-archive.tar.xz - touch ./job.db - cd .. - ## Install backend - go get - go build - ## initialize job archive and SQLite - ./cc-backend --init-db --add-user demo:admin:AdminDev --no-server - cd .. +if [ ! -d data/cc-metric-store ]; then + mkdir -p data/cc-metric-store/checkpoints + mkdir -p data/cc-metric-store/archive + cd data/cc-metric-store/checkpoints + wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/cc-metric-store-checkpoints.tar.xz + tar xf cc-metric-store-checkpoints.tar.xz + rm cc-metric-store-checkpoints.tar.xz + cd ../../../ else - echo "'cc-backend' already exists!" - echo -n "Remove existing folder and reinstall? [yes to reinstall / no to continue] " + echo "'data/cc-metric-store' already exists!" + echo -n "Remove existing folder and redownload? [yes to redownload / no to continue] " read -r answer if [ "$answer" == "yes" ]; then - echo "Removing 'cc-backend' ..." - rm -rf cc-backend - echo "Reinstall 'cc-backend'..." - ## Get backend git [can use --recursive to load frontend via ssh directly] - git clone https://github.com/ClusterCockpit/cc-backend.git - cd cc-backend - ## Get frontend git [http variant] - cd frontend - ### Comment ths if --recursive - git clone https://github.com/ClusterCockpit/cc-frontend.git . - yarn install - yarn build - cd .. - ## Download Demo-Archive and prepare SQLite DB - mkdir ./var - cd ./var - wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive.tar.xz - tar xJf job-archive.tar.xz - rm ./job-archive.tar.xz - touch ./job.db - cd .. - ## Install backend - go get - go build - ## initialize job archive and SQLite - ./cc-backend --init-db --add-user demo:admin:AdminDev --no-server - cd .. + echo "Removing 'data/cc-metric-store' ..." + rm -rf data/cc-metric-store + echo "Reinstall 'data/cc-metric-store'..." + mkdir -p data/cc-metric-store/checkpoints + mkdir -p data/cc-metric-store/archive + cd data/cc-metric-store/checkpoints + wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/cc-metric-store-checkpoints.tar.xz + tar xf cc-metric-store-checkpoints.tar.xz + rm cc-metric-store-checkpoints.tar.xz + cd ../../../ + echo "done." else - echo "'cc-backend' unchanged." + echo "'data/cc-metric-store' unchanged." fi fi - # Check dotenv-file and docker-compose-yml, copy accordingly if not present and build docker services # !! By default, this decides which metric database is used based on the selected argument !! if [ ! -d .env ]; then - cp templates/env.$1 ./.env + cp templates/env.default ./.env fi if [ ! -d docker-compose.yml ]; then - cp templates/docker-compose.yml.$1 ./docker-compose.yml + cp templates/docker-compose.yml.default ./docker-compose.yml fi docker-compose build diff --git a/startDev.sh b/startDev.sh index 0af5804..217a5fe 100644 --- a/startDev.sh +++ b/startDev.sh @@ -1,6 +1,3 @@ #!/bin/bash docker-compose up -d - -cd cc-backend -./cc-backend diff --git a/templates/docker-compose.yml.ccms b/templates/docker-compose.yml.ccms deleted file mode 100644 index fb8c96e..0000000 --- a/templates/docker-compose.yml.ccms +++ /dev/null @@ -1,29 +0,0 @@ -services: - nats: - container_name: cc-nats - image: nats:alpine - ports: - - "4222:4222" - - "8222:8222" - - cc-metric-store: - container_name: cc-metric-store - build: - context: ./cc-metric-store - ports: - - "8084:8084" - volumes: - - ${DATADIR}/cc-metric-store:/data - depends_on: - - nats - - openldap: - container_name: cc-ldap - image: osixia/openldap:1.5.0 - command: --copy-service --loglevel debug - environment: - - LDAP_ADMIN_PASSWORD=${LDAP_ADMIN_PASSWORD} - - LDAP_ORGANISATION=${LDAP_ORGANISATION} - - LDAP_DOMAIN=${LDAP_DOMAIN} - volumes: - - ${DATADIR}/ldap:/container/service/slapd/assets/config/bootstrap/ldif/custom diff --git a/templates/docker-compose.yml.influxdb b/templates/docker-compose.yml.default similarity index 75% rename from templates/docker-compose.yml.influxdb rename to templates/docker-compose.yml.default index a71e5e1..c341252 100644 --- a/templates/docker-compose.yml.influxdb +++ b/templates/docker-compose.yml.default @@ -1,4 +1,22 @@ services: + nats: + container_name: cc-nats + image: nats:alpine + ports: + - "4222:4222" + - "8222:8222" + + cc-metric-store: + container_name: cc-metric-store + build: + context: ./cc-metric-store + ports: + - "8084:8084" + volumes: + - ${DATADIR}/cc-metric-store:/data + depends_on: + - nats + influxdb: container_name: cc-influxdb image: influxdb diff --git a/templates/docker-compose.yml.mariadb b/templates/docker-compose.yml.mariadb new file mode 100644 index 0000000..7de8052 --- /dev/null +++ b/templates/docker-compose.yml.mariadb @@ -0,0 +1,28 @@ +services: + db: + container_name: cc-db + image: mariadb:latest + command: ["--default-authentication-plugin=mysql_native_password"] + environment: + MARIADB_ROOT_PASSWORD: ${MARIADB_ROOT_PASSWORD} + MARIADB_DATABASE: ${MARIADB_DATABASE} + MARIADB_USER: ${MARIADB_USER} + MARIADB_PASSWORD: ${MARIADB_PASSWORD} + ports: + - "127.0.0.1:3336:3306" + volumes: + - ${DATADIR}/sql-init:/docker-entrypoint-initdb.d + cap_add: + - SYS_NICE + + phpmyadmin: + container_name: cc-phpmyadmin + image: phpmyadmin + environment: + - PMA_HOST=cc-db + - PMA_USER=root + - PMA_PASSWORD=${MARIADB_ROOT_PASSWORD} + ports: + - "127.0.0.1:${PHPMYADMIN_PORT}:80" + depends_on: + - db diff --git a/templates/env.influxdb b/templates/env.default similarity index 87% rename from templates/env.influxdb rename to templates/env.default index 4ab3171..8f71d0c 100644 --- a/templates/env.influxdb +++ b/templates/env.default @@ -1,5 +1,5 @@ ######################################################################## -# CCBACKEND DEVEL DOCKER SETTINGS FOR INFLUXDB AND LDAP +# CCBACKEND DEVEL DOCKER SETTINGS ######################################################################## ######################################################################## @@ -13,7 +13,7 @@ INFLUXDB_BUCKET=ClusterCockpit # Whether or not to check SSL Cert in Symfony Client, Default: false INFLUXDB_SSL=false -######################################################################## +######################################### # LDAP ######################################################################## LDAP_ADMIN_PASSWORD=mashup diff --git a/templates/env.ccms b/templates/env.mariadb similarity index 54% rename from templates/env.ccms rename to templates/env.mariadb index 4b43034..ff9b12f 100644 --- a/templates/env.ccms +++ b/templates/env.mariadb @@ -1,15 +1,16 @@ ######################################################################## -# CCBACKEND DEVEL DOCKER SETTINGS FOR LDAP (CCMS-Devel) +# ADDITIONAL ENV VARIABLES FOR MARIABD AND PHPMYADMIN CONTAINERS ######################################################################## -######################################### -# LDAP ######################################################################## -LDAP_ADMIN_PASSWORD=mashup -LDAP_ORGANISATION=NHR@FAU -LDAP_DOMAIN=rrze.uni-erlangen.de +# MARIADB +######################################################################## +MARIADB_ROOT_PASSWORD=root +MARIADB_DATABASE=ClusterCockpit +MARIADB_USER=clustercockpit +MARIADB_PASSWORD=clustercockpit ######################################################################## -# INTERNAL SETTINGS +# PHPMyAdmin ######################################################################## -DATADIR=./data +PHPMYADMIN_PORT=8081 diff --git a/templates/env.mysql b/templates/env.mysql index 0f87bc4..fdccd44 100644 --- a/templates/env.mysql +++ b/templates/env.mysql @@ -4,12 +4,11 @@ ######################################################################## # MySQL -# The SQL port is not exposed outside the container ######################################################################## MYSQL_ROOT_PASSWORD=root MYSQL_DATABASE=ClusterCockpit -MYSQL_USER=symfony -MYSQL_PASSWORD=symfony +MYSQL_USER=clustercockpit +MYSQL_PASSWORD=clustercockpit ######################################################################## # PHPMyAdmin