Merge pull request #440 from ClusterCockpit/dev

Dev

Makefile

@@ -1,6 +1,4 @@
 TARGET = ./cc-backend
-VAR = ./var
-CFG = config.json .env
 FRONTEND = ./web/frontend
 VERSION = 1.4.4
 GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development')
@@ -42,7 +40,7 @@ SVELTE_SRC = $(wildcard $(FRONTEND)/src/*.svelte)                 \
 
 .NOTPARALLEL:
 
-$(TARGET): $(VAR) $(CFG) $(SVELTE_TARGETS)
+$(TARGET): $(SVELTE_TARGETS)
 	$(info ===>  BUILD cc-backend)
 	@go build -ldflags=${LD_FLAGS} ./cmd/cc-backend
 
@@ -68,7 +66,7 @@ distclean:
 	@$(MAKE) clean
 	$(info ===>  DISTCLEAN)
 	@rm -rf $(FRONTEND)/node_modules
-	@rm -rf $(VAR)
+	@rm -rf ./var
 
 test:
 	$(info ===>  TESTING)
@@ -84,14 +82,6 @@ tags:
 $(VAR):
 	@mkdir -p $(VAR)
 
-config.json:
-	$(info ===>  Initialize config.json file)
-	@cp configs/config.json config.json
-
-.env:
-	$(info ===>  Initialize .env file)
-	@cp configs/env-template.txt .env
-
 $(SVELTE_TARGETS): $(SVELTE_SRC)
 	$(info ===>  BUILD frontend)
 	cd web/frontend && npm install && npm run build

cmd/cc-backend/init.go

@@ -28,18 +28,10 @@ SESSION_KEY="67d829bf61dc5f87a73fd814e2c9f629"
 
 const configString = `
 {
+  "main": {
     "addr": "127.0.0.1:8080",
-    "archive": {
-        "kind": "file",
-        "path": "./var/job-archive"
-    },
-    "jwts": {
-        "max-age": "2000h"
-    },
-    "apiAllowedIPs": [
-      "*"
-    ],
-    "enable-resampling": {
+    "short-running-jobs-duration": 300,
+    "resampling": {
       "trigger": 30,
       "resolutions": [
         600,
@@ -48,30 +40,49 @@ const configString = `
         60
       ]
     },
-    "clusters": [
-        {
-            "name": "name",
-            "metricDataRepository": {
-                "kind": "cc-metric-store",
-                "url": "http://localhost:8082",
-                "token": ""
-            },
-            "filterRanges": {
-                "numNodes": {
-                    "from": 1,
-                    "to": 64
-                },
-                "duration": {
-                    "from": 0,
-                    "to": 86400
-                },
-                "startTime": {
-                    "from": "2023-01-01T00:00:00Z",
-                    "to": null
-                }
-            }
-        }
-    ]
+    "apiAllowedIPs": [
+      "*"
+    ],
+    "emission-constant": 317
+  },
+  "cron": {
+    "commit-job-worker": "2m",
+    "duration-worker": "5m",
+    "footprint-worker": "10m"
+  },
+  "archive": {
+    "kind": "file",
+    "path": "./var/job-archive"
+  },
+  "auth": {
+    "jwts": {
+      "max-age": "2000h"
+    }
+  },
+ "clusters": [
+     {
+         "name": "name",
+         "metricDataRepository": {
+             "kind": "cc-metric-store",
+             "url": "http://localhost:8082",
+             "token": ""
+         },
+         "filterRanges": {
+             "numNodes": {
+                 "from": 1,
+                 "to": 64
+             },
+             "duration": {
+                 "from": 0,
+                 "to": 86400
+             },
+             "startTime": {
+                 "from": "2023-01-01T00:00:00Z",
+                 "to": null
+             }
+         }
+     }
+ ]
 }
 `
 

configs/config-demo.json

@@ -4,16 +4,9 @@
     "short-running-jobs-duration": 300,
     "resampling": {
       "trigger": 30,
-      "resolutions": [
-        600,
-        300,
-        120,
-        60
-      ]
+      "resolutions": [600, 300, 120, 60]
     },
-    "apiAllowedIPs": [
-      "*"
-    ],
+    "apiAllowedIPs": ["*"],
     "emission-constant": 317
   },
   "cron": {
@@ -88,6 +81,6 @@
       "directory": "./var/archive"
     },
     "retention-in-memory": "48h"
-  },
-  "ui-file": "./configs/uiConfig.json"
-}
+  }
+}
+

startDemo.sh

@@ -7,16 +7,13 @@ if [ -d './var' ]; then
   ./cc-backend -server -dev
 else
   make
-  wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-dev.tar
-  tar xf job-archive-dev.tar
-  rm ./job-archive-dev.tar
-
-  cp ./configs/env-template.txt .env
+  ./cc-backend --init
   cp ./configs/config-demo.json config.json
 
-  ./cc-backend -migrate-db
+  wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-demo.tar
+  tar xf job-archive-demo.tar
+  rm ./job-archive-demo.tar
+
   ./cc-backend -dev -init-db -add-user demo:admin,api:demo
-
   ./cc-backend -server -dev
-
 fi

tools/archive-migration/README.md

@@ -15,20 +15,26 @@ The `archive-migration` tool migrates job archives from old schema versions to t
 ## Schema Transformations
 
 ### Exclusive → Shared
+
 Converts the old `exclusive` integer field to the new `shared` string field:
+
 - `0` → `"multi_user"`
 - `1` → `"none"`
 - `2` → `"single_user"`
 
 ### Missing Fields
+
 Adds fields required by current schema:
+
 - `submitTime`: Defaults to `startTime` if missing
 - `energy`: Defaults to `0.0`
 - `requestedMemory`: Defaults to `0`
 - `shared`: Defaults to `"none"` if still missing after transformation
 
 ### Deprecated Fields
+
 Removes fields no longer in schema:
+
 - `mem_used_max`, `flops_any_avg`, `mem_bw_avg`
 - `load_avg`, `net_bw_avg`, `net_data_vol_total`
 - `file_bw_avg`, `file_data_vol_total`
@@ -36,17 +42,20 @@ Removes fields no longer in schema:
 ## Usage
 
 ### Build
+
 ```bash
-cd /Users/jan/prg/cc-backend/tools/archive-migration
+cd ./tools/archive-migration
 go build
 ```
 
 ### Dry Run (Preview Changes)
+
 ```bash
 ./archive-migration --archive /path/to/archive --dry-run
 ```
 
 ### Migrate Archive
+
 ```bash
 # IMPORTANT: Backup your archive first!
 cp -r /path/to/archive /path/to/archive-backup
@@ -106,6 +115,7 @@ cd ../archive-manager
 ### Migration Failures
 
 If individual jobs fail to migrate:
+
 - Check the error messages for specific files
 - Examine the failing `meta.json` files manually
 - Fix invalid JSON or unexpected field types
@@ -114,6 +124,7 @@ If individual jobs fail to migrate:
 ### Performance
 
 For large archives:
+
 - Increase `--workers` for more parallelism
 - Use `--loglevel warn` to reduce log output
 - Monitor disk I/O if migration is slow
@@ -121,6 +132,7 @@ For large archives:
 ## Technical Details
 
 The migration process:
+
 1. Walks archive directory recursively
 2. Finds all `meta.json` files
 3. Distributes jobs to worker pool