diff --git a/Makefile b/Makefile index d7abb18..0e19095 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,4 @@ TARGET = ./cc-backend -VAR = ./var -CFG = config.json .env FRONTEND = ./web/frontend VERSION = 1.4.4 GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development') @@ -42,7 +40,7 @@ SVELTE_SRC = $(wildcard $(FRONTEND)/src/*.svelte) \ .NOTPARALLEL: -$(TARGET): $(VAR) $(CFG) $(SVELTE_TARGETS) +$(TARGET): $(SVELTE_TARGETS) $(info ===> BUILD cc-backend) @go build -ldflags=${LD_FLAGS} ./cmd/cc-backend @@ -68,7 +66,7 @@ distclean: @$(MAKE) clean $(info ===> DISTCLEAN) @rm -rf $(FRONTEND)/node_modules - @rm -rf $(VAR) + @rm -rf ./var test: $(info ===> TESTING) @@ -84,14 +82,6 @@ tags: $(VAR): @mkdir -p $(VAR) -config.json: - $(info ===> Initialize config.json file) - @cp configs/config.json config.json - -.env: - $(info ===> Initialize .env file) - @cp configs/env-template.txt .env - $(SVELTE_TARGETS): $(SVELTE_SRC) $(info ===> BUILD frontend) cd web/frontend && npm install && npm run build diff --git a/cmd/cc-backend/init.go b/cmd/cc-backend/init.go index 3ccb67a..dd044ba 100644 --- a/cmd/cc-backend/init.go +++ b/cmd/cc-backend/init.go @@ -28,18 +28,10 @@ SESSION_KEY="67d829bf61dc5f87a73fd814e2c9f629" const configString = ` { + "main": { "addr": "127.0.0.1:8080", - "archive": { - "kind": "file", - "path": "./var/job-archive" - }, - "jwts": { - "max-age": "2000h" - }, - "apiAllowedIPs": [ - "*" - ], - "enable-resampling": { + "short-running-jobs-duration": 300, + "resampling": { "trigger": 30, "resolutions": [ 600, @@ -48,30 +40,49 @@ const configString = ` 60 ] }, - "clusters": [ - { - "name": "name", - "metricDataRepository": { - "kind": "cc-metric-store", - "url": "http://localhost:8082", - "token": "" - }, - "filterRanges": { - "numNodes": { - "from": 1, - "to": 64 - }, - "duration": { - "from": 0, - "to": 86400 - }, - "startTime": { - "from": "2023-01-01T00:00:00Z", - "to": null - } - } - } - ] + "apiAllowedIPs": [ + "*" + ], + "emission-constant": 317 + }, + "cron": { + "commit-job-worker": "2m", + "duration-worker": "5m", + "footprint-worker": "10m" + }, + "archive": { + "kind": "file", + "path": "./var/job-archive" + }, + "auth": { + "jwts": { + "max-age": "2000h" + } + }, + "clusters": [ + { + "name": "name", + "metricDataRepository": { + "kind": "cc-metric-store", + "url": "http://localhost:8082", + "token": "" + }, + "filterRanges": { + "numNodes": { + "from": 1, + "to": 64 + }, + "duration": { + "from": 0, + "to": 86400 + }, + "startTime": { + "from": "2023-01-01T00:00:00Z", + "to": null + } + } + } + ] } ` diff --git a/configs/config-demo.json b/configs/config-demo.json index 7daff78..6f4c053 100644 --- a/configs/config-demo.json +++ b/configs/config-demo.json @@ -4,16 +4,9 @@ "short-running-jobs-duration": 300, "resampling": { "trigger": 30, - "resolutions": [ - 600, - 300, - 120, - 60 - ] + "resolutions": [600, 300, 120, 60] }, - "apiAllowedIPs": [ - "*" - ], + "apiAllowedIPs": ["*"], "emission-constant": 317 }, "cron": { @@ -88,6 +81,6 @@ "directory": "./var/archive" }, "retention-in-memory": "48h" - }, - "ui-file": "./configs/uiConfig.json" -} \ No newline at end of file + } +} + diff --git a/startDemo.sh b/startDemo.sh index af72a80..10ba7f0 100755 --- a/startDemo.sh +++ b/startDemo.sh @@ -7,16 +7,13 @@ if [ -d './var' ]; then ./cc-backend -server -dev else make - wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-dev.tar - tar xf job-archive-dev.tar - rm ./job-archive-dev.tar - - cp ./configs/env-template.txt .env + ./cc-backend --init cp ./configs/config-demo.json config.json - ./cc-backend -migrate-db + wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-demo.tar + tar xf job-archive-demo.tar + rm ./job-archive-demo.tar + ./cc-backend -dev -init-db -add-user demo:admin,api:demo - ./cc-backend -server -dev - fi diff --git a/tools/archive-migration/README.md b/tools/archive-migration/README.md index 0a5b711..0c6cbfc 100644 --- a/tools/archive-migration/README.md +++ b/tools/archive-migration/README.md @@ -15,20 +15,26 @@ The `archive-migration` tool migrates job archives from old schema versions to t ## Schema Transformations ### Exclusive → Shared + Converts the old `exclusive` integer field to the new `shared` string field: + - `0` → `"multi_user"` - `1` → `"none"` - `2` → `"single_user"` ### Missing Fields + Adds fields required by current schema: + - `submitTime`: Defaults to `startTime` if missing - `energy`: Defaults to `0.0` - `requestedMemory`: Defaults to `0` - `shared`: Defaults to `"none"` if still missing after transformation ### Deprecated Fields + Removes fields no longer in schema: + - `mem_used_max`, `flops_any_avg`, `mem_bw_avg` - `load_avg`, `net_bw_avg`, `net_data_vol_total` - `file_bw_avg`, `file_data_vol_total` @@ -36,17 +42,20 @@ Removes fields no longer in schema: ## Usage ### Build + ```bash -cd /Users/jan/prg/cc-backend/tools/archive-migration +cd ./tools/archive-migration go build ``` ### Dry Run (Preview Changes) + ```bash ./archive-migration --archive /path/to/archive --dry-run ``` ### Migrate Archive + ```bash # IMPORTANT: Backup your archive first! cp -r /path/to/archive /path/to/archive-backup @@ -106,6 +115,7 @@ cd ../archive-manager ### Migration Failures If individual jobs fail to migrate: + - Check the error messages for specific files - Examine the failing `meta.json` files manually - Fix invalid JSON or unexpected field types @@ -114,6 +124,7 @@ If individual jobs fail to migrate: ### Performance For large archives: + - Increase `--workers` for more parallelism - Use `--loglevel warn` to reduce log output - Monitor disk I/O if migration is slow @@ -121,6 +132,7 @@ For large archives: ## Technical Details The migration process: + 1. Walks archive directory recursively 2. Finds all `meta.json` files 3. Distributes jobs to worker pool diff --git a/tools/archive-migration/archive-migration b/tools/archive-migration/archive-migration deleted file mode 100755 index cd73728..0000000 Binary files a/tools/archive-migration/archive-migration and /dev/null differ