diff --git a/cmd/cc-backend/init.go b/cmd/cc-backend/init.go index e11576b1..7a8f08be 100644 --- a/cmd/cc-backend/init.go +++ b/cmd/cc-backend/init.go @@ -47,6 +47,14 @@ const configString = ` "public-key": "kzfYrYy+TzpanWZHJ5qSdMj5uKUWgq74BWhQG6copP0=", "private-key": "dtPC/6dWJFKZK7KZ78CvWuynylOmjBFyMsUWArwmodOTN9itjL5POlqdZkcnmpJ0yPm4pRaCrvgFaFAbpyik/Q==" } + }, + "metric-store": { + "retention-in-memory": "48h", + "memory-cap": 8, + "checkpoints": { + "file-format": "wal", + "directory": "./var/checkpoints" + } } } ` diff --git a/configs/cluster.json b/configs/cluster.json index 1e15be6d..643e8685 100644 --- a/configs/cluster.json +++ b/configs/cluster.json @@ -1,15 +1,215 @@ { + "name": "alex", + "metricConfig": [ + { + "name": "cpu_load", + "unit": { + "base": "" + }, + "scope": "node", + "aggregation": "avg", + "footprint": "avg", + "timestep": 60, + "peak": 128, + "normal": 128, + "caution": 10, + "alert": 5 + }, + { + "name": "cpu_user", + "unit": { + "base": "" + }, + "scope": "hwthread", + "aggregation": "avg", + "timestep": 60, + "peak": 100, + "normal": 50, + "caution": 20, + "alert": 10 + }, + { + "name": "mem_used", + "unit": { + "base": "B", + "prefix": "G" + }, + "scope": "node", + "aggregation": "sum", + "footprint": "max", + "lowerIsBetter": true, + "timestep": 60, + "peak": 512, + "normal": 128, + "caution": 200, + "alert": 240 + }, + { + "name": "flops_any", + "unit": { + "base": "F/s", + "prefix": "G" + }, + "scope": "hwthread", + "aggregation": "sum", + "footprint": "avg", + "timestep": 60, + "peak": 9216, + "normal": 1000, + "caution": 200, + "alert": 50 + }, + { + "name": "mem_bw", + "unit": { + "base": "B/s", + "prefix": "G" + }, + "scope": "socket", + "aggregation": "sum", + "footprint": "avg", + "timestep": 60, + "peak": 350, + "normal": 100, + "caution": 50, + "alert": 10 + }, + { + "name": "clock", + "unit": { + "base": "Hz", + "prefix": "M" + }, + "scope": "hwthread", + "aggregation": "avg", + "timestep": 60, + "peak": 3000, + "normal": 2400, + "caution": 1800, + "alert": 1200 + }, + { + "name": "core_power", + "unit": { + "base": "W" + }, + "scope": "hwthread", + "aggregation": "sum", + "energy": "power", + "timestep": 60, + "peak": 500, + "normal": 250, + "caution": 100, + "alert": 50 + }, + { + "name": "cpu_power", + "unit": { + "base": "W" + }, + "scope": "socket", + "aggregation": "sum", + "energy": "power", + "timestep": 60, + "peak": 500, + "normal": 250, + "caution": 100, + "alert": 50 + }, + { + "name": "ipc", + "unit": { + "base": "IPC" + }, + "scope": "hwthread", + "aggregation": "avg", + "timestep": 60, + "peak": 4, + "normal": 2, + "caution": 1, + "alert": 0.5 + }, + { + "name": "acc_utilization", + "unit": { + "base": "" + }, + "scope": "accelerator", + "aggregation": "avg", + "footprint": "avg", + "timestep": 60, + "peak": 100, + "normal": 80, + "caution": 50, + "alert": 20 + }, + { + "name": "acc_mem_used", + "unit": { + "base": "B", + "prefix": "G" + }, + "scope": "accelerator", + "aggregation": "sum", + "timestep": 60, + "peak": 48, + "normal": 24, + "caution": 10, + "alert": 5, + "subClusters": [ + { + "name": "a100", + "peak": 40, + "normal": 20, + "caution": 10, + "alert": 5 + } + ] + }, + { + "name": "acc_power", + "unit": { + "base": "W" + }, + "scope": "accelerator", + "aggregation": "sum", + "energy": "power", + "timestep": 60, + "peak": 400, + "normal": 200, + "caution": 50, + "alert": 20 + } + ], "subClusters": [ { "name": "a40", - "numberOfNodes": 38, + "nodes": "a[0121-0129],a[0221-0229],a[0321-0329],a[0421-0429]", "processorType": "AMD Milan", "socketsPerNode": 2, "coresPerSocket": 64, "threadsPerCore": 1, - "flopRateScalar": 432, - "flopRateSimd": 9216, - "memoryBandwidth": 400, + "flopRateScalar": { + "unit": { + "base": "F/s", + "prefix": "G" + }, + "value": 432 + }, + "flopRateSimd": { + "unit": { + "base": "F/s", + "prefix": "G" + }, + "value": 9216 + }, + "memoryBandwidth": { + "unit": { + "base": "B/s", + "prefix": "G" + }, + "value": 400 + }, "topology": { "node": [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127], "socket": [ @@ -19,9 +219,7 @@ "memoryDomain": [ [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127] ], - "core": [ - [0],[1],[2],[3],[4],[5],[6],[7],[8],[9],[10],[11],[12],[13],[14],[15],[16],[17],[18],[19],[20],[21],[22],[23],[24],[25],[26],[27],[28],[29],[30],[31],[32],[33],[34],[35],[36],[37],[38],[39],[40],[41],[42],[43],[44],[45],[46],[47],[48],[49],[50],[51],[52],[53],[54],[55],[56],[57],[58],[59],[60],[61],[62],[63],[64],[65],[66],[67],[68],[69],[70],[71],[73],[74],[75],[76],[77],[78],[79],[80],[81],[82],[83],[84],[85],[86],[87],[88],[89],[90],[91],[92],[93],[94],[95],[96],[97],[98],[99],[100],[101],[102],[103],[104],[105],[106],[107],[108],[109],[110],[111],[112],[113],[114],[115],[116],[117],[118],[119],[120],[121],[122],[123],[124],[125],[126],[127] - ], + "core": [[0],[1],[2],[3],[4],[5],[6],[7],[8],[9],[10],[11],[12],[13],[14],[15],[16],[17],[18],[19],[20],[21],[22],[23],[24],[25],[26],[27],[28],[29],[30],[31],[32],[33],[34],[35],[36],[37],[38],[39],[40],[41],[42],[43],[44],[45],[46],[47],[48],[49],[50],[51],[52],[53],[54],[55],[56],[57],[58],[59],[60],[61],[62],[63],[64],[65],[66],[67],[68],[69],[70],[71],[72],[73],[74],[75],[76],[77],[78],[79],[80],[81],[82],[83],[84],[85],[86],[87],[88],[89],[90],[91],[92],[93],[94],[95],[96],[97],[98],[99],[100],[101],[102],[103],[104],[105],[106],[107],[108],[109],[110],[111],[112],[113],[114],[115],[116],[117],[118],[119],[120],[121],[122],[123],[124],[125],[126],[127]], "accelerators": [ { "id": "00000000:01:00.0", @@ -68,14 +266,32 @@ }, { "name": "a100", - "numberOfNodes": 20, + "nodes": "a[0601-0605],a[0701-0705],a[0801-0805],a[0901-0905]", "processorType": "AMD Milan", "socketsPerNode": 2, "coresPerSocket": 64, "threadsPerCore": 1, - "flopRateScalar": 432, - "flopRateSimd": 9216, - "memoryBandwidth": 400, + "flopRateScalar": { + "unit": { + "base": "F/s", + "prefix": "G" + }, + "value": 432 + }, + "flopRateSimd": { + "unit": { + "base": "F/s", + "prefix": "G" + }, + "value": 9216 + }, + "memoryBandwidth": { + "unit": { + "base": "B/s", + "prefix": "G" + }, + "value": 400 + }, "topology": { "node": [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127], "socket": [ @@ -85,9 +301,7 @@ "memoryDomain": [ [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127] ], - "core": [ - [0],[1],[2],[3],[4],[5],[6],[7],[8],[9],[10],[11],[12],[13],[14],[15],[16],[17],[18],[19],[20],[21],[22],[23],[24],[25],[26],[27],[28],[29],[30],[31],[32],[33],[34],[35],[36],[37],[38],[39],[40],[41],[42],[43],[44],[45],[46],[47],[48],[49],[50],[51],[52],[53],[54],[55],[56],[57],[58],[59],[60],[61],[62],[63],[64],[65],[66],[67],[68],[69],[70],[71],[73],[74],[75],[76],[77],[78],[79],[80],[81],[82],[83],[84],[85],[86],[87],[88],[89],[90],[91],[92],[93],[94],[95],[96],[97],[98],[99],[100],[101],[102],[103],[104],[105],[106],[107],[108],[109],[110],[111],[112],[113],[114],[115],[116],[117],[118],[119],[120],[121],[122],[123],[124],[125],[126],[127] - ], + "core": [[0],[1],[2],[3],[4],[5],[6],[7],[8],[9],[10],[11],[12],[13],[14],[15],[16],[17],[18],[19],[20],[21],[22],[23],[24],[25],[26],[27],[28],[29],[30],[31],[32],[33],[34],[35],[36],[37],[38],[39],[40],[41],[42],[43],[44],[45],[46],[47],[48],[49],[50],[51],[52],[53],[54],[55],[56],[57],[58],[59],[60],[61],[62],[63],[64],[65],[66],[67],[68],[69],[70],[71],[72],[73],[74],[75],[76],[77],[78],[79],[80],[81],[82],[83],[84],[85],[86],[87],[88],[89],[90],[91],[92],[93],[94],[95],[96],[97],[98],[99],[100],[101],[102],[103],[104],[105],[106],[107],[108],[109],[110],[111],[112],[113],[114],[115],[116],[117],[118],[119],[120],[121],[122],[123],[124],[125],[126],[127]], "accelerators": [ { "id": "00000000:0E:00.0", @@ -132,106 +346,5 @@ ] } } - ], - "metricConfig": [ - { - "name": "cpu_load", - "scope": "node", - "unit": "load 1m", - "timestep": 60, - "aggregation": null, - "peak": 128, - "normal": 128, - "caution": 10, - "alert": 5 - }, - { - "name": "cpu_user", - "scope": "hwthread", - "unit": "cpu user", - "timestep": 60, - "aggregation": "avg", - "peak": 100, - "normal": 50, - "caution": 20, - "alert": 10 - }, - { - "name": "mem_used", - "scope": "node", - "unit": "GB", - "timestep": 60, - "aggregation": null, - "peak": 512, - "normal": 128, - "caution": 200, - "alert": 240 - }, - { - "name": "flops_any", - "scope": "hwthread", - "unit": "GF/s", - "timestep": 60, - "aggregation": "sum", - "peak": 9216, - "normal": 1000, - "caution": 200, - "alert": 50 - }, - { - "name": "mem_bw", - "scope": "socket", - "unit": "GB/s", - "timestep": 60, - "aggregation": "sum", - "peak": 350, - "normal": 100, - "caution": 50, - "alert": 10 - }, - { - "name": "clock", - "scope": "hwthread", - "unit": "MHz", - "timestep": 60, - "aggregation": "avg", - "peak": 3000, - "normal": 2400, - "caution": 1800, - "alert": 1200 - }, - { - "name": "core_power", - "scope": "hwthread", - "unit": "W", - "timestep": 60, - "aggregation": "sum", - "peak": 500, - "normal": 250, - "caution": 100, - "alert": 50 - }, - { - "name": "cpu_power", - "scope": "socket", - "unit": "W", - "timestep": 60, - "aggregation": "sum", - "peak": 500, - "normal": 250, - "caution": 100, - "alert": 50 - }, - { - "name": "ipc", - "scope": "hwthread", - "unit": "IPC", - "timestep": 60, - "aggregation": "avg", - "peak": 4, - "normal": 2, - "caution": 1, - "alert": 0.5 - } ] } diff --git a/configs/config-large.json b/configs/config-large.json index 79ba02a3..29bc79c8 100644 --- a/configs/config-large.json +++ b/configs/config-large.json @@ -3,13 +3,29 @@ "addr": "0.0.0.0:8088", "short-running-jobs-duration": 300, "emission-constant": 317, + "enable-job-taggers": true, + "db-config": { + "cache-size-mb": 16384, + "soft-heap-limit-mb": 131072, + "max-open-connections": 8, + "max-idle-connections": 8, + "max-idle-time-minutes": 30 + }, + "nodestate-retention": { + "policy": "move", + "age": 168, + "target-kind": "file", + "target-path": "./var/nodestate-archive" + }, "resampling": { "default-policy": "medium", "default-algo": "average" }, "api-subjects": { "subject-job-event": "cc.job.event", - "subject-node-state": "cc.node.state" + "subject-node-state": "cc.node.state", + "job-concurrency": 8, + "node-concurrency": 2 } }, "nats": { @@ -58,7 +74,16 @@ }, "archive": { "kind": "file", - "path": "./var/job-archive" + "path": "./var/job-archive", + "compression": 7, + "retention": { + "policy": "move", + "format": "parquet", + "age": 365, + "include-db": true, + "target-kind": "file", + "target-path": "./var/job-archive-retention" + } }, "metric-store-external": [ { @@ -74,6 +99,16 @@ ], "metric-store": { "retention-in-memory": "24h", - "memory-cap": 100 + "memory-cap": 100, + "num-workers": 10, + "checkpoints": { + "file-format": "wal", + "directory": "./var/checkpoints" + }, + "checkpoint-interval": "12h", + "cleanup": { + "mode": "archive", + "directory": "./var/metric-archive" + } } } diff --git a/configs/config.json b/configs/config.json index 89995746..1c919ce0 100644 --- a/configs/config.json +++ b/configs/config.json @@ -23,7 +23,9 @@ }, "api-subjects": { "subject-job-event": "cc.job.event", - "subject-node-state": "cc.node.state" + "subject-node-state": "cc.node.state", + "job-concurrency": 8, + "node-concurrency": 2 } }, "nats": { @@ -51,8 +53,12 @@ "secret-key": "xx", "retention": { "policy": "move", + "format": "parquet", "age": 365, - "location": "./var/archive" + "include-db": true, + "target-kind": "file", + "target-path": "./var/job-archive-retention", + "max-file-size-mb": 512 } }, "metric-store-external": [ @@ -79,8 +85,11 @@ ], "metric-store": { "checkpoints": { - "directory": "./var/checkpoints" + "file-format": "wal", + "directory": "./var/checkpoints", + "max-wal-size": 268435456 }, + "checkpoint-interval": "12h", "memory-cap": 100, "retention-in-memory": "24h", "cleanup": { @@ -98,5 +107,5 @@ } ] }, - "ui-file": "ui-config.json" + "ui-file": "uiConfig.json" } diff --git a/configs/uiConfig.json b/configs/uiConfig.json index f2b89232..397f0334 100644 --- a/configs/uiConfig.json +++ b/configs/uiConfig.json @@ -1,7 +1,7 @@ { "job-list": { "use-paging": false, - "show-footprint":false + "show-footprint": false }, "job-view": { "show-polar-plot": true, @@ -9,6 +9,9 @@ "show-roofline": true, "show-stat-table": true }, + "node-list": { + "use-paging": true + }, "metric-config": { "job-list-metrics": ["mem_bw", "flops_dp"], "job-view-plot-metrics": ["mem_bw", "flops_dp"], @@ -25,21 +28,21 @@ } ] }, - "node-list": { - "use-paging": true - }, "plot-configuration": { "plots-per-row": 3, "color-background": true, "line-width": 3, + "smoothing-window": 0, + "resample-policy": "medium", + "resample-algo": "average", "color-scheme": [ - "#00bfff", - "#0000ff", - "#ff00ff", - "#ff0000", - "#ff8000", - "#ffff00", - "#80ff00" + "#00bfff", + "#0000ff", + "#ff00ff", + "#ff0000", + "#ff8000", + "#ffff00", + "#80ff00" ] } } diff --git a/internal/configexample/example_config_test.go b/internal/configexample/example_config_test.go new file mode 100644 index 00000000..e9932e10 --- /dev/null +++ b/internal/configexample/example_config_test.go @@ -0,0 +1,156 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// Package configexample guards the shipped example configuration files in +// ./configs against drift: every section must still be accepted by the code +// that parses it at startup. The test lives in its own package because it has +// to import every config-consuming package at once. +package configexample + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/ClusterCockpit/cc-backend/internal/config" + "github.com/ClusterCockpit/cc-backend/internal/metricdispatch" + "github.com/ClusterCockpit/cc-backend/internal/taskmanager" + "github.com/ClusterCockpit/cc-backend/pkg/metricstore" + "github.com/ClusterCockpit/cc-backend/web" + ccschema "github.com/ClusterCockpit/cc-lib/v2/schema" +) + +const configDir = "../../configs" + +// knownSections mirrors the list in cmd/cc-backend/main.go. +var knownSections = map[string]bool{ + "main": true, "auth": true, "nats": true, "archive": true, + "metric-store": true, "metric-store-external": true, "cron": true, "ui": true, +} + +// archiveConfig covers the union of all archive backend options plus the +// retention/compression keys consumed by the task manager. +type archiveConfig struct { + Kind string `json:"kind"` + Path string `json:"path"` + DBPath string `json:"db-path"` + Endpoint string `json:"endpoint"` + Bucket string `json:"bucket"` + Region string `json:"region"` + UsePathStyle bool `json:"use-path-style"` + AccessKey string `json:"access-key"` + SecretKey string `json:"secret-key"` + Retention taskmanager.Retention `json:"retention"` + Compression int `json:"compression"` +} + +func decodeStrict(t *testing.T, name string, raw json.RawMessage, target any) { + t.Helper() + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(target); err != nil { + t.Errorf("%s: %v", name, err) + } +} + +func TestExampleConfigs(t *testing.T) { + for _, name := range []string{"config.json", "config-demo.json", "config-large.json"} { + t.Run(name, func(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(configDir, name)) + if err != nil { + t.Fatal(err) + } + var sections map[string]json.RawMessage + if err := json.Unmarshal(raw, §ions); err != nil { + t.Fatal(err) + } + + for key, value := range sections { + // ccConfig resolves a "-file" key by loading the + // referenced file and storing it under "". + section := key + if base, ok := strings.CutSuffix(key, "-file"); ok { + section = base + var path string + if err := json.Unmarshal(value, &path); err != nil { + t.Errorf("%s: value of %q is not a file path: %v", name, key, err) + continue + } + b, err := os.ReadFile(filepath.Join(configDir, path)) + if err != nil { + t.Errorf("%s: %q references %s: %v", name, key, path, err) + continue + } + value = json.RawMessage(b) + } + + if !knownSections[section] { + t.Errorf("%s: unknown top-level section %q", name, key) + continue + } + + switch section { + case "main": + var cfg config.ProgramConfig + decodeStrict(t, key, value, &cfg) + case "cron": + var cfg taskmanager.CronFrequency + decodeStrict(t, key, value, &cfg) + case "archive": + var cfg archiveConfig + decodeStrict(t, key, value, &cfg) + case "metric-store": + var cfg metricstore.MetricStoreConfig + decodeStrict(t, key, value, &cfg) + case "metric-store-external": + var cfg []metricdispatch.CCMetricStoreConfig + decodeStrict(t, key, value, &cfg) + case "ui": + var cfg web.WebConfig + decodeStrict(t, key, value, &cfg) + } + } + }) + } +} + +func TestExampleClusterConfig(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(configDir, "cluster.json")) + if err != nil { + t.Fatal(err) + } + if err := ccschema.Validate(ccschema.ClusterCfg, bytes.NewReader(raw)); err != nil { + t.Errorf("cluster.json does not validate: %v", err) + } + var cluster ccschema.Cluster + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&cluster); err != nil { + t.Errorf("cluster.json: %v", err) + } +} + +// TestArchiveTestdataClusterConfigs keeps the cluster.json fixtures valid +// against the cc-lib schema. They are copied into real deployments, and an +// invalid one only fails at runtime with main.validate enabled. +func TestArchiveTestdataClusterConfigs(t *testing.T) { + for _, path := range []string{ + "../../pkg/archive/testdata/archive/alex/cluster.json", + "../../pkg/archive/testdata/archive/fritz/cluster.json", + "../../pkg/archive/testdata/archive/emmy/cluster.json", + "../../internal/importer/testdata/cluster-fritz.json", + } { + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if err := ccschema.Validate(ccschema.ClusterCfg, bytes.NewReader(raw)); err != nil { + t.Errorf("%s does not validate: %v", path, err) + } + } +} diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go index 94a9c3c8..5701876b 100644 --- a/pkg/archive/archive.go +++ b/pkg/archive/archive.go @@ -31,7 +31,7 @@ // } // } // -// For S3 backend (endpoint, region, and usePathStyle are optional): +// For S3 backend (endpoint, region, and use-path-style are optional): // // { // "archive": { @@ -39,9 +39,9 @@ // "endpoint": "http://192.168.178.10", // "bucket": "my-job-archive", // "region": "us-east-1", -// "usePathStyle": true, -// "accessKey": "...", -// "secretKey": "..." +// "use-path-style": true, +// "access-key": "...", +// "secret-key": "..." // } // } // @@ -50,7 +50,7 @@ // { // "archive": { // "kind": "sqlite", -// "dbPath": "/var/lib/archive.db" +// "db-path": "/var/lib/archive.db" // } // } //