ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2025-10-31 07:55:06 +01:00
Code Issues Packages Releases Activity

Merge branch 'dev' of github.com:ClusterCockpit/cc-backend into dev

Browse Source
This commit is contained in:
Jan Eitzinger
2025-10-29 09:01:05 +01:00
parent ae5d202661 2287f4493a
commit cb5d06decd
4 changed files with 97 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
configs/config-demo.json
View File

@@ -34,9 +34,9 @@
{ {
"name": "fritz", "name": "fritz",
"metricDataRepository": { "metricDataRepository": {
"kind": "cc-metric-store", "kind": "cc-metric-store-internal",
"url": "http://localhost:8082", "url": "http://localhost:8082",
"token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9" "token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw"
}, },
"filterRanges": { "filterRanges": {
"numNodes": { "numNodes": {
@@ -56,9 +56,9 @@
{ {
"name": "alex", "name": "alex",
"metricDataRepository": { "metricDataRepository": {
"kind": "cc-metric-store", "kind": "cc-metric-store-internal",
"url": "http://localhost:8082", "url": "http://localhost:8082",
"token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9" "token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw"
}, },
"filterRanges": { "filterRanges": {
"numNodes": { "numNodes": {
@@ -79,14 +79,15 @@
"metric-store": { "metric-store": {
"checkpoints": { "checkpoints": {
"file-format": "avro", "file-format": "avro",
"interval": "2h", "interval": "1h",
"directory": "./var/checkpoints", "directory": "./var/checkpoints",
"restore": "48h" "restore": "48h"
}, },
"archive": { "archive": {
"interval": "2h", "interval": "1h",
"directory": "./var/archive" "directory": "./var/archive"
}, },
"retention-in-memory": "48h" "retention-in-memory": "48h"
} },
"ui-file": "./configs/uiConfig.json"
} }

9
internal/memorystore/avroCheckpoint.go
View File

@@ -25,7 +25,7 @@ import (
) )
var NumAvroWorkers int = 4 var NumAvroWorkers int = 4
var startUp bool = true
var ErrNoNewData error = errors.New("no data in the pool") var ErrNoNewData error = errors.New("no data in the pool")
func (as *AvroStore) ToCheckpoint(dir string, dumpAll bool) (int, error) { func (as *AvroStore) ToCheckpoint(dir string, dumpAll bool) (int, error) {
@@ -96,6 +96,9 @@ func (as *AvroStore) ToCheckpoint(dir string, dumpAll bool) (int, error) {
if errs > 0 { if errs > 0 {
return int(n), fmt.Errorf("%d errors happend while creating avro checkpoints (%d successes)", errs, n) return int(n), fmt.Errorf("%d errors happend while creating avro checkpoints (%d successes)", errs, n)
} }
startUp = false
return int(n), nil return int(n), nil
} }
@@ -143,6 +146,10 @@ func getTimestamp(dir string) int64 {
interval, _ := time.ParseDuration(Keys.Checkpoints.Interval) interval, _ := time.ParseDuration(Keys.Checkpoints.Interval)
updateTime := time.Unix(maxTS, 0).Add(interval).Add(time.Duration(CheckpointBufferMinutes-1) * time.Minute).Unix() updateTime := time.Unix(maxTS, 0).Add(interval).Add(time.Duration(CheckpointBufferMinutes-1) * time.Minute).Unix()
if startUp {
return 0
}
if updateTime < time.Now().Unix() { if updateTime < time.Now().Unix() {
return 0 return 0
} }

251
internal/memorystore/configSchema.go
View File

@@ -6,185 +6,90 @@
package memorystore package memorystore
const configSchema = `{ const configSchema = `{
"type": "object", "type": "object",
"properties": { "description": "Configuration specific to built-in metric-store.",
"jobList": { "properties": {
"description": "Job list defaults. Applies to user- and jobs views.", "checkpoints": {
"type": "object", "description": "Configuration for checkpointing the metrics within metric-store",
"properties": {
"usePaging": {
"description": "If classic paging is used instead of continuous scrolling by default.",
"type": "boolean"
},
"showFootprint": {
"description": "If footprint bars are shown as first column by default.",
"type": "boolean"
}
}
},
"nodeList": {
"description": "Node list defaults. Applies to node list view.",
"type": "object",
"properties": {
"usePaging": {
"description": "If classic paging is used instead of continuous scrolling by default.",
"type": "boolean"
}
}
},
"jobView": {
"description": "Job view defaults.",
"type": "object",
"properties": {
"showPolarPlot": {
"description": "If the job metric footprints polar plot is shown by default.",
"type": "boolean"
},
"showFootprint": {
"description": "If the annotated job metric footprint bars are shown by default.",
"type": "boolean"
},
"showRoofline": {
"description": "If the job roofline plot is shown by default.",
"type": "boolean"
},
"showStatTable": {
"description": "If the job metric statistics table is shown by default.",
"type": "boolean"
}
}
},
"metricConfig": {
"description": "Global initial metric selections for primary views of all clusters.",
"type": "object",
"properties": {
"jobListMetrics": {
"description": "Initial metrics shown for new users in job lists (User and jobs view).",
"type": "array",
"items": {
"type": "string",
"minItems": 1
}
},
"jobViewPlotMetrics": {
"description": "Initial metrics shown for new users as job view metric plots.",
"type": "array",
"items": {
"type": "string",
"minItems": 1
}
},
"jobViewTableMetrics": {
"description": "Initial metrics shown for new users in job view statistics table.",
"type": "array",
"items": {
"type": "string",
"minItems": 1
}
},
"clusters": {
"description": "Overrides for global defaults by cluster and subcluster.",
"type": "array",
"items": {
"type": "object", "type": "object",
"properties": { "properties": {
"name": { "file-format": {
"description": "The name of the cluster." "description": "Specify the type of checkpoint file. There are 2 variants: 'avro' and 'json'. If nothing is specified, 'avro' is default.",
}, "type": "string"
"jobListMetrics": { },
"description": "Initial metrics shown for new users in job lists (User and jobs view) for subcluster.", "interval": {
"type": "array", "description": "Interval at which the metrics should be checkpointed.",
"items": { "type": "string"
"type": "string", },
"minItems": 1 "directory": {
"description": "Specify the parent directy in which the checkpointed files should be placed.",
"type": "string"
},
"restore": {
"description": "When cc-backend starts up, look for checkpointed files that are less than X hours old and load metrics from these selected checkpoint files.",
"type": "string"
} }
}, }
"jobViewPlotMetrics": {
"description": "Initial metrics shown for new users as job view timeplots for subcluster.",
"type": "array",
"items": {
"type": "string",
"minItems": 1
}
},
"jobViewTableMetrics": {
"description": "Initial metrics shown for new users in job view statistics table for subcluster.",
"type": "array",
"items": {
"type": "string",
"minItems": 1
}
},
"subClusters": {
"description": "The array of overrides per subcluster.",
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"description": "The name of the subcluster.",
"type": "string"
},
"jobListMetrics": {
"description": "Initial metrics shown for new users in job lists (User and jobs view) for subcluster.",
"type": "array",
"items": {
"type": "string",
"minItems": 1
}
},
"jobViewPlotMetrics": {
"description": "Initial metrics shown for new users as job view timeplots for subcluster.",
"type": "array",
"items": {
"type": "string",
"minItems": 1
}
},
"jobViewTableMetrics": {
"description": "Initial metrics shown for new users in job view statistics table for subcluster.",
"type": "array",
"items": {
"type": "string",
"minItems": 1
}
}
},
"required": ["name"],
"minItems": 1
}
}
},
"required": ["name", "subClusters"],
"minItems": 1
}
}
}
},
"plotConfiguration": {
"description": "Initial settings for plot render options.",
"type": "object",
"properties": {
"colorBackground": {
"description": "If the metric plot backgrounds are initially colored by threshold limits.",
"type": "boolean"
}, },
"plotsPerRow": { "archive": {
"description": "How many plots are initially rendered in per row. Applies to job, single node, and analysis views.", "description": "Configuration for archiving the already checkpointed files.",
"type": "integer" "type": "object",
"properties": {
"interval": {
"description": "Interval at which the checkpointed files should be archived.",
"type": "string"
},
"directory": {
"description": "Specify the parent directy in which the archived files should be placed.",
"type": "string"
}
}
}, },
"lineWidth": { "retention-in-memory": {
"description": "Initial thickness of rendered plotlines. Applies to metric plot, job compare plot and roofline.", "description": "Keep the metrics within memory for given time interval. Retention for X hours, then the metrics would be freed.",
"type": "integer"
},
"colorScheme": {
"description": "Initial colorScheme to be used for metric plots.",
"type": "array",
"items": {
"type": "string" "type": "string"
} },
"nats": {
"description": "Configuration for accepting published data through NATS.",
"type": "array",
"items": {
"type": "object",
"properties": {
"address": {
"description": "Address of the NATS server.",
"type": "string"
},
"username": {
"description": "Optional: If configured with username/password method.",
"type": "string"
},
"password": {
"description": "Optional: If configured with username/password method.",
"type": "string"
},
"creds-file-path": {
"description": "Optional: If configured with Credential File method. Path to your NATS cred file.",
"type": "string"
},
"subscriptions": {
"description": "Array of various subscriptions. Allows to subscibe to different subjects and publishers.",
"type": "array",
"items": {
"type": "object",
"properties": {
"subscribe-to": {
"description": "Channel name",
"type": "string"
},
"cluster-tag": {
"description": "Optional: Allow lines without a cluster tag, use this as default",
"type": "string"
}
}
}
}
}
}
} }
}
} }
}
}` }`

3
startDemo.sh
View File

@@ -1,5 +1,7 @@
#!/bin/sh #!/bin/sh
# rm -rf var
if [ -d './var' ]; then if [ -d './var' ]; then
echo 'Directory ./var already exists! Skipping initialization.' echo 'Directory ./var already exists! Skipping initialization.'
./cc-backend -server -dev ./cc-backend -server -dev
@@ -14,6 +16,7 @@ else
./cc-backend -migrate-db ./cc-backend -migrate-db
./cc-backend -dev -init-db -add-user demo:admin,api:demo ./cc-backend -dev -init-db -add-user demo:admin,api:demo
./cc-backend -server -dev ./cc-backend -server -dev
fi fi