ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2025-07-01 11:13:50 +02:00
Code Issues Packages Releases Activity

Merge branch 'dev' of https://github.com/ClusterCockpit/cc-backend into dev

Browse Source
This commit is contained in:
Christoph Kluge 2025-06-27 15:52:57 +02:00
commit 805ea91fc2
7 changed files with 40 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/repository/tags.go
View File

@ -152,13 +152,13 @@ func (r *JobRepository) removeTagFromArchiveJobs(jobIds []int64) {
for _, j := range jobIds { for _, j := range jobIds {
tags, err := r.getArchiveTags(&j) tags, err := r.getArchiveTags(&j)
if err != nil { if err != nil {
log.Warn("Error while getting tags for job") log.Warnf("Error while getting tags for job %d", j)
continue continue
} }
job, err := r.FindByIdDirect(j) job, err := r.FindByIdDirect(j)
if err != nil { if err != nil {
log.Warn("Error while getting job") log.Warnf("Error while getting job %d", j)
continue continue
} }

3
internal/tagger/apps/qe.txt
View File

@ -1,3 +0,0 @@
pw
neb
ph

2
internal/tagger/apps/starccm.txt Normal file
View File

@ -0,0 +1,2 @@
starccm+
-podkey

4
internal/tagger/detectApp.go
View File

@ -11,6 +11,7 @@ import (
"io/fs" "io/fs"
"os" "os"
"path/filepath" "path/filepath"
"regexp"
"strings" "strings"
"github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/internal/repository"
@ -111,7 +112,8 @@ func (t *AppTagger) Match(job *schema.Job) {
for _, a := range t.apps { for _, a := range t.apps {
tag := a.tag tag := a.tag
for _, s := range a.strings { for _, s := range a.strings {
if strings.Contains(strings.ToLower(jobscript), s) { matched, _ := regexp.MatchString(s, strings.ToLower(jobscript))
if matched {
if !r.HasTag(id, t.tagType, tag) { if !r.HasTag(id, t.tagType, tag) {
r.AddTagOrCreateDirect(id, t.tagType, tag) r.AddTagOrCreateDirect(id, t.tagType, tag)
break out break out

6
internal/tagger/jobclasses/highload.json
View File

@ -14,13 +14,13 @@
"variables": [ "variables": [
{ {
"name": "load_threshold", "name": "load_threshold",
"expr": "(job.numCores / job.numNodes) * excessivecpuload_threshold_factor" "expr": "cpu_load.limits.peak * excessivecpuload_threshold_factor"
}, },
{ {
"name": "load_perc", "name": "load_perc",
"expr": "1.0 - (cpu_load.avg / cpu_load.limits.peak)" "expr": "1.0 - (cpu_load.avg / cpu_load.limits.peak)"
} }
], ],
"rule": "cpu_load.avg > cpu_load.limits.peak", "rule": "cpu_load.avg > load_threshold",
"hint": "This job was detected as excessiveload because the average cpu load {{.cpu_load.avg}} falls above the threshold {{.cpu_load.limits.peak}}." "hint": "This job was detected as excessiveload because the average cpu load {{.cpu_load.avg}} falls above the threshold {{.load_threshold}}."
} }

12
pkg/archive/archive.go
View File

@ -7,6 +7,7 @@ package archive
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"maps"
"sync" "sync"
"github.com/ClusterCockpit/cc-backend/pkg/log" "github.com/ClusterCockpit/cc-backend/pkg/log"
@ -60,6 +61,7 @@ var (
cache *lrucache.Cache = lrucache.New(128 * 1024 * 1024) cache *lrucache.Cache = lrucache.New(128 * 1024 * 1024)
ar ArchiveBackend ar ArchiveBackend
useArchive bool useArchive bool
mutex sync.Mutex
) )
func Init(rawConfig json.RawMessage, disableArchive bool) error { func Init(rawConfig json.RawMessage, disableArchive bool) error {
@ -184,6 +186,9 @@ func GetStatistics(job *schema.Job) (map[string]schema.JobStatistics, error) {
// If the job is archived, find its `meta.json` file and override the Metadata // If the job is archived, find its `meta.json` file and override the Metadata
// in that JSON file. If the job is not archived, nothing is done. // in that JSON file. If the job is not archived, nothing is done.
func UpdateMetadata(job *schema.Job, metadata map[string]string) error { func UpdateMetadata(job *schema.Job, metadata map[string]string) error {
mutex.Lock()
defer mutex.Unlock()
if job.State == schema.JobStateRunning || !useArchive { if job.State == schema.JobStateRunning || !useArchive {
return nil return nil
} }
@ -194,9 +199,7 @@ func UpdateMetadata(job *schema.Job, metadata map[string]string) error {
return err return err
} }
for k, v := range metadata { maps.Copy(jobMeta.MetaData, metadata)
jobMeta.MetaData[k] = v
}
return ar.StoreJobMeta(jobMeta) return ar.StoreJobMeta(jobMeta)
} }
@ -204,6 +207,9 @@ func UpdateMetadata(job *schema.Job, metadata map[string]string) error {
// If the job is archived, find its `meta.json` file and override the tags list // If the job is archived, find its `meta.json` file and override the tags list
// in that JSON file. If the job is not archived, nothing is done. // in that JSON file. If the job is not archived, nothing is done.
func UpdateTags(job *schema.Job, tags []*schema.Tag) error { func UpdateTags(job *schema.Job, tags []*schema.Tag) error {
mutex.Lock()
defer mutex.Unlock()
if job.State == schema.JobStateRunning || !useArchive { if job.State == schema.JobStateRunning || !useArchive {
return nil return nil
} }

94
pkg/schema/schemas/config.schema.json
View File

@ -38,10 +38,7 @@
"db-driver": { "db-driver": {
"description": "sqlite3 or mysql (mysql will work for mariadb as well).", "description": "sqlite3 or mysql (mysql will work for mariadb as well).",
"type": "string", "type": "string",
"enum": [ "enum": ["sqlite3", "mysql"]
"sqlite3",
"mysql"
]
}, },
"db": { "db": {
"description": "For sqlite3 a filename, for mysql a DSN in this format: https://github.com/go-sql-driver/mysql#dsn-data-source-name (Without query parameters!).", "description": "For sqlite3 a filename, for mysql a DSN in this format: https://github.com/go-sql-driver/mysql#dsn-data-source-name (Without query parameters!).",
@ -54,10 +51,7 @@
"kind": { "kind": {
"description": "Backend type for job-archive", "description": "Backend type for job-archive",
"type": "string", "type": "string",
"enum": [ "enum": ["file", "s3"]
"file",
"s3"
]
}, },
"path": { "path": {
"description": "Path to job archive for file backend", "description": "Path to job archive for file backend",
@ -74,11 +68,7 @@
"policy": { "policy": {
"description": "Retention policy", "description": "Retention policy",
"type": "string", "type": "string",
"enum": [ "enum": ["none", "delete", "move"]
"none",
"delete",
"move"
]
}, },
"includeDB": { "includeDB": {
"description": "Also remove jobs from database", "description": "Also remove jobs from database",
@ -93,19 +83,19 @@
"type": "string" "type": "string"
} }
}, },
"required": [ "required": ["policy"]
"policy"
]
} }
}, },
"required": [ "required": ["kind"]
"kind"
]
}, },
"disable-archive": { "disable-archive": {
"description": "Keep all metric data in the metric data repositories, do not write to the job-archive.", "description": "Keep all metric data in the metric data repositories, do not write to the job-archive.",
"type": "boolean" "type": "boolean"
}, },
"enable-job-taggers": {
"description": "Turn on automatic application and jobclass taggers",
"type": "boolean"
},
"validate": { "validate": {
"description": "Validate all input json documents against json schema.", "description": "Validate all input json documents against json schema.",
"type": "boolean" "type": "boolean"
@ -168,10 +158,7 @@
} }
} }
}, },
"required": [ "required": ["trigger", "resolutions"]
"trigger",
"resolutions"
]
}, },
"jwts": { "jwts": {
"description": "For JWT token authentication.", "description": "For JWT token authentication.",
@ -198,9 +185,7 @@
"type": "boolean" "type": "boolean"
} }
}, },
"required": [ "required": ["max-age"]
"max-age"
]
}, },
"oidc": { "oidc": {
"provider": { "provider": {
@ -215,9 +200,7 @@
"description": "", "description": "",
"type": "boolean" "type": "boolean"
}, },
"required": [ "required": ["provider"]
"provider"
]
}, },
"ldap": { "ldap": {
"description": "For LDAP Authentication and user synchronisation.", "description": "For LDAP Authentication and user synchronisation.",
@ -260,13 +243,7 @@
"type": "boolean" "type": "boolean"
} }
}, },
"required": [ "required": ["url", "user_base", "search_dn", "user_bind", "user_filter"]
"url",
"user_base",
"search_dn",
"user_bind",
"user_filter"
]
}, },
"clusters": { "clusters": {
"description": "Configuration for the clusters to be displayed.", "description": "Configuration for the clusters to be displayed.",
@ -284,12 +261,7 @@
"properties": { "properties": {
"kind": { "kind": {
"type": "string", "type": "string",
"enum": [ "enum": ["influxdb", "prometheus", "cc-metric-store", "test"]
"influxdb",
"prometheus",
"cc-metric-store",
"test"
]
}, },
"url": { "url": {
"type": "string" "type": "string"
@ -298,10 +270,7 @@
"type": "string" "type": "string"
} }
}, },
"required": [ "required": ["kind", "url"]
"kind",
"url"
]
}, },
"filterRanges": { "filterRanges": {
"description": "This option controls the slider ranges for the UI controls of numNodes, duration, and startTime.", "description": "This option controls the slider ranges for the UI controls of numNodes, duration, and startTime.",
@ -318,10 +287,7 @@
"type": "integer" "type": "integer"
} }
}, },
"required": [ "required": ["from", "to"]
"from",
"to"
]
}, },
"duration": { "duration": {
"description": "UI slider range for duration", "description": "UI slider range for duration",
@ -334,10 +300,7 @@
"type": "integer" "type": "integer"
} }
}, },
"required": [ "required": ["from", "to"]
"from",
"to"
]
}, },
"startTime": { "startTime": {
"description": "UI slider range for start time", "description": "UI slider range for start time",
@ -351,24 +314,13 @@
"type": "null" "type": "null"
} }
}, },
"required": [ "required": ["from", "to"]
"from",
"to"
]
} }
}, },
"required": [ "required": ["numNodes", "duration", "startTime"]
"numNodes",
"duration",
"startTime"
]
} }
}, },
"required": [ "required": ["name", "metricDataRepository", "filterRanges"],
"name",
"metricDataRepository",
"filterRanges"
],
"minItems": 1 "minItems": 1
} }
}, },
@ -490,9 +442,5 @@
] ]
} }
}, },
"required": [ "required": ["jwts", "clusters", "apiAllowedIPs"]
"jwts",
"clusters",
"apiAllowedIPs"
]
} }