diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index 871c8dd..56018c3 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -5,6 +5,7 @@ package main import ( + "encoding/json" "fmt" "os" "os/signal" @@ -22,8 +23,9 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/tagger" "github.com/ClusterCockpit/cc-backend/internal/taskManager" "github.com/ClusterCockpit/cc-backend/pkg/archive" - "github.com/ClusterCockpit/cc-backend/pkg/runtimeEnv" + ccconf "github.com/ClusterCockpit/cc-lib/ccConfig" cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + "github.com/ClusterCockpit/cc-lib/runtimeEnv" "github.com/ClusterCockpit/cc-lib/schema" "github.com/ClusterCockpit/cc-lib/util" "github.com/google/gops/agent" @@ -85,14 +87,17 @@ func main() { // Initialize sub-modules and handle command line flags. // The order here is important! - config.Init(flagConfigFile) + ccconf.Init(flagConfigFile) - // As a special case for `db`, allow using an environment variable instead of the value - // stored in the config. This can be done for people having security concerns about storing - // the password for their mysql database in config.json. - if strings.HasPrefix(config.Keys.DB, "env:") { - envvar := strings.TrimPrefix(config.Keys.DB, "env:") - config.Keys.DB = os.Getenv(envvar) + // Load and check main configuration + if cfg := ccconf.GetPackageConfig("main"); cfg != nil { + if clustercfg := ccconf.GetPackageConfig("clusters"); clustercfg != nil { + config.Init(cfg, clustercfg) + } else { + cclog.Abort("Cluster configuration must be present") + } + } else { + cclog.Abort("Main configuration must be present") } if flagMigrateDB { @@ -123,7 +128,12 @@ func main() { if !config.Keys.DisableAuthentication { - auth.Init() + if cfg := ccconf.GetPackageConfig("auth"); cfg != nil { + auth.Init(&cfg) + } else { + cclog.Warn("Authentication disabled due to missing configuration") + auth.Init(nil) + } if flagNewUser != "" { parts := strings.SplitN(flagNewUser, ":", 3) @@ -188,7 +198,12 @@ func main() { cclog.Abort("Error: Arguments '--add-user' and '--del-user' can only be used if authentication is enabled. No changes, exited.") } - if err := archive.Init(config.Keys.Archive, config.Keys.DisableArchive); err != nil { + if archiveCfg := ccconf.GetPackageConfig("archive"); archiveCfg != nil { + err = archive.Init(archiveCfg, config.Keys.DisableArchive) + } else { + err = archive.Init(json.RawMessage(`{\"kind\":\"file\",\"path\":\"./var/job-archive\"}`), config.Keys.DisableArchive) + } + if err != nil { cclog.Abortf("Init: Failed to initialize archive.\nError: %s\n", err.Error()) } @@ -228,7 +243,8 @@ func main() { archiver.Start(repository.GetJobRepository()) - taskManager.Start() + taskManager.Start(ccconf.GetPackageConfig("cron"), + ccconf.GetPackageConfig("archive")) serverInit() var wg sync.WaitGroup diff --git a/cmd/cc-backend/server.go b/cmd/cc-backend/server.go index c01008a..3983268 100644 --- a/cmd/cc-backend/server.go +++ b/cmd/cc-backend/server.go @@ -27,9 +27,9 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/graph" "github.com/ClusterCockpit/cc-backend/internal/graph/generated" "github.com/ClusterCockpit/cc-backend/internal/routerConfig" - "github.com/ClusterCockpit/cc-backend/pkg/runtimeEnv" "github.com/ClusterCockpit/cc-backend/web" cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + "github.com/ClusterCockpit/cc-lib/runtimeEnv" "github.com/gorilla/handlers" "github.com/gorilla/mux" httpSwagger "github.com/swaggo/http-swagger" @@ -93,7 +93,7 @@ func serverInit() { info := map[string]any{} info["hasOpenIDConnect"] = false - if config.Keys.OpenIDConfig != nil { + if auth.Keys.OpenIDConfig != nil { openIDConnect := auth.NewOIDC(authHandle) openIDConnect.RegisterEndpoints(router) info["hasOpenIDConnect"] = true diff --git a/configs/config-demo.json b/configs/config-demo.json index 9425bd2..d388d78 100644 --- a/configs/config-demo.json +++ b/configs/config-demo.json @@ -1,26 +1,19 @@ { - "addr": "127.0.0.1:8080", - "short-running-jobs-duration": 300, - "archive": { - "kind": "file", - "path": "./var/job-archive" + "main": { + "addr": "127.0.0.1:8080", + "short-running-jobs-duration": 300, + "resampling": { + "trigger": 30, + "resolutions": [600, 300, 120, 60] + }, + "apiAllowedIPs": ["*"], + "emission-constant": 317 }, - "jwts": { - "max-age": "2000h" + "auth": { + "jwts": { + "max-age": "2000h" + } }, - "enable-resampling": { - "trigger": 30, - "resolutions": [ - 600, - 300, - 120, - 60 - ] - }, - "apiAllowedIPs": [ - "*" - ], - "emission-constant": 317, "clusters": [ { "name": "fritz", diff --git a/configs/config-mariadb.json b/configs/config-mariadb.json index e068439..38bb8a9 100644 --- a/configs/config-mariadb.json +++ b/configs/config-mariadb.json @@ -12,12 +12,7 @@ "db": "clustercockpit:demo@tcp(127.0.0.1:3306)/clustercockpit", "enable-resampling": { "trigger": 30, - "resolutions": [ - 600, - 300, - 120, - 60 - ] + "resolutions": [600, 300, 120, 60] }, "emission-constant": 317, "clusters": [ diff --git a/configs/config.json b/configs/config.json index f946b20..27c4ce2 100644 --- a/configs/config.json +++ b/configs/config.json @@ -1,24 +1,18 @@ { - "addr": "0.0.0.0:443", - "ldap": { - "url": "ldaps://test", - "user_base": "ou=people,ou=hpc,dc=test,dc=de", - "search_dn": "cn=hpcmonitoring,ou=roadm,ou=profile,ou=hpc,dc=test,dc=de", - "user_bind": "uid={username},ou=people,ou=hpc,dc=test,dc=de", - "user_filter": "(&(objectclass=posixAccount))" + "main": { + "addr": "0.0.0.0:443", + "https-cert-file": "/etc/letsencrypt/live/url/fullchain.pem", + "https-key-file": "/etc/letsencrypt/live/url/privkey.pem", + "user": "clustercockpit", + "group": "clustercockpit", + "validate": false, + "apiAllowedIPs": ["*"], + "short-running-jobs-duration": 300, + "resampling": { + "trigger": 30, + "resolutions": [600, 300, 120, 60] + } }, - "https-cert-file": "/etc/letsencrypt/live/url/fullchain.pem", - "https-key-file": "/etc/letsencrypt/live/url/privkey.pem", - "user": "clustercockpit", - "group": "clustercockpit", - "archive": { - "kind": "file", - "path": "./var/job-archive" - }, - "validate": false, - "apiAllowedIPs": [ - "*" - ], "clusters": [ { "name": "test", @@ -42,21 +36,5 @@ } } } - ], - "jwts": { - "cookieName": "", - "validateUser": false, - "max-age": "2000h", - "trustedIssuer": "" - }, - "enable-resampling": { - "trigger": 30, - "resolutions": [ - 600, - 300, - 120, - 60 - ] - }, - "short-running-jobs-duration": 300 + ] } diff --git a/go.mod b/go.mod index 4b5171c..1714807 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,10 @@ toolchain go1.24.1 require ( github.com/99designs/gqlgen v0.17.66 - github.com/ClusterCockpit/cc-lib v0.3.0 + github.com/ClusterCockpit/cc-lib v0.5.0 github.com/Masterminds/squirrel v1.5.4 github.com/coreos/go-oidc/v3 v3.12.0 - github.com/expr-lang/expr v1.17.3 + github.com/expr-lang/expr v1.17.5 github.com/go-co-op/gocron/v2 v2.16.0 github.com/go-ldap/ldap/v3 v3.4.10 github.com/go-sql-driver/mysql v1.9.0 @@ -23,14 +23,14 @@ require ( github.com/joho/godotenv v1.5.1 github.com/mattn/go-sqlite3 v1.14.24 github.com/prometheus/client_golang v1.22.0 - github.com/prometheus/common v0.63.0 + github.com/prometheus/common v0.65.0 github.com/qustavo/sqlhooks/v2 v2.1.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/swaggo/http-swagger v1.3.4 github.com/swaggo/swag v1.16.4 github.com/vektah/gqlparser/v2 v2.5.22 - golang.org/x/crypto v0.37.0 - golang.org/x/oauth2 v0.27.0 + golang.org/x/crypto v0.39.0 + golang.org/x/oauth2 v0.30.0 golang.org/x/time v0.5.0 ) @@ -77,13 +77,13 @@ require ( github.com/urfave/cli/v2 v2.27.5 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect go.uber.org/atomic v1.11.0 // indirect - golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect - golang.org/x/mod v0.24.0 // indirect - golang.org/x/net v0.39.0 // indirect - golang.org/x/sync v0.13.0 // indirect - golang.org/x/sys v0.32.0 // indirect - golang.org/x/text v0.24.0 // indirect - golang.org/x/tools v0.32.0 // indirect + golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect + golang.org/x/mod v0.25.0 // indirect + golang.org/x/net v0.41.0 // indirect + golang.org/x/sync v0.15.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/text v0.26.0 // indirect + golang.org/x/tools v0.34.0 // indirect google.golang.org/protobuf v1.36.6 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index f3d25ad..0cb3dd9 100644 --- a/go.sum +++ b/go.sum @@ -6,14 +6,16 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= -github.com/ClusterCockpit/cc-lib v0.3.0 h1:HEWOgnzRM01U10ZFfpiUWMzkLHg5nPdXZqdsiI2q4x0= -github.com/ClusterCockpit/cc-lib v0.3.0/go.mod h1:7CuXVNIJdynMZf6B9v4m54VCbbFg3ZD0tvLw2bVxN0A= +github.com/ClusterCockpit/cc-lib v0.5.0 h1:DSKAD1TxjVWyd1x3GWvxFeEkANF9o13T97nirj3CbRU= +github.com/ClusterCockpit/cc-lib v0.5.0/go.mod h1:0zLbJprwOWLA+OSNQ+OlUKLscZszwf9J2j8Ly5ztplk= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/NVIDIA/go-nvml v0.12.9-0 h1:e344UK8ZkeMeeLkdQtRhmXRxNf+u532LDZPGMtkdus0= +github.com/NVIDIA/go-nvml v0.12.9-0/go.mod h1:+KNA7c7gIBH7SKSJ1ntlwkfN80zdx8ovl4hrK3LmPt4= github.com/PuerkitoBio/goquery v1.9.3 h1:mpJr/ikUA9/GNJB/DBZcGeFDXUtosHRyRrwh7KGdTG0= github.com/PuerkitoBio/goquery v1.9.3/go.mod h1:1ndLHPdTz+DyQPICCWYlYQMPl0oXZj0G6D4LCYA6u4U= github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM= @@ -24,6 +26,8 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNg github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= +github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= +github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -49,8 +53,8 @@ github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/expr-lang/expr v1.17.3 h1:myeTTuDFz7k6eFe/JPlep/UsiIjVhG61FMHFu63U7j0= -github.com/expr-lang/expr v1.17.3/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= +github.com/expr-lang/expr v1.17.5 h1:i1WrMvcdLF249nSNlpQZN1S6NXuW9WaOfF5tPi3aw3k= +github.com/expr-lang/expr v1.17.5/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= @@ -120,6 +124,12 @@ github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/C github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/influxdata/influxdb-client-go/v2 v2.14.0 h1:AjbBfJuq+QoaXNcrova8smSjwJdUHnwvfjMF71M1iI4= +github.com/influxdata/influxdb-client-go/v2 v2.14.0/go.mod h1:Ahpm3QXKMJslpXl3IftVLVezreAUtBOTZssDrjZEFHI= +github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf h1:7JTmneyiNEwVBOHSjoMxiWAqB992atOeepeFYegn5RU= +github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/influxdata/line-protocol/v2 v2.2.1 h1:EAPkqJ9Km4uAxtMRgUubJyqAr6zgWM0dznKMLRauQRE= +github.com/influxdata/line-protocol/v2 v2.2.1/go.mod h1:DmB3Cnh+3oxmG6LOBIxce4oaL4CPj3OmMPgvauXh+tM= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -144,6 +154,8 @@ github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2E github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -176,6 +188,14 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nats-io/nats.go v1.43.0 h1:uRFZ2FEoRvP64+UUhaTokyS18XBCR/xM2vQZKO4i8ug= +github.com/nats-io/nats.go v1.43.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g= +github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0= +github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= +github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= @@ -189,8 +209,8 @@ github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/ github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k= -github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= +github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE= +github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/qustavo/sqlhooks/v2 v2.1.0 h1:54yBemHnGHp/7xgT+pxwmIlMSDNYKx5JW5dfRAiCZi0= @@ -250,17 +270,17 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= -golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= -golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM= -golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= -golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= +golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -272,10 +292,10 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= -golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= -golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= -golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -283,8 +303,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= -golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= +golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -296,8 +316,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= -golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -316,8 +336,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= -golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -326,8 +346,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU= -golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s= +golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= +golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= diff --git a/internal/api/api_test.go b/internal/api/api_test.go index 9b792c2..eeb093e 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -27,6 +27,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/metricdata" "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/archive" + ccconf "github.com/ClusterCockpit/cc-lib/ccConfig" cclog "github.com/ClusterCockpit/cc-lib/ccLogger" "github.com/ClusterCockpit/cc-lib/schema" "github.com/gorilla/mux" @@ -36,18 +37,22 @@ import ( func setup(t *testing.T) *api.RestApi { const testconfig = `{ + "main": { "addr": "0.0.0.0:8080", "validate": false, + "apiAllowedIPs": [ + "*" + ] + }, "archive": { "kind": "file", "path": "./var/job-archive" }, - "jwts": { - "max-age": "2m" - }, - "apiAllowedIPs": [ - "*" - ], + "auth": { + "jwts": { + "max-age": "2m" + } + }, "clusters": [ { "name": "testcluster", @@ -146,7 +151,18 @@ func setup(t *testing.T) *api.RestApi { t.Fatal(err) } - config.Init(cfgFilePath) + ccconf.Init(cfgFilePath) + + // Load and check main configuration + if cfg := ccconf.GetPackageConfig("main"); cfg != nil { + if clustercfg := ccconf.GetPackageConfig("clusters"); clustercfg != nil { + config.Init(cfg, clustercfg) + } else { + cclog.Abort("Cluster configuration must be present") + } + } else { + cclog.Abort("Main configuration must be present") + } archiveCfg := fmt.Sprintf("{\"kind\": \"file\",\"path\": \"%s\"}", jobarchive) repository.Connect("sqlite3", dbfilepath) @@ -160,7 +176,14 @@ func setup(t *testing.T) *api.RestApi { } archiver.Start(repository.GetJobRepository()) - auth.Init() + + if cfg := ccconf.GetPackageConfig("auth"); cfg != nil { + auth.Init(&cfg) + } else { + cclog.Warn("Authentication disabled due to missing configuration") + auth.Init(nil) + } + graph.Init() return api.New() diff --git a/internal/auth/auth.go b/internal/auth/auth.go index ad78397..6564878 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -5,10 +5,12 @@ package auth import ( + "bytes" "context" "crypto/rand" "database/sql" "encoding/base64" + "encoding/json" "errors" "fmt" "net" @@ -51,6 +53,14 @@ func getIPUserLimiter(ip, username string) *rate.Limiter { return limiter.(*rate.Limiter) } +type AuthConfig struct { + LdapConfig *LdapConfig `json:"ldap"` + JwtConfig *JWTAuthConfig `json:"jwts"` + OpenIDConfig *OpenIDConfig `json:"oidc"` +} + +var Keys AuthConfig + type Authentication struct { sessionStore *sessions.CookieStore LdapAuth *LdapAuthenticator @@ -87,7 +97,7 @@ func (auth *Authentication) AuthViaSession( }, nil } -func Init() { +func Init(authCfg *json.RawMessage) { initOnce.Do(func() { authInstance = &Authentication{} @@ -111,7 +121,18 @@ func Init() { authInstance.SessionMaxAge = d } - if config.Keys.LdapConfig != nil { + if authCfg == nil { + return + } + + config.Validate(configSchema, *authCfg) + dec := json.NewDecoder(bytes.NewReader(*authCfg)) + dec.DisallowUnknownFields() + if err := dec.Decode(&Keys); err != nil { + cclog.Errorf("error while decoding ldap config: %v", err) + } + + if Keys.LdapConfig != nil { ldapAuth := &LdapAuthenticator{} if err := ldapAuth.Init(); err != nil { cclog.Warn("Error while initializing authentication -> ldapAuth init failed") @@ -123,7 +144,7 @@ func Init() { cclog.Info("Missing LDAP configuration: No LDAP support!") } - if config.Keys.JwtConfig != nil { + if Keys.JwtConfig != nil { authInstance.JwtAuth = &JWTAuthenticator{} if err := authInstance.JwtAuth.Init(); err != nil { cclog.Fatal("Error while initializing authentication -> jwtAuth init failed") @@ -168,11 +189,11 @@ func handleTokenUser(tokenUser *schema.User) { if err != nil && err != sql.ErrNoRows { cclog.Errorf("Error while loading user '%s': %v", tokenUser.Username, err) - } else if err == sql.ErrNoRows && config.Keys.JwtConfig.SyncUserOnLogin { // Adds New User + } else if err == sql.ErrNoRows && Keys.JwtConfig.SyncUserOnLogin { // Adds New User if err := r.AddUser(tokenUser); err != nil { cclog.Errorf("Error while adding user '%s' to DB: %v", tokenUser.Username, err) } - } else if err == nil && config.Keys.JwtConfig.UpdateUserOnLogin { // Update Existing User + } else if err == nil && Keys.JwtConfig.UpdateUserOnLogin { // Update Existing User if err := r.UpdateUser(dbUser, tokenUser); err != nil { cclog.Errorf("Error while updating user '%s' to DB: %v", dbUser.Username, err) } @@ -185,11 +206,11 @@ func handleOIDCUser(OIDCUser *schema.User) { if err != nil && err != sql.ErrNoRows { cclog.Errorf("Error while loading user '%s': %v", OIDCUser.Username, err) - } else if err == sql.ErrNoRows && config.Keys.OpenIDConfig.SyncUserOnLogin { // Adds New User + } else if err == sql.ErrNoRows && Keys.OpenIDConfig.SyncUserOnLogin { // Adds New User if err := r.AddUser(OIDCUser); err != nil { cclog.Errorf("Error while adding user '%s' to DB: %v", OIDCUser.Username, err) } - } else if err == nil && config.Keys.OpenIDConfig.UpdateUserOnLogin { // Update Existing User + } else if err == nil && Keys.OpenIDConfig.UpdateUserOnLogin { // Update Existing User if err := r.UpdateUser(dbUser, OIDCUser); err != nil { cclog.Errorf("Error while updating user '%s' to DB: %v", dbUser.Username, err) } diff --git a/internal/auth/jwt.go b/internal/auth/jwt.go index 2cc2c37..91f92b5 100644 --- a/internal/auth/jwt.go +++ b/internal/auth/jwt.go @@ -13,13 +13,34 @@ import ( "strings" "time" - "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/repository" cclog "github.com/ClusterCockpit/cc-lib/ccLogger" "github.com/ClusterCockpit/cc-lib/schema" "github.com/golang-jwt/jwt/v5" ) +type JWTAuthConfig struct { + // Specifies for how long a JWT token shall be valid + // as a string parsable by time.ParseDuration(). + MaxAge string `json:"max-age"` + + // Specifies which cookie should be checked for a JWT token (if no authorization header is present) + CookieName string `json:"cookieName"` + + // Deny login for users not in database (but defined in JWT). + // Ignore user roles defined in JWTs ('roles' claim), get them from db. + ValidateUser bool `json:"validateUser"` + + // Specifies which issuer should be accepted when validating external JWTs ('iss' claim) + TrustedIssuer string `json:"trustedIssuer"` + + // Should an non-existent user be added to the DB based on the information in the token + SyncUserOnLogin bool `json:"syncUserOnLogin"` + + // Should an existent user be updated in the DB based on the information in the token + UpdateUserOnLogin bool `json:"updateUserOnLogin"` +} + type JWTAuthenticator struct { publicKey ed25519.PublicKey privateKey ed25519.PrivateKey @@ -62,7 +83,7 @@ func (ja *JWTAuthenticator) AuthViaJWT( return nil, nil } - token, err := jwt.Parse(rawtoken, func(t *jwt.Token) (interface{}, error) { + token, err := jwt.Parse(rawtoken, func(t *jwt.Token) (any, error) { if t.Method != jwt.SigningMethodEdDSA { return nil, errors.New("only Ed25519/EdDSA supported") } @@ -85,7 +106,7 @@ func (ja *JWTAuthenticator) AuthViaJWT( var roles []string // Validate user + roles from JWT against database? - if config.Keys.JwtConfig.ValidateUser { + if Keys.JwtConfig.ValidateUser { ur := repository.GetUserRepository() user, err := ur.GetUser(sub) // Deny any logins for unknown usernames @@ -97,7 +118,7 @@ func (ja *JWTAuthenticator) AuthViaJWT( roles = user.Roles } else { // Extract roles from JWT (if present) - if rawroles, ok := claims["roles"].([]interface{}); ok { + if rawroles, ok := claims["roles"].([]any); ok { for _, rr := range rawroles { if r, ok := rr.(string); ok { roles = append(roles, r) @@ -126,8 +147,8 @@ func (ja *JWTAuthenticator) ProvideJWT(user *schema.User) (string, error) { "roles": user.Roles, "iat": now.Unix(), } - if config.Keys.JwtConfig.MaxAge != "" { - d, err := time.ParseDuration(config.Keys.JwtConfig.MaxAge) + if Keys.JwtConfig.MaxAge != "" { + d, err := time.ParseDuration(Keys.JwtConfig.MaxAge) if err != nil { return "", errors.New("cannot parse max-age config key") } diff --git a/internal/auth/jwtCookieSession.go b/internal/auth/jwtCookieSession.go index 8f6d064..590a408 100644 --- a/internal/auth/jwtCookieSession.go +++ b/internal/auth/jwtCookieSession.go @@ -13,7 +13,6 @@ import ( "net/http" "os" - "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/repository" cclog "github.com/ClusterCockpit/cc-lib/ccLogger" "github.com/ClusterCockpit/cc-lib/schema" @@ -63,17 +62,16 @@ func (ja *JWTCookieSessionAuthenticator) Init() error { return errors.New("environment variable 'CROSS_LOGIN_JWT_PUBLIC_KEY' not set (cross login token based authentication will not work)") } - jc := config.Keys.JwtConfig // Warn if other necessary settings are not configured - if jc != nil { - if jc.CookieName == "" { + if Keys.JwtConfig != nil { + if Keys.JwtConfig.CookieName == "" { cclog.Info("cookieName for JWTs not configured (cross login via JWT cookie will fail)") return errors.New("cookieName for JWTs not configured (cross login via JWT cookie will fail)") } - if !jc.ValidateUser { + if !Keys.JwtConfig.ValidateUser { cclog.Info("forceJWTValidationViaDatabase not set to true: CC will accept users and roles defined in JWTs regardless of its own database!") } - if jc.TrustedIssuer == "" { + if Keys.JwtConfig.TrustedIssuer == "" { cclog.Info("trustedExternalIssuer for JWTs not configured (cross login via JWT cookie will fail)") return errors.New("trustedExternalIssuer for JWTs not configured (cross login via JWT cookie will fail)") } @@ -92,7 +90,7 @@ func (ja *JWTCookieSessionAuthenticator) CanLogin( rw http.ResponseWriter, r *http.Request, ) (*schema.User, bool) { - jc := config.Keys.JwtConfig + jc := Keys.JwtConfig cookieName := "" if jc.CookieName != "" { cookieName = jc.CookieName @@ -115,7 +113,7 @@ func (ja *JWTCookieSessionAuthenticator) Login( rw http.ResponseWriter, r *http.Request, ) (*schema.User, error) { - jc := config.Keys.JwtConfig + jc := Keys.JwtConfig jwtCookie, err := r.Cookie(jc.CookieName) var rawtoken string @@ -123,7 +121,7 @@ func (ja *JWTCookieSessionAuthenticator) Login( rawtoken = jwtCookie.Value } - token, err := jwt.Parse(rawtoken, func(t *jwt.Token) (interface{}, error) { + token, err := jwt.Parse(rawtoken, func(t *jwt.Token) (any, error) { if t.Method != jwt.SigningMethodEdDSA { return nil, errors.New("only Ed25519/EdDSA supported") } @@ -169,8 +167,8 @@ func (ja *JWTCookieSessionAuthenticator) Login( } } else { var name string - if wrap, ok := claims["name"].(map[string]interface{}); ok { - if vals, ok := wrap["values"].([]interface{}); ok { + if wrap, ok := claims["name"].(map[string]any); ok { + if vals, ok := wrap["values"].([]any); ok { if len(vals) != 0 { name = fmt.Sprintf("%v", vals[0]) @@ -182,7 +180,7 @@ func (ja *JWTCookieSessionAuthenticator) Login( } // Extract roles from JWT (if present) - if rawroles, ok := claims["roles"].([]interface{}); ok { + if rawroles, ok := claims["roles"].([]any); ok { for _, rr := range rawroles { if r, ok := rr.(string); ok { roles = append(roles, r) diff --git a/internal/auth/jwtSession.go b/internal/auth/jwtSession.go index 9c79e72..e0ca7a9 100644 --- a/internal/auth/jwtSession.go +++ b/internal/auth/jwtSession.go @@ -13,7 +13,6 @@ import ( "os" "strings" - "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/repository" cclog "github.com/ClusterCockpit/cc-lib/ccLogger" "github.com/ClusterCockpit/cc-lib/schema" @@ -60,7 +59,7 @@ func (ja *JWTSessionAuthenticator) Login( rawtoken = r.URL.Query().Get("login-token") } - token, err := jwt.Parse(rawtoken, func(t *jwt.Token) (interface{}, error) { + token, err := jwt.Parse(rawtoken, func(t *jwt.Token) (any, error) { if t.Method == jwt.SigningMethodHS256 || t.Method == jwt.SigningMethodHS512 { return ja.loginTokenKey, nil } @@ -82,7 +81,7 @@ func (ja *JWTSessionAuthenticator) Login( var roles []string projects := make([]string, 0) - if config.Keys.JwtConfig.ValidateUser { + if Keys.JwtConfig.ValidateUser { var err error user, err = repository.GetUserRepository().GetUser(sub) if err != nil && err != sql.ErrNoRows { @@ -96,8 +95,8 @@ func (ja *JWTSessionAuthenticator) Login( } } else { var name string - if wrap, ok := claims["name"].(map[string]interface{}); ok { - if vals, ok := wrap["values"].([]interface{}); ok { + if wrap, ok := claims["name"].(map[string]any); ok { + if vals, ok := wrap["values"].([]any); ok { if len(vals) != 0 { name = fmt.Sprintf("%v", vals[0]) @@ -109,7 +108,7 @@ func (ja *JWTSessionAuthenticator) Login( } // Extract roles from JWT (if present) - if rawroles, ok := claims["roles"].([]interface{}); ok { + if rawroles, ok := claims["roles"].([]any); ok { for _, rr := range rawroles { if r, ok := rr.(string); ok { if schema.IsValidRole(r) { @@ -119,7 +118,7 @@ func (ja *JWTSessionAuthenticator) Login( } } - if rawprojs, ok := claims["projects"].([]interface{}); ok { + if rawprojs, ok := claims["projects"].([]any); ok { for _, pp := range rawprojs { if p, ok := pp.(string); ok { projects = append(projects, p) @@ -138,7 +137,7 @@ func (ja *JWTSessionAuthenticator) Login( AuthSource: schema.AuthViaToken, } - if config.Keys.JwtConfig.SyncUserOnLogin || config.Keys.JwtConfig.UpdateUserOnLogin { + if Keys.JwtConfig.SyncUserOnLogin || Keys.JwtConfig.UpdateUserOnLogin { handleTokenUser(user) } } diff --git a/internal/auth/ldap.go b/internal/auth/ldap.go index d7843e4..bf5dc7a 100644 --- a/internal/auth/ldap.go +++ b/internal/auth/ldap.go @@ -11,13 +11,26 @@ import ( "os" "strings" - "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/repository" cclog "github.com/ClusterCockpit/cc-lib/ccLogger" "github.com/ClusterCockpit/cc-lib/schema" "github.com/go-ldap/ldap/v3" ) +type LdapConfig struct { + Url string `json:"url"` + UserBase string `json:"user_base"` + SearchDN string `json:"search_dn"` + UserBind string `json:"user_bind"` + UserFilter string `json:"user_filter"` + UserAttr string `json:"username_attr"` + SyncInterval string `json:"sync_interval"` // Parsed using time.ParseDuration. + SyncDelOldUsers bool `json:"sync_del_old_users"` + + // Should an non-existent user be added to the DB if user exists in ldap directory + SyncUserOnLogin bool `json:"syncUserOnLogin"` +} + type LdapAuthenticator struct { syncPassword string UserAttr string @@ -31,10 +44,8 @@ func (la *LdapAuthenticator) Init() error { cclog.Warn("environment variable 'LDAP_ADMIN_PASSWORD' not set (ldap sync will not work)") } - lc := config.Keys.LdapConfig - - if lc.UserAttr != "" { - la.UserAttr = lc.UserAttr + if Keys.LdapConfig.UserAttr != "" { + la.UserAttr = Keys.LdapConfig.UserAttr } else { la.UserAttr = "gecos" } @@ -48,7 +59,7 @@ func (la *LdapAuthenticator) CanLogin( rw http.ResponseWriter, r *http.Request, ) (*schema.User, bool) { - lc := config.Keys.LdapConfig + lc := Keys.LdapConfig if user != nil { if user.AuthSource == schema.AuthViaLDAP { @@ -119,7 +130,7 @@ func (la *LdapAuthenticator) Login( } defer l.Close() - userDn := strings.Replace(config.Keys.LdapConfig.UserBind, "{username}", user.Username, -1) + userDn := strings.Replace(Keys.LdapConfig.UserBind, "{username}", user.Username, -1) if err := l.Bind(userDn, r.FormValue("password")); err != nil { cclog.Errorf("AUTH/LDAP > Authentication for user %s failed: %v", user.Username, err) @@ -134,7 +145,7 @@ func (la *LdapAuthenticator) Sync() error { const IN_LDAP int = 2 const IN_BOTH int = 3 ur := repository.GetUserRepository() - lc := config.Keys.LdapConfig + lc := Keys.LdapConfig users := map[string]int{} usernames, err := ur.GetLdapUsernames() @@ -210,7 +221,7 @@ func (la *LdapAuthenticator) Sync() error { } func (la *LdapAuthenticator) getLdapConnection(admin bool) (*ldap.Conn, error) { - lc := config.Keys.LdapConfig + lc := Keys.LdapConfig conn, err := ldap.DialURL(lc.Url) if err != nil { cclog.Warn("LDAP URL dial failed") diff --git a/internal/auth/oidc.go b/internal/auth/oidc.go index f688aab..fd856fb 100644 --- a/internal/auth/oidc.go +++ b/internal/auth/oidc.go @@ -13,7 +13,6 @@ import ( "os" "time" - "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/repository" cclog "github.com/ClusterCockpit/cc-lib/ccLogger" "github.com/ClusterCockpit/cc-lib/schema" @@ -22,6 +21,12 @@ import ( "golang.org/x/oauth2" ) +type OpenIDConfig struct { + Provider string `json:"provider"` + SyncUserOnLogin bool `json:"syncUserOnLogin"` + UpdateUserOnLogin bool `json:"updateUserOnLogin"` +} + type OIDC struct { client *oauth2.Config provider *oidc.Provider @@ -49,7 +54,7 @@ func setCallbackCookie(w http.ResponseWriter, r *http.Request, name, value strin } func NewOIDC(a *Authentication) *OIDC { - provider, err := oidc.NewProvider(context.Background(), config.Keys.OpenIDConfig.Provider) + provider, err := oidc.NewProvider(context.Background(), Keys.OpenIDConfig.Provider) if err != nil { cclog.Fatal(err) } @@ -168,7 +173,7 @@ func (oa *OIDC) OAuth2Callback(rw http.ResponseWriter, r *http.Request) { AuthSource: schema.AuthViaOIDC, } - if config.Keys.OpenIDConfig.SyncUserOnLogin || config.Keys.OpenIDConfig.UpdateUserOnLogin { + if Keys.OpenIDConfig.SyncUserOnLogin || Keys.OpenIDConfig.UpdateUserOnLogin { handleOIDCUser(user) } diff --git a/internal/auth/schema.go b/internal/auth/schema.go new file mode 100644 index 0000000..121cc06 --- /dev/null +++ b/internal/auth/schema.go @@ -0,0 +1,95 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +package auth + +var configSchema = ` + { + "jwts": { + "description": "For JWT token authentication.", + "type": "object", + "properties": { + "max-age": { + "description": "Configure how long a token is valid. As string parsable by time.ParseDuration()", + "type": "string" + }, + "cookieName": { + "description": "Cookie that should be checked for a JWT token.", + "type": "string" + }, + "validateUser": { + "description": "Deny login for users not in database (but defined in JWT). Overwrite roles in JWT with database roles.", + "type": "boolean" + }, + "trustedIssuer": { + "description": "Issuer that should be accepted when validating external JWTs ", + "type": "string" + }, + "syncUserOnLogin": { + "description": "Add non-existent user to DB at login attempt with values provided in JWT.", + "type": "boolean" + } + }, + "required": ["max-age"] + }, + "oidc": { + "provider": { + "description": "", + "type": "string" + }, + "syncUserOnLogin": { + "description": "", + "type": "boolean" + }, + "updateUserOnLogin": { + "description": "", + "type": "boolean" + }, + "required": ["provider"] + }, + "ldap": { + "description": "For LDAP Authentication and user synchronisation.", + "type": "object", + "properties": { + "url": { + "description": "URL of LDAP directory server.", + "type": "string" + }, + "user_base": { + "description": "Base DN of user tree root.", + "type": "string" + }, + "search_dn": { + "description": "DN for authenticating LDAP admin account with general read rights.", + "type": "string" + }, + "user_bind": { + "description": "Expression used to authenticate users via LDAP bind. Must contain uid={username}.", + "type": "string" + }, + "user_filter": { + "description": "Filter to extract users for syncing.", + "type": "string" + }, + "username_attr": { + "description": "Attribute with full username. Default: gecos", + "type": "string" + }, + "sync_interval": { + "description": "Interval used for syncing local user table with LDAP directory. Parsed using time.ParseDuration.", + "type": "string" + }, + "sync_del_old_users": { + "description": "Delete obsolete users in database.", + "type": "boolean" + }, + "syncUserOnLogin": { + "description": "Add non-existent user to DB at login attempt if user exists in Ldap directory", + "type": "boolean" + } + }, + "required": ["url", "user_base", "search_dn", "user_bind", "user_filter"] + }, + "required": ["jwts"] + }` diff --git a/internal/config/config.go b/internal/config/config.go index bb965b8..7332941 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -7,25 +7,123 @@ package config import ( "bytes" "encoding/json" - "os" + "time" cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" ) -var Keys schema.ProgramConfig = schema.ProgramConfig{ +type ResampleConfig struct { + // Array of resampling target resolutions, in seconds; Example: [600,300,60] + Resolutions []int `json:"resolutions"` + // Trigger next zoom level at less than this many visible datapoints + Trigger int `json:"trigger"` +} + +// Format of the configuration (file). See below for the defaults. +type ProgramConfig struct { + // Address where the http (or https) server will listen on (for example: 'localhost:80'). + Addr string `json:"addr"` + + // Addresses from which secured admin API endpoints can be reached, can be wildcard "*" + ApiAllowedIPs []string `json:"apiAllowedIPs"` + + // Drop root permissions once .env was read and the port was taken. + User string `json:"user"` + Group string `json:"group"` + + // Disable authentication (for everything: API, Web-UI, ...) + DisableAuthentication bool `json:"disable-authentication"` + + // If `embed-static-files` is true (default), the frontend files are directly + // embeded into the go binary and expected to be in web/frontend. Only if + // it is false the files in `static-files` are served instead. + EmbedStaticFiles bool `json:"embed-static-files"` + StaticFiles string `json:"static-files"` + + // 'sqlite3' or 'mysql' (mysql will work for mariadb as well) + DBDriver string `json:"db-driver"` + + // For sqlite3 a filename, for mysql a DSN in this format: https://github.com/go-sql-driver/mysql#dsn-data-source-name (Without query parameters!). + DB string `json:"db"` + + // Keep all metric data in the metric data repositories, + // do not write to the job-archive. + DisableArchive bool `json:"disable-archive"` + + EnableJobTaggers bool `json:"enable-job-taggers"` + + // Validate json input against schema + Validate bool `json:"validate"` + + // If 0 or empty, the session does not expire! + SessionMaxAge string `json:"session-max-age"` + + // If both those options are not empty, use HTTPS using those certificates. + HttpsCertFile string `json:"https-cert-file"` + HttpsKeyFile string `json:"https-key-file"` + + // If not the empty string and `addr` does not end in ":80", + // redirect every request incoming at port 80 to that url. + RedirectHttpTo string `json:"redirect-http-to"` + + // If overwritten, at least all the options in the defaults below must + // be provided! Most options here can be overwritten by the user. + UiDefaults map[string]any `json:"ui-defaults"` + + // Where to store MachineState files + MachineStateDir string `json:"machine-state-dir"` + + // If not zero, automatically mark jobs as stopped running X seconds longer than their walltime. + StopJobsExceedingWalltime int `json:"stop-jobs-exceeding-walltime"` + + // Defines time X in seconds in which jobs are considered to be "short" and will be filtered in specific views. + ShortRunningJobsDuration int `json:"short-running-jobs-duration"` + + // Energy Mix CO2 Emission Constant [g/kWh] + // If entered, displays estimated CO2 emission for job based on jobs totalEnergy + EmissionConstant int `json:"emission-constant"` + + // If exists, will enable dynamic zoom in frontend metric plots using the configured values + EnableResampling *ResampleConfig `json:"resampling"` +} + +type IntRange struct { + From int `json:"from"` + To int `json:"to"` +} + +type TimeRange struct { + From *time.Time `json:"from"` + To *time.Time `json:"to"` + Range string `json:"range,omitempty"` +} + +type FilterRanges struct { + Duration *IntRange `json:"duration"` + NumNodes *IntRange `json:"numNodes"` + StartTime *TimeRange `json:"startTime"` +} + +type ClusterConfig struct { + Name string `json:"name"` + FilterRanges *FilterRanges `json:"filterRanges"` + MetricDataRepository json.RawMessage `json:"metricDataRepository"` +} + +var Clusters []*ClusterConfig + +var Keys ProgramConfig = ProgramConfig{ Addr: "localhost:8080", DisableAuthentication: false, EmbedStaticFiles: true, DBDriver: "sqlite3", DB: "./var/job.db", - Archive: json.RawMessage(`{\"kind\":\"file\",\"path\":\"./var/job-archive\"}`), DisableArchive: false, Validate: false, SessionMaxAge: "168h", StopJobsExceedingWalltime: 0, ShortRunningJobsDuration: 5 * 60, - UiDefaults: map[string]interface{}{ + UiDefaults: map[string]any{ "analysis_view_histogramMetrics": []string{"flops_any", "mem_bw", "mem_used"}, "analysis_view_scatterPlotMetrics": [][]string{{"flops_any", "mem_bw"}, {"flops_any", "cpu_load"}, {"cpu_load", "mem_bw"}}, "job_view_nodestats_selectedMetrics": []string{"flops_any", "mem_bw", "mem_used"}, @@ -49,24 +147,22 @@ var Keys schema.ProgramConfig = schema.ProgramConfig{ }, } -func Init(flagConfigFile string) { - raw, err := os.ReadFile(flagConfigFile) - if err != nil { - if !os.IsNotExist(err) { - cclog.Abortf("Config Init: Could not read config file '%s'.\nError: %s\n", flagConfigFile, err.Error()) - } - } else { - if err := schema.Validate(schema.Config, bytes.NewReader(raw)); err != nil { - cclog.Abortf("Config Init: Could not validate config file '%s'.\nError: %s\n", flagConfigFile, err.Error()) - } - dec := json.NewDecoder(bytes.NewReader(raw)) - dec.DisallowUnknownFields() - if err := dec.Decode(&Keys); err != nil { - cclog.Abortf("Config Init: Could not decode config file '%s'.\nError: %s\n", flagConfigFile, err.Error()) - } +func Init(mainConfig json.RawMessage, clusterConfig json.RawMessage) { + Validate(configSchema, mainConfig) + dec := json.NewDecoder(bytes.NewReader(mainConfig)) + dec.DisallowUnknownFields() + if err := dec.Decode(&Keys); err != nil { + cclog.Abortf("Config Init: Could not decode config file '%s'.\nError: %s\n", mainConfig, err.Error()) + } - if Keys.Clusters == nil || len(Keys.Clusters) < 1 { - cclog.Abort("Config Init: At least one cluster required in config. Exited with error.") - } + Validate(clustersSchema, clusterConfig) + dec = json.NewDecoder(bytes.NewReader(clusterConfig)) + dec.DisallowUnknownFields() + if err := dec.Decode(&Clusters); err != nil { + cclog.Abortf("Config Init: Could not decode config file '%s'.\nError: %s\n", mainConfig, err.Error()) + } + + if Clusters == nil || len(Clusters) < 1 { + cclog.Abort("Config Init: At least one cluster required in config. Exited with error.") } } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 993b6f0..7dc76c3 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -6,11 +6,24 @@ package config import ( "testing" + + ccconf "github.com/ClusterCockpit/cc-lib/ccConfig" + cclog "github.com/ClusterCockpit/cc-lib/ccLogger" ) func TestInit(t *testing.T) { fp := "../../configs/config.json" - Init(fp) + ccconf.Init(fp) + if cfg := ccconf.GetPackageConfig("main"); cfg != nil { + if clustercfg := ccconf.GetPackageConfig("clusters"); clustercfg != nil { + Init(cfg, clustercfg) + } else { + cclog.Abort("Cluster configuration must be present") + } + } else { + cclog.Abort("Main configuration must be present") + } + if Keys.Addr != "0.0.0.0:443" { t.Errorf("wrong addr\ngot: %s \nwant: 0.0.0.0:443", Keys.Addr) } @@ -18,7 +31,17 @@ func TestInit(t *testing.T) { func TestInitMinimal(t *testing.T) { fp := "../../configs/config-demo.json" - Init(fp) + ccconf.Init(fp) + if cfg := ccconf.GetPackageConfig("main"); cfg != nil { + if clustercfg := ccconf.GetPackageConfig("clusters"); clustercfg != nil { + Init(cfg, clustercfg) + } else { + cclog.Abort("Cluster configuration must be present") + } + } else { + cclog.Abort("Main configuration must be present") + } + if Keys.Addr != "127.0.0.1:8080" { t.Errorf("wrong addr\ngot: %s \nwant: 127.0.0.1:8080", Keys.Addr) } diff --git a/internal/config/schema.go b/internal/config/schema.go new file mode 100644 index 0000000..37d662a --- /dev/null +++ b/internal/config/schema.go @@ -0,0 +1,199 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +package config + +var configSchema = ` + { + "type": "object", + "properties": { + "addr": { + "description": "Address where the http (or https) server will listen on (for example: 'localhost:80').", + "type": "string" + }, + "apiAllowedIPs": { + "description": "Addresses from which secured API endpoints can be reached", + "type": "array", + "items": { + "type": "string" + } + }, + "user": { + "description": "Drop root permissions once .env was read and the port was taken. Only applicable if using privileged port.", + "type": "string" + }, + "group": { + "description": "Drop root permissions once .env was read and the port was taken. Only applicable if using privileged port.", + "type": "string" + }, + "disable-authentication": { + "description": "Disable authentication (for everything: API, Web-UI, ...).", + "type": "boolean" + }, + "embed-static-files": { + "description": "If all files in web/frontend/public should be served from within the binary itself (they are embedded) or not.", + "type": "boolean" + }, + "static-files": { + "description": "Folder where static assets can be found, if embed-static-files is false.", + "type": "string" + }, + "db": { + "description": "For sqlite3 a filename, for mysql a DSN in this format: https://github.com/go-sql-driver/mysql#dsn-data-source-name (Without query parameters!).", + "type": "string" + }, + "disable-archive": { + "description": "Keep all metric data in the metric data repositories, do not write to the job-archive.", + "type": "boolean" + }, + "enable-job-taggers": { + "description": "Turn on automatic application and jobclass taggers", + "type": "boolean" + }, + "validate": { + "description": "Validate all input json documents against json schema.", + "type": "boolean" + }, + "session-max-age": { + "description": "Specifies for how long a session shall be valid as a string parsable by time.ParseDuration(). If 0 or empty, the session/token does not expire!", + "type": "string" + }, + "https-cert-file": { + "description": "Filepath to SSL certificate. If also https-key-file is set use HTTPS using those certificates.", + "type": "string" + }, + "https-key-file": { + "description": "Filepath to SSL key file. If also https-cert-file is set use HTTPS using those certificates.", + "type": "string" + }, + "redirect-http-to": { + "description": "If not the empty string and addr does not end in :80, redirect every request incoming at port 80 to that url.", + "type": "string" + }, + "stop-jobs-exceeding-walltime": { + "description": "If not zero, automatically mark jobs as stopped running X seconds longer than their walltime. Only applies if walltime is set for job.", + "type": "integer" + }, + "short-running-jobs-duration": { + "description": "Do not show running jobs shorter than X seconds.", + "type": "integer" + }, + "emission-constant": { + "description": ".", + "type": "integer" + }, + "cron-frequency": { + "description": "Frequency of cron job workers.", + "type": "object", + "properties": { + "duration-worker": { + "description": "Duration Update Worker [Defaults to '5m']", + "type": "string" + }, + "footprint-worker": { + "description": "Metric-Footprint Update Worker [Defaults to '10m']", + "type": "string" + } + } + }, + "enable-resampling": { + "description": "Enable dynamic zoom in frontend metric plots.", + "type": "object", + "properties": { + "trigger": { + "description": "Trigger next zoom level at less than this many visible datapoints.", + "type": "integer" + }, + "resolutions": { + "description": "Array of resampling target resolutions, in seconds.", + "type": "array", + "items": { + "type": "integer" + } + } + }, + "required": ["trigger", "resolutions"] + } + }, + "required": ["apiAllowedIPs"] + }` + +var clustersSchema = ` + { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "description": "The name of the cluster.", + "type": "string" + }, + "metricDataRepository": { + "description": "Type of the metric data repository for this cluster", + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": ["influxdb", "prometheus", "cc-metric-store", "test"] + }, + "url": { + "type": "string" + }, + "token": { + "type": "string" + } + }, + "required": ["kind", "url"] + }, + "filterRanges": { + "description": "This option controls the slider ranges for the UI controls of numNodes, duration, and startTime.", + "type": "object", + "properties": { + "numNodes": { + "description": "UI slider range for number of nodes", + "type": "object", + "properties": { + "from": { + "type": "integer" + }, + "to": { + "type": "integer" + } + }, + "required": ["from", "to"] + }, + "duration": { + "description": "UI slider range for duration", + "type": "object", + "properties": { + "from": { + "type": "integer" + }, + "to": { + "type": "integer" + } + }, + "required": ["from", "to"] + }, + "startTime": { + "description": "UI slider range for start time", + "type": "object", + "properties": { + "from": { + "type": "string", + "format": "date-time" + }, + "to": { + "type": "null" + } + }, + "required": ["from", "to"] + } + }, + "required": ["numNodes", "duration", "startTime"] + } + }, + "required": ["name", "metricDataRepository", "filterRanges"], + "minItems": 1 + }}` diff --git a/internal/config/validate.go b/internal/config/validate.go new file mode 100644 index 0000000..1614488 --- /dev/null +++ b/internal/config/validate.go @@ -0,0 +1,28 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +package config + +import ( + "encoding/json" + + cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + "github.com/santhosh-tekuri/jsonschema/v5" +) + +func Validate(schema string, instance json.RawMessage) { + sch, err := jsonschema.CompileString("schema.json", schema) + if err != nil { + cclog.Fatalf("%#v", err) + } + + var v any + if err := json.Unmarshal([]byte(instance), &v); err != nil { + cclog.Fatal(err) + } + + if err = sch.Validate(v); err != nil { + cclog.Fatalf("%#v", err) + } +} diff --git a/internal/importer/importer_test.go b/internal/importer/importer_test.go index 11e7afe..ff28619 100644 --- a/internal/importer/importer_test.go +++ b/internal/importer/importer_test.go @@ -16,6 +16,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/importer" "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/archive" + ccconf "github.com/ClusterCockpit/cc-lib/ccConfig" cclog "github.com/ClusterCockpit/cc-lib/ccLogger" ) @@ -36,18 +37,16 @@ func copyFile(s string, d string) error { func setup(t *testing.T) *repository.JobRepository { const testconfig = `{ + "main": { "addr": "0.0.0.0:8080", "validate": false, + "apiAllowedIPs": [ + "*" + ]}, "archive": { "kind": "file", "path": "./var/job-archive" }, - "jwts": { - "max-age": "2m" - }, - "apiAllowedIPs": [ - "*" - ], "clusters": [ { "name": "testcluster", @@ -108,7 +107,19 @@ func setup(t *testing.T) *repository.JobRepository { t.Fatal(err) } - config.Init(cfgFilePath) + ccconf.Init(cfgFilePath) + + // Load and check main configuration + if cfg := ccconf.GetPackageConfig("main"); cfg != nil { + if clustercfg := ccconf.GetPackageConfig("clusters"); clustercfg != nil { + config.Init(cfg, clustercfg) + } else { + t.Fatal("Cluster configuration must be present") + } + } else { + t.Fatal("Main configuration must be present") + } + archiveCfg := fmt.Sprintf("{\"kind\": \"file\",\"path\": \"%s\"}", jobarchive) if err := archive.Init(json.RawMessage(archiveCfg), config.Keys.DisableArchive); err != nil { diff --git a/internal/metricDataDispatcher/dataLoader.go b/internal/metricDataDispatcher/dataLoader.go index 6307843..2b73e11 100644 --- a/internal/metricDataDispatcher/dataLoader.go +++ b/internal/metricDataDispatcher/dataLoader.go @@ -41,7 +41,7 @@ func LoadData(job *schema.Job, ctx context.Context, resolution int, ) (schema.JobData, error) { - data := cache.Get(cacheKey(job, metrics, scopes, resolution), func() (_ interface{}, ttl time.Duration, size int) { + data := cache.Get(cacheKey(job, metrics, scopes, resolution), func() (_ any, ttl time.Duration, size int) { var jd schema.JobData var err error diff --git a/internal/metricdata/metricdata.go b/internal/metricdata/metricdata.go index aa3a87c..87867af 100644 --- a/internal/metricdata/metricdata.go +++ b/internal/metricdata/metricdata.go @@ -40,7 +40,7 @@ type MetricDataRepository interface { var metricDataRepos map[string]MetricDataRepository = map[string]MetricDataRepository{} func Init() error { - for _, cluster := range config.Keys.Clusters { + for _, cluster := range config.Clusters { if cluster.MetricDataRepository != nil { var kind struct { Kind string `json:"kind"` diff --git a/internal/repository/userConfig_test.go b/internal/repository/userConfig_test.go index d200763..b8cfd5e 100644 --- a/internal/repository/userConfig_test.go +++ b/internal/repository/userConfig_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/ClusterCockpit/cc-backend/internal/config" + ccconf "github.com/ClusterCockpit/cc-lib/ccConfig" cclog "github.com/ClusterCockpit/cc-lib/ccLogger" "github.com/ClusterCockpit/cc-lib/schema" _ "github.com/mattn/go-sqlite3" @@ -17,17 +18,16 @@ import ( func setupUserTest(t *testing.T) *UserCfgRepo { const testconfig = `{ - "addr": "0.0.0.0:8080", + "main": { + "addr": "0.0.0.0:8080", + "apiAllowedIPs": [ + "*" + ] + }, "archive": { "kind": "file", "path": "./var/job-archive" }, - "jwts": { - "max-age": "2m" - }, - "apiAllowedIPs": [ - "*" - ], "clusters": [ { "name": "testcluster", @@ -36,7 +36,8 @@ func setupUserTest(t *testing.T) *UserCfgRepo { "numNodes": { "from": 1, "to": 64 }, "duration": { "from": 0, "to": 86400 }, "startTime": { "from": "2022-01-01T00:00:00Z", "to": null } - } } ] + } + }] }` cclog.Init("info", true) @@ -53,7 +54,19 @@ func setupUserTest(t *testing.T) *UserCfgRepo { t.Fatal(err) } - config.Init(cfgFilePath) + ccconf.Init(cfgFilePath) + + // Load and check main configuration + if cfg := ccconf.GetPackageConfig("main"); cfg != nil { + if clustercfg := ccconf.GetPackageConfig("clusters"); clustercfg != nil { + config.Init(cfg, clustercfg) + } else { + t.Fatal("Cluster configuration must be present") + } + } else { + t.Fatal("Main configuration must be present") + } + return GetUserCfgRepo() } diff --git a/internal/taskManager/commitJobService.go b/internal/taskManager/commitJobService.go index 5489007..e7c169a 100644 --- a/internal/taskManager/commitJobService.go +++ b/internal/taskManager/commitJobService.go @@ -7,7 +7,6 @@ package taskManager import ( "time" - "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/repository" cclog "github.com/ClusterCockpit/cc-lib/ccLogger" "github.com/go-co-op/gocron/v2" @@ -15,8 +14,8 @@ import ( func RegisterCommitJobService() { var frequency string - if config.Keys.CronFrequency != nil && config.Keys.CronFrequency.CommitJobWorker != "" { - frequency = config.Keys.CronFrequency.CommitJobWorker + if Keys.CommitJobWorker != "" { + frequency = Keys.CommitJobWorker } else { frequency = "2m" } diff --git a/internal/taskManager/taskManager.go b/internal/taskManager/taskManager.go index 5f51040..7ed5aac 100644 --- a/internal/taskManager/taskManager.go +++ b/internal/taskManager/taskManager.go @@ -5,9 +5,11 @@ package taskManager import ( + "bytes" "encoding/json" "time" + "github.com/ClusterCockpit/cc-backend/internal/auth" "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/repository" cclog "github.com/ClusterCockpit/cc-lib/ccLogger" @@ -15,9 +17,26 @@ import ( "github.com/go-co-op/gocron/v2" ) +type Retention struct { + Policy string `json:"policy"` + Location string `json:"location"` + Age int `json:"age"` + IncludeDB bool `json:"includeDB"` +} + +type CronFrequency struct { + // Duration Update Worker [Defaults to '2m'] + CommitJobWorker string `json:"commit-job-worker"` + // Duration Update Worker [Defaults to '5m'] + DurationWorker string `json:"duration-worker"` + // Metric-Footprint Update Worker [Defaults to '10m'] + FootprintWorker string `json:"footprint-worker"` +} + var ( s gocron.Scheduler jobRepo *repository.JobRepository + Keys CronFrequency ) func parseDuration(s string) (time.Duration, error) { @@ -35,7 +54,7 @@ func parseDuration(s string) (time.Duration, error) { return interval, nil } -func Start() { +func Start(cronCfg, archiveConfig json.RawMessage) { var err error jobRepo = repository.GetJobRepository() s, err = gocron.NewScheduler() @@ -47,13 +66,19 @@ func Start() { RegisterStopJobsExceedTime() } + dec := json.NewDecoder(bytes.NewReader(cronCfg)) + dec.DisallowUnknownFields() + if err := dec.Decode(&Keys); err != nil { + cclog.Errorf("error while decoding ldap config: %v", err) + } + var cfg struct { Retention schema.Retention `json:"retention"` Compression int `json:"compression"` } cfg.Retention.IncludeDB = true - if err := json.Unmarshal(config.Keys.Archive, &cfg); err != nil { + if err := json.Unmarshal(archiveConfig, &cfg); err != nil { cclog.Warn("Error while unmarshaling raw config json") } @@ -73,7 +98,7 @@ func Start() { RegisterCompressionService(cfg.Compression) } - lc := config.Keys.LdapConfig + lc := auth.Keys.LdapConfig if lc != nil && lc.SyncInterval != "" { RegisterLdapSyncService(lc.SyncInterval) diff --git a/internal/taskManager/updateDurationService.go b/internal/taskManager/updateDurationService.go index 70ec506..d650afb 100644 --- a/internal/taskManager/updateDurationService.go +++ b/internal/taskManager/updateDurationService.go @@ -7,15 +7,14 @@ package taskManager import ( "time" - "github.com/ClusterCockpit/cc-backend/internal/config" cclog "github.com/ClusterCockpit/cc-lib/ccLogger" "github.com/go-co-op/gocron/v2" ) func RegisterUpdateDurationWorker() { var frequency string - if config.Keys.CronFrequency != nil && config.Keys.CronFrequency.DurationWorker != "" { - frequency = config.Keys.CronFrequency.DurationWorker + if Keys.DurationWorker != "" { + frequency = Keys.DurationWorker } else { frequency = "5m" } diff --git a/internal/taskManager/updateFootprintService.go b/internal/taskManager/updateFootprintService.go index 41c5837..4025849 100644 --- a/internal/taskManager/updateFootprintService.go +++ b/internal/taskManager/updateFootprintService.go @@ -9,7 +9,6 @@ import ( "math" "time" - "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/metricdata" "github.com/ClusterCockpit/cc-backend/pkg/archive" cclog "github.com/ClusterCockpit/cc-lib/ccLogger" @@ -20,8 +19,8 @@ import ( func RegisterFootprintWorker() { var frequency string - if config.Keys.CronFrequency != nil && config.Keys.CronFrequency.FootprintWorker != "" { - frequency = config.Keys.CronFrequency.FootprintWorker + if Keys.FootprintWorker != "" { + frequency = Keys.FootprintWorker } else { frequency = "10m" } diff --git a/pkg/archive/ConfigSchema.go b/pkg/archive/ConfigSchema.go new file mode 100644 index 0000000..d4721f6 --- /dev/null +++ b/pkg/archive/ConfigSchema.go @@ -0,0 +1,49 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +package archive + +var configSchema = ` + { + "type": "object", + "properties": { + "kind": { + "description": "Backend type for job-archive", + "type": "string", + "enum": ["file", "s3"] + }, + "path": { + "description": "Path to job archive for file backend", + "type": "string" + }, + "compression": { + "description": "Setup automatic compression for jobs older than number of days", + "type": "integer" + }, + "retention": { + "description": "Configuration keys for retention", + "type": "object", + "properties": { + "policy": { + "description": "Retention policy", + "type": "string", + "enum": ["none", "delete", "move"] + }, + "includeDB": { + "description": "Also remove jobs from database", + "type": "boolean" + }, + "age": { + "description": "Act on jobs with startTime older than age (in days)", + "type": "integer" + }, + "location": { + "description": "The target directory for retention. Only applicable for retention move.", + "type": "string" + } + }, + "required": ["policy"] + } + }, + "required": ["kind"]}` diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go index f69cde3..db6f1f8 100644 --- a/pkg/archive/archive.go +++ b/pkg/archive/archive.go @@ -10,6 +10,7 @@ import ( "maps" "sync" + "github.com/ClusterCockpit/cc-backend/internal/config" cclog "github.com/ClusterCockpit/cc-lib/ccLogger" "github.com/ClusterCockpit/cc-lib/lrucache" "github.com/ClusterCockpit/cc-lib/schema" @@ -74,6 +75,7 @@ func Init(rawConfig json.RawMessage, disableArchive bool) error { Kind string `json:"kind"` } + config.Validate(configSchema, rawConfig) if err = json.Unmarshal(rawConfig, &cfg); err != nil { cclog.Warn("Error while unmarshaling raw config json") return diff --git a/pkg/archive/fsBackend.go b/pkg/archive/fsBackend.go index 8f10360..53a374f 100644 --- a/pkg/archive/fsBackend.go +++ b/pkg/archive/fsBackend.go @@ -147,17 +147,17 @@ func loadJobStats(filename string, isCompressed bool) (schema.ScopedJobStats, er } func (fsa *FsArchive) Init(rawConfig json.RawMessage) (uint64, error) { - var config FsArchiveConfig - if err := json.Unmarshal(rawConfig, &config); err != nil { + var cfg FsArchiveConfig + if err := json.Unmarshal(rawConfig, &cfg); err != nil { cclog.Warnf("Init() > Unmarshal error: %#v", err) return 0, err } - if config.Path == "" { + if cfg.Path == "" { err := fmt.Errorf("Init() : empty config.Path") cclog.Errorf("Init() > config.Path error: %v", err) return 0, err } - fsa.path = config.Path + fsa.path = cfg.Path b, err := os.ReadFile(filepath.Join(fsa.path, "version.txt")) if err != nil { diff --git a/pkg/archive/fsBackend_test.go b/pkg/archive/fsBackend_test.go index 7b1fe74..cdc892f 100644 --- a/pkg/archive/fsBackend_test.go +++ b/pkg/archive/fsBackend_test.go @@ -1,5 +1,5 @@ // Copyright (C) NHR@FAU, University Erlangen-Nuremberg. -// All rights reserved. This file is part of cc-backend. +// All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. package archive diff --git a/pkg/runtimeEnv/setup.go b/pkg/runtimeEnv/setup.go deleted file mode 100644 index e23a004..0000000 --- a/pkg/runtimeEnv/setup.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. -// All rights reserved. This file is part of cc-backend. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. -package runtimeEnv - -import ( - "fmt" - "os" - "os/exec" - "os/user" - "strconv" - "syscall" - - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" -) - -// Changes the processes user and group to that -// specified in the config.json. The go runtime -// takes care of all threads (and not only the calling one) -// executing the underlying systemcall. -func DropPrivileges(username string, group string) error { - if group != "" { - g, err := user.LookupGroup(group) - if err != nil { - cclog.Warn("Error while looking up group") - return err - } - - gid, _ := strconv.Atoi(g.Gid) - if err := syscall.Setgid(gid); err != nil { - cclog.Warn("Error while setting gid") - return err - } - } - - if username != "" { - u, err := user.Lookup(username) - if err != nil { - cclog.Warn("Error while looking up user") - return err - } - - uid, _ := strconv.Atoi(u.Uid) - if err := syscall.Setuid(uid); err != nil { - cclog.Warn("Error while setting uid") - return err - } - } - - return nil -} - -// If started via systemd, inform systemd that we are running: -// https://www.freedesktop.org/software/systemd/man/sd_notify.html -func SystemdNotifiy(ready bool, status string) { - if os.Getenv("NOTIFY_SOCKET") == "" { - // Not started using systemd - return - } - - args := []string{fmt.Sprintf("--pid=%d", os.Getpid())} - if ready { - args = append(args, "--ready") - } - - if status != "" { - args = append(args, fmt.Sprintf("--status=%s", status)) - } - - cmd := exec.Command("systemd-notify", args...) - cmd.Run() // errors ignored on purpose, there is not much to do anyways. -} diff --git a/tools/archive-manager/main.go b/tools/archive-manager/main.go index 0cf5f98..5073d5d 100644 --- a/tools/archive-manager/main.go +++ b/tools/archive-manager/main.go @@ -13,6 +13,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/pkg/archive" + ccconf "github.com/ClusterCockpit/cc-lib/ccConfig" cclog "github.com/ClusterCockpit/cc-lib/ccLogger" ) @@ -47,7 +48,19 @@ func main() { archiveCfg := fmt.Sprintf("{\"kind\": \"file\",\"path\": \"%s\"}", srcPath) cclog.Init(flagLogLevel, flagLogDateTime) - config.Init(flagConfigFile) + + ccconf.Init(flagConfigFile) + + // Load and check main configuration + if cfg := ccconf.GetPackageConfig("main"); cfg != nil { + if clustercfg := ccconf.GetPackageConfig("clusters"); clustercfg != nil { + config.Init(cfg, clustercfg) + } else { + cclog.Abort("Cluster configuration must be present") + } + } else { + cclog.Abort("Main configuration must be present") + } if err := archive.Init(json.RawMessage(archiveCfg), false); err != nil { cclog.Fatal(err) diff --git a/web/web.go b/web/web.go index 7318284..0c8c160 100644 --- a/web/web.go +++ b/web/web.go @@ -90,12 +90,12 @@ type Page struct { User schema.User // Information about the currently logged in user (Full User Info) Roles map[string]schema.Role // Available roles for frontend render checks Build Build // Latest information about the application - Clusters []schema.ClusterConfig // List of all clusters for use in the Header + Clusters []config.ClusterConfig // List of all clusters for use in the Header SubClusters map[string][]string // Map per cluster of all subClusters for use in the Header FilterPresets map[string]interface{} // For pages with the Filter component, this can be used to set initial filters. Infos map[string]interface{} // For generic use (e.g. username for /monitoring/user/, job id for /monitoring/job/) Config map[string]interface{} // UI settings for the currently logged in user (e.g. line width, ...) - Resampling *schema.ResampleConfig // If not nil, defines resampling trigger and resolutions + Resampling *config.ResampleConfig // If not nil, defines resampling trigger and resolutions Redirect string // The originally requested URL, for intermediate login handling } @@ -106,8 +106,8 @@ func RenderTemplate(rw http.ResponseWriter, file string, page *Page) { } if page.Clusters == nil { - for _, c := range config.Keys.Clusters { - page.Clusters = append(page.Clusters, schema.ClusterConfig{Name: c.Name, FilterRanges: c.FilterRanges, MetricDataRepository: nil}) + for _, c := range config.Clusters { + page.Clusters = append(page.Clusters, config.ClusterConfig{Name: c.Name, FilterRanges: c.FilterRanges, MetricDataRepository: nil}) } }