From d61bf212f5aa0cc8fbec496cf49e70ccf3c00873 Mon Sep 17 00:00:00 2001 From: Michael Panzlaff Date: Mon, 3 Feb 2025 17:02:13 +0100 Subject: [PATCH 01/82] Fix 'make -B', don't fail if $(VAR) already exists --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 52f0d39..dd33827 100644 --- a/Makefile +++ b/Makefile @@ -82,7 +82,7 @@ tags: @ctags -R $(VAR): - @mkdir $(VAR) + @mkdir -p $(VAR) config.json: $(info ===> Initialize config.json file) From 0191bc3821a50cef12613a53bd6d210aa1500faf Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 25 Feb 2025 10:21:48 +0100 Subject: [PATCH 02/82] Annotate and review log functions, add stdout writers --- pkg/log/log.go | 229 ++++++++++++++++++++++++++++--------------------- 1 file changed, 129 insertions(+), 100 deletions(-) diff --git a/pkg/log/log.go b/pkg/log/log.go index a40c656..a4e7703 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -84,109 +84,138 @@ func Init(lvl string, logdate bool) { loglevel = lvl } -/* PRINT */ - -// Private helper -func printStr(v ...interface{}) string { - return fmt.Sprint(v...) -} - -// Uses Info() -> If errorpath required at some point: -// Will need own writer with 'Output(2, out)' to correctly render path -func Print(v ...interface{}) { - Info(v...) -} - -func Debug(v ...interface{}) { - DebugLog.Output(2, printStr(v...)) -} - -func Info(v ...interface{}) { - InfoLog.Output(2, printStr(v...)) -} - -func Warn(v ...interface{}) { - WarnLog.Output(2, printStr(v...)) -} - -func Error(v ...interface{}) { - ErrLog.Output(2, printStr(v...)) -} - -// Writes panic stacktrace, but keeps application alive -func Panic(v ...interface{}) { - ErrLog.Output(2, printStr(v...)) - panic("Panic triggered ...") -} - -func Crit(v ...interface{}) { - CritLog.Output(2, printStr(v...)) -} - -// Writes critical log, stops application -func Fatal(v ...interface{}) { - CritLog.Output(2, printStr(v...)) - os.Exit(1) -} - -/* PRINT FORMAT*/ - -// Private helper -func printfStr(format string, v ...interface{}) string { - return fmt.Sprintf(format, v...) -} - -// Uses Infof() -> If errorpath required at some point: -// Will need own writer with 'Output(2, out)' to correctly render path -func Printf(format string, v ...interface{}) { - Infof(format, v...) -} - -func Debugf(format string, v ...interface{}) { - DebugLog.Output(2, printfStr(format, v...)) -} - -func Infof(format string, v ...interface{}) { - InfoLog.Output(2, printfStr(format, v...)) -} - -func Warnf(format string, v ...interface{}) { - WarnLog.Output(2, printfStr(format, v...)) -} - -func Errorf(format string, v ...interface{}) { - ErrLog.Output(2, printfStr(format, v...)) -} - -// Writes panic stacktrace, but keeps application alive -func Panicf(format string, v ...interface{}) { - ErrLog.Output(2, printfStr(format, v...)) - panic("Panic triggered ...") -} - -func Critf(format string, v ...interface{}) { - CritLog.Output(2, printfStr(format, v...)) -} - -// Writes crit log, stops application -func Fatalf(format string, v ...interface{}) { - CritLog.Output(2, printfStr(format, v...)) - os.Exit(1) -} +/* HELPER */ func Loglevel() string { return loglevel } -/* SPECIAL */ +/* PRIVATE HELPER */ -// func Finfof(w io.Writer, format string, v ...interface{}) { -// if w != io.Discard { -// if logDateTime { -// currentTime := time.Now() -// fmt.Fprintf(InfoWriter, currentTime.String()+InfoPrefix+format+"\n", v...) -// } else { -// fmt.Fprintf(InfoWriter, InfoPrefix+format+"\n", v...) -// } -// } -// } +// Return unformatted string +func printStr(v ...interface{}) string { + return fmt.Sprint(v...) +} + +// Return formatted string +func printfStr(format string, v ...interface{}) string { + return fmt.Sprintf(format, v...) +} + +/* PRINT */ + +// Prints to STDOUT without string formatting; application continues. +// Used for special cases not requiring log information like date or location. +func Print(v ...interface{}) { + fmt.Fprint(os.Stdout, v...) +} + +// Prints to STDOUT without string formatting; application exits with error code 0. +// Used for exiting succesfully with message after expected outcome, e.g. successful single-call application runs. +func Exit(v ...interface{}) { + fmt.Fprint(os.Stdout, v...) + os.Exit(0) +} + +// Prints to STDOUT without string formatting; application exits with error code 1. +// Used for terminating with message after to be expected errors, e.g. wrong arguments or during init(). +func Abort(v ...interface{}) { + fmt.Fprint(os.Stdout, v...) + os.Exit(1) +} + +// Prints to DEBUG writer without string formatting; application continues. +// Used for logging additional information, primarily for development. +func Debug(v ...interface{}) { + DebugLog.Output(2, printStr(v...)) +} + +// Prints to INFO writer without string formatting; application continues. +// Used for logging additional information, e.g. notable returns or common fail-cases. +func Info(v ...interface{}) { + InfoLog.Output(2, printStr(v...)) +} + +// Prints to WARNING writer without string formatting; application continues. +// Used for logging important information, e.g. uncommon edge-cases or administration related information. +func Warn(v ...interface{}) { + WarnLog.Output(2, printStr(v...)) +} + +// Prints to ERROR writer without string formatting; application continues. +// Used for logging errors, but code still can return default(s) or nil. +func Error(v ...interface{}) { + ErrLog.Output(2, printStr(v...)) +} + +// Prints to CRITICAL writer without string formatting; application exits with error code 1. +// Used for terminating on unexpected errors with date and code location. +func Fatal(v ...interface{}) { + CritLog.Output(2, printStr(v...)) + os.Exit(1) +} + +// Prints to PANIC function without string formatting; application exits with panic. +// Used for terminating on unexpected errors with stacktrace. +func Panic(v ...interface{}) { + panic(printStr(v...)) +} + +/* PRINT FORMAT*/ + +// Prints to STDOUT with string formatting; application continues. +// Used for special cases not requiring log information like date or location. +func Printf(format string, v ...interface{}) { + fmt.Fprintf(os.Stdout, format, v...) +} + +// Prints to STDOUT with string formatting; application exits with error code 0. +// Used for exiting succesfully with message after expected outcome, e.g. successful single-call application runs. +func Exitf(format string, v ...interface{}) { + fmt.Fprintf(os.Stdout, format, v...) + os.Exit(0) +} + +// Prints to STDOUT with string formatting; application exits with error code 1. +// Used for terminating with message after to be expected errors, e.g. wrong arguments or during init(). +func Abortf(format string, v ...interface{}) { + fmt.Fprintf(os.Stdout, format, v...) + os.Exit(1) +} + +// Prints to DEBUG writer with string formatting; application continues. +// Used for logging additional information, primarily for development. +func Debugf(format string, v ...interface{}) { + DebugLog.Output(2, printfStr(format, v...)) +} + +// Prints to INFO writer with string formatting; application continues. +// Used for logging additional information, e.g. notable returns or common fail-cases. +func Infof(format string, v ...interface{}) { + InfoLog.Output(2, printfStr(format, v...)) +} + +// Prints to WARNING writer with string formatting; application continues. +// Used for logging important information, e.g. uncommon edge-cases or administration related information. +func Warnf(format string, v ...interface{}) { + WarnLog.Output(2, printfStr(format, v...)) +} + +// Prints to ERROR writer with string formatting; application continues. +// Used for logging errors, but code still can return default(s) or nil. +func Errorf(format string, v ...interface{}) { + ErrLog.Output(2, printfStr(format, v...)) +} + +// Prints to CRITICAL writer with string formatting; application exits with error code 1. +// Used for terminating on unexpected errors with date and code location. +func Fatalf(format string, v ...interface{}) { + CritLog.Output(2, printfStr(format, v...)) + os.Exit(1) +} + +// Prints to PANIC function with string formatting; application exits with panic. +// Used for terminating on unexpected errors with stacktrace. +func Panicf(format string, v ...interface{}) { + panic(printfStr(format, v...)) +} From 632b9fc5eaacda5a5ca2388777f4f441f81856a0 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 26 Feb 2025 12:54:50 +0100 Subject: [PATCH 03/82] Prepare Bugfix release 1.4.3 --- Makefile | 2 +- ReleaseNotes.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index dd33827..0721fc4 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ TARGET = ./cc-backend VAR = ./var CFG = config.json .env FRONTEND = ./web/frontend -VERSION = 1.4.2 +VERSION = 1.4.3 GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development') CURRENT_TIME = $(shell date +"%Y-%m-%d:T%H:%M:%S") LD_FLAGS = '-s -X main.date=${CURRENT_TIME} -X main.version=${VERSION} -X main.commit=${GIT_HASH}' diff --git a/ReleaseNotes.md b/ReleaseNotes.md index 2659964..0015727 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -1,8 +1,8 @@ -# `cc-backend` version 1.4.2 +# `cc-backend` version 1.4.3 Supports job archive version 2 and database version 8. -This is a small bug fix release of `cc-backend`, the API backend and frontend +This is a bug fix release of `cc-backend`, the API backend and frontend implementation of ClusterCockpit. For release specific notes visit the [ClusterCockpit Documentation](https://clusterockpit.org/docs/release/). From d20954796802bb9f51d246eb18090c193f68c4d1 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 26 Feb 2025 14:40:54 +0100 Subject: [PATCH 04/82] Remove dedicated fatal loglevel, change to Fprintln for unformatted --- cmd/cc-backend/cli.go | 8 ++++---- pkg/log/log.go | 13 ++++++------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/cmd/cc-backend/cli.go b/cmd/cc-backend/cli.go index 8bc6681..8d9e7e6 100644 --- a/cmd/cc-backend/cli.go +++ b/cmd/cc-backend/cli.go @@ -12,7 +12,7 @@ var ( ) func cliInit() { - flag.BoolVar(&flagInit, "init", false, "Setup var directory, initialize swlite database file, config.json and .env") + flag.BoolVar(&flagInit, "init", false, "Setup var directory, initialize sqlite database file, config.json and .env") flag.BoolVar(&flagReinitDB, "init-db", false, "Go through job-archive and re-initialize the 'job', 'tag', and 'jobtag' tables (all running jobs will be lost!)") flag.BoolVar(&flagSyncLDAP, "sync-ldap", false, "Sync the 'hpc_user' table with ldap") flag.BoolVar(&flagServer, "server", false, "Start a server, continues listening on port after initialization and argument handling") @@ -24,10 +24,10 @@ func cliInit() { flag.BoolVar(&flagForceDB, "force-db", false, "Force database version, clear dirty flag and exit") flag.BoolVar(&flagLogDateTime, "logdate", false, "Set this flag to add date and time to log messages") flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`") - flag.StringVar(&flagNewUser, "add-user", "", "Add a new user. Argument format: `:[admin,support,manager,api,user]:`") - flag.StringVar(&flagDelUser, "del-user", "", "Remove user by `username`") + flag.StringVar(&flagNewUser, "add-user", "", "Add a new user. Argument format: :[admin,support,manager,api,user]:") + flag.StringVar(&flagDelUser, "del-user", "", "Remove a existing user. Argument format: ") flag.StringVar(&flagGenJWT, "jwt", "", "Generate and print a JWT for the user specified by its `username`") flag.StringVar(&flagImportJob, "import-job", "", "Import a job. Argument format: `:,...`") - flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug,info,warn (default),err,fatal,crit]`") + flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug, info (default), warn, err, crit]`") flag.Parse() } diff --git a/pkg/log/log.go b/pkg/log/log.go index a4e7703..ef14535 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -46,12 +46,12 @@ var loglevel string = "info" /* CONFIG */ func Init(lvl string, logdate bool) { - + // Discard I/O for all writers below selected loglevel; is always written. switch lvl { case "crit": ErrWriter = io.Discard fallthrough - case "err", "fatal": + case "err": WarnWriter = io.Discard fallthrough case "warn": @@ -63,8 +63,7 @@ func Init(lvl string, logdate bool) { // Nothing to do... break default: - fmt.Printf("pkg/log: Flag 'loglevel' has invalid value %#v\npkg/log: Will use default loglevel 'debug'\n", lvl) - //SetLogLevel("debug") + fmt.Printf("pkg/log: Flag 'loglevel' has invalid value %#v\npkg/log: Will use default loglevel '%s'\n", lvl, loglevel) } if !logdate { @@ -107,20 +106,20 @@ func printfStr(format string, v ...interface{}) string { // Prints to STDOUT without string formatting; application continues. // Used for special cases not requiring log information like date or location. func Print(v ...interface{}) { - fmt.Fprint(os.Stdout, v...) + fmt.Fprintln(os.Stdout, v...) } // Prints to STDOUT without string formatting; application exits with error code 0. // Used for exiting succesfully with message after expected outcome, e.g. successful single-call application runs. func Exit(v ...interface{}) { - fmt.Fprint(os.Stdout, v...) + fmt.Fprintln(os.Stdout, v...) os.Exit(0) } // Prints to STDOUT without string formatting; application exits with error code 1. // Used for terminating with message after to be expected errors, e.g. wrong arguments or during init(). func Abort(v ...interface{}) { - fmt.Fprint(os.Stdout, v...) + fmt.Fprintln(os.Stdout, v...) os.Exit(1) } From fc0c76bd77803495a05d9d995627302ab3400490 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 26 Feb 2025 15:20:25 +0100 Subject: [PATCH 05/82] Apply new log funtion to init and main, review or add logtexts --- cmd/cc-backend/init.go | 12 +++---- cmd/cc-backend/main.go | 75 ++++++++++++++++++++++++------------------ 2 files changed, 48 insertions(+), 39 deletions(-) diff --git a/cmd/cc-backend/init.go b/cmd/cc-backend/init.go index 5a00a11..f899ec1 100644 --- a/cmd/cc-backend/init.go +++ b/cmd/cc-backend/init.go @@ -5,7 +5,6 @@ package main import ( - "fmt" "os" "github.com/ClusterCockpit/cc-backend/internal/repository" @@ -62,24 +61,23 @@ const configString = ` func initEnv() { if util.CheckFileExists("var") { - fmt.Print("Directory ./var already exists. Exiting!\n") - os.Exit(0) + log.Exit("Directory ./var already exists. Cautiously exiting application initialization.") } if err := os.WriteFile("config.json", []byte(configString), 0o666); err != nil { - log.Fatalf("Writing config.json failed: %s", err.Error()) + log.Abortf("Could not write default ./config.json with permissions '0o666'. Application initialization failed, exited.\nError: %s\n", err.Error()) } if err := os.WriteFile(".env", []byte(envString), 0o666); err != nil { - log.Fatalf("Writing .env failed: %s", err.Error()) + log.Abortf("Could not write default ./.env file with permissions '0o666'. Application initialization failed, exited.\nError: %s\n", err.Error()) } if err := os.Mkdir("var", 0o777); err != nil { - log.Fatalf("Mkdir var failed: %s", err.Error()) + log.Abortf("Could not create default ./var folder with permissions '0o777'. Application initialization failed, exited.\nError: %s\n", err.Error()) } err := repository.MigrateDB("sqlite3", "./var/job.db") if err != nil { - log.Fatalf("Initialize job.db failed: %s", err.Error()) + log.Abortf("Could not initialize default sqlite3 database as './var/job.db'. Application initialization failed, exited.\nError: %s\n", err.Error()) } } diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index 33bab07..62a9b9b 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -61,15 +61,23 @@ func main() { // Apply config flags for pkg/log log.Init(flagLogLevel, flagLogDateTime) + // If init flag set, run tasks here before any file dependencies cause errors + if flagInit { + initEnv() + log.Exit("Successfully setup environment!\n" + + "Please review config.json and .env and adjust it to your needs.\n" + + "Add your job-archive at ./var/job-archive.") + } + // See https://github.com/google/gops (Runtime overhead is almost zero) if flagGops { if err := agent.Listen(agent.Options{}); err != nil { - log.Fatalf("gops/agent.Listen failed: %s", err.Error()) + log.Abortf("Could not start gops agent with 'gops/agent.Listen(agent.Options{})'. Application startup failed, exited.\nError: %s\n", err.Error()) } } if err := runtimeEnv.LoadEnv("./.env"); err != nil && !os.IsNotExist(err) { - log.Fatalf("parsing './.env' file failed: %s", err.Error()) + log.Abortf("Could not parse existing .env file at location './.env'. Application startup failed, exited.\nError: %s\n", err.Error()) } // Initialize sub-modules and handle command line flags. @@ -87,37 +95,29 @@ func main() { if flagMigrateDB { err := repository.MigrateDB(config.Keys.DBDriver, config.Keys.DB) if err != nil { - log.Fatal(err) + log.Abortf("MigrateDB Failed: Could not migrate '%s' database at location '%s' to version %d.\nError: %s\n", config.Keys.DBDriver, config.Keys.DB, repository.Version, err.Error()) } - os.Exit(0) + log.Exitf("MigrateDB Success: Migrated '%s' database at location '%s' to version %d.\n", config.Keys.DBDriver, config.Keys.DB, repository.Version) } if flagRevertDB { err := repository.RevertDB(config.Keys.DBDriver, config.Keys.DB) if err != nil { - log.Fatal(err) + log.Abortf("RevertDB Failed: Could not revert '%s' database at location '%s' to version %d.\nError: %s\n", config.Keys.DBDriver, config.Keys.DB, (repository.Version - 1), err.Error()) } - os.Exit(0) + log.Exitf("RevertDB Success: Reverted '%s' database at location '%s' to version %d.\n", config.Keys.DBDriver, config.Keys.DB, (repository.Version - 1)) } if flagForceDB { err := repository.ForceDB(config.Keys.DBDriver, config.Keys.DB) if err != nil { - log.Fatal(err) + log.Abortf("ForceDB Failed: Could not force '%s' database at location '%s' to version %d.\nError: %s\n", config.Keys.DBDriver, config.Keys.DB, repository.Version, err.Error()) } - os.Exit(0) + log.Exitf("ForceDB Success: Forced '%s' database at location '%s' to version %d.\n", config.Keys.DBDriver, config.Keys.DB, repository.Version) } repository.Connect(config.Keys.DBDriver, config.Keys.DB) - if flagInit { - initEnv() - fmt.Print("Successfully setup environment!\n") - fmt.Print("Please review config.json and .env and adjust it to your needs.\n") - fmt.Print("Add your job-archive at ./var/job-archive.\n") - os.Exit(0) - } - if !config.Keys.DisableAuthentication { auth.Init() @@ -125,20 +125,27 @@ func main() { if flagNewUser != "" { parts := strings.SplitN(flagNewUser, ":", 3) if len(parts) != 3 || len(parts[0]) == 0 { - log.Fatal("invalid argument format for user creation") + log.Abortf("Add User: Could not parse supplied argument format: No changes.\n"+ + "Want: :[admin,support,manager,api,user]:\n"+ + "Have: %s\n", flagNewUser) } ur := repository.GetUserRepository() if err := ur.AddUser(&schema.User{ Username: parts[0], Projects: make([]string, 0), Password: parts[2], Roles: strings.Split(parts[1], ","), }); err != nil { - log.Fatalf("adding '%s' user authentication failed: %v", parts[0], err) + log.Abortf("Add User: Could not add new user authentication for '%s' and roles '%s'.\nError: %s\n", parts[0], parts[1], err.Error()) + } else { + log.Printf("Add User: Added new user '%s' with roles '%s'.\n", parts[0], parts[1]) } } + if flagDelUser != "" { ur := repository.GetUserRepository() if err := ur.DelUser(flagDelUser); err != nil { - log.Fatalf("deleting user failed: %v", err) + log.Abortf("Delete User: Could not delete user '%s' from DB.\nError: %s\n", flagDelUser, err.Error()) + } else { + log.Printf("Delete User: Deleted user '%s' from DB.\n", flagDelUser) } } @@ -146,60 +153,64 @@ func main() { if flagSyncLDAP { if authHandle.LdapAuth == nil { - log.Fatal("cannot sync: LDAP authentication is not configured") + log.Abort("Sync LDAP: LDAP authentication is not configured, could not synchronize. No changes, exited.") } if err := authHandle.LdapAuth.Sync(); err != nil { - log.Fatalf("LDAP sync failed: %v", err) + log.Abortf("Sync LDAP: Could not synchronize, failed with error.\nError: %s\n", err.Error()) } - log.Info("LDAP sync successfull") + log.Print("Sync LDAP: LDAP synchronization successfull.") } if flagGenJWT != "" { ur := repository.GetUserRepository() user, err := ur.GetUser(flagGenJWT) if err != nil { - log.Fatalf("could not get user from JWT: %v", err) + log.Abortf("JWT: Could not get supplied user '%s' from DB. No changes, exited.\nError: %s\n", flagGenJWT, err.Error()) } if !user.HasRole(schema.RoleApi) { - log.Warnf("user '%s' does not have the API role", user.Username) + log.Warnf("JWT: User '%s' does not have the role 'api'. REST API endpoints will return error!\n", user.Username) } jwt, err := authHandle.JwtAuth.ProvideJWT(user) if err != nil { - log.Fatalf("failed to provide JWT to user '%s': %v", user.Username, err) + log.Abortf("JWT: User '%s' found in DB, but failed to provide JWT.\nError: %s\n", user.Username, err.Error()) } - fmt.Printf("MAIN > JWT for '%s': %s\n", user.Username, jwt) + log.Printf("JWT: Successfully generated JWT for user '%s': %s\n", user.Username, jwt) } } else if flagNewUser != "" || flagDelUser != "" { - log.Fatal("arguments --add-user and --del-user can only be used if authentication is enabled") + log.Abort("Error: Arguments '--add-user' and '--del-user' can only be used if authentication is enabled. No changes, exited.") } if err := archive.Init(config.Keys.Archive, config.Keys.DisableArchive); err != nil { - log.Fatalf("failed to initialize archive: %s", err.Error()) + log.Abortf("Init: Failed to initialize archive.\nError: %s\n", err.Error()) } if err := metricdata.Init(); err != nil { - log.Fatalf("failed to initialize metricdata repository: %s", err.Error()) + log.Abortf("Init: Failed to initialize metricdata repository.\nError %s\n", err.Error()) } if flagReinitDB { if err := importer.InitDB(); err != nil { - log.Fatalf("failed to re-initialize repository DB: %s", err.Error()) + log.Abortf("Init DB: Failed to re-initialize repository DB.\nError: %s\n", err.Error()) + } else { + log.Print("Init DB: Sucessfully re-initialized repository DB.") } } if flagImportJob != "" { if err := importer.HandleImportFlag(flagImportJob); err != nil { - log.Fatalf("job import failed: %s", err.Error()) + log.Abortf("Import Job: Job import failed.\nError: %s\n", err.Error()) + } else { + log.Printf("Import Job: Imported Job '%s' into DB.\n", flagImportJob) } } if !flagServer { - return + log.Exit("No errors, server flag not set. Exiting cc-backend.") } archiver.Start(repository.GetJobRepository()) From bd0cc69668655b11d0e8c92a67fa593e8c89f7be Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 27 Feb 2025 18:10:04 +0100 Subject: [PATCH 06/82] Review fatalf log calls and messages --- cmd/cc-backend/server.go | 14 +++++++------- internal/api/rest.go | 2 -- internal/config/config.go | 10 +++++----- internal/repository/dbConnection.go | 14 ++++++-------- internal/repository/migration.go | 4 ++-- internal/repository/userConfig.go | 2 +- internal/taskManager/taskManager.go | 2 +- tools/archive-manager/main.go | 3 +-- web/web.go | 6 ++---- 9 files changed, 25 insertions(+), 32 deletions(-) diff --git a/cmd/cc-backend/server.go b/cmd/cc-backend/server.go index 0770e81..1408162 100644 --- a/cmd/cc-backend/server.go +++ b/cmd/cc-backend/server.go @@ -64,7 +64,7 @@ func serverInit() { case string: return fmt.Errorf("MAIN > Panic: %s", e) case error: - return fmt.Errorf("MAIN > Panic caused by: %w", e) + return fmt.Errorf("MAIN > Panic caused by: %s", e.Error()) } return errors.New("MAIN > Internal server error (panic)") @@ -268,7 +268,7 @@ func serverStart() { // Start http or https server listener, err := net.Listen("tcp", config.Keys.Addr) if err != nil { - log.Fatalf("starting http listener failed: %v", err) + log.Abortf("Server Start: Starting http listener on '%s' failed.\nError: %s\n", config.Keys.Addr, err.Error()) } if !strings.HasSuffix(config.Keys.Addr, ":80") && config.Keys.RedirectHttpTo != "" { @@ -281,7 +281,7 @@ func serverStart() { cert, err := tls.LoadX509KeyPair( config.Keys.HttpsCertFile, config.Keys.HttpsKeyFile) if err != nil { - log.Fatalf("loading X509 keypair failed: %v", err) + log.Abortf("Server Start: Loading X509 keypair failed. Check options 'https-cert-file' and 'https-key-file' in 'config.json'.\nError: %s\n", err.Error()) } listener = tls.NewListener(listener, &tls.Config{ Certificates: []tls.Certificate{cert}, @@ -292,20 +292,20 @@ func serverStart() { MinVersion: tls.VersionTLS12, PreferServerCipherSuites: true, }) - fmt.Printf("HTTPS server listening at %s...", config.Keys.Addr) + log.Printf("HTTPS server listening at %s...\n", config.Keys.Addr) } else { - fmt.Printf("HTTP server listening at %s...", config.Keys.Addr) + log.Printf("HTTP server listening at %s...\n", config.Keys.Addr) } // // Because this program will want to bind to a privileged port (like 80), the listener must // be established first, then the user can be changed, and after that, // the actual http server can be started. if err := runtimeEnv.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil { - log.Fatalf("error while preparing server start: %s", err.Error()) + log.Abortf("Server Start: Error while preparing server start.\nError: %s\n", err.Error()) } if err = server.Serve(listener); err != nil && err != http.ErrServerClosed { - log.Fatalf("starting server failed: %v", err) + log.Abortf("Server Start: Starting server failed.\nError: %s\n", err.Error()) } } diff --git a/internal/api/rest.go b/internal/api/rest.go index b76da0b..73cd316 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -1423,8 +1423,6 @@ func (api *RestApi) updateConfiguration(rw http.ResponseWriter, r *http.Request) rw.Header().Set("Content-Type", "text/plain") key, value := r.FormValue("key"), r.FormValue("value") - // fmt.Printf("REST > KEY: %#v\nVALUE: %#v\n", key, value) - if err := repository.GetUserCfgRepo().UpdateConfig(key, value, repository.GetUserFromContext(r.Context())); err != nil { http.Error(rw, err.Error(), http.StatusUnprocessableEntity) return diff --git a/internal/config/config.go b/internal/config/config.go index 4f1a8c3..31760c7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -7,9 +7,9 @@ package config import ( "bytes" "encoding/json" - "log" "os" + "github.com/ClusterCockpit/cc-backend/pkg/log" "github.com/ClusterCockpit/cc-backend/pkg/schema" ) @@ -53,20 +53,20 @@ func Init(flagConfigFile string) { raw, err := os.ReadFile(flagConfigFile) if err != nil { if !os.IsNotExist(err) { - log.Fatalf("CONFIG ERROR: %v", err) + log.Abortf("Config Init: Could not read config file '%s'.\nError: %s\n", flagConfigFile, err.Error()) } } else { if err := schema.Validate(schema.Config, bytes.NewReader(raw)); err != nil { - log.Fatalf("Validate config: %v\n", err) + log.Abortf("Config Init: Could not validate config file '%s'.\nError: %s\n", flagConfigFile, err.Error()) } dec := json.NewDecoder(bytes.NewReader(raw)) dec.DisallowUnknownFields() if err := dec.Decode(&Keys); err != nil { - log.Fatalf("could not decode: %v", err) + log.Abortf("Config Init: Could not decode config file '%s'.\nError: %s\n", flagConfigFile, err.Error()) } if Keys.Clusters == nil || len(Keys.Clusters) < 1 { - log.Fatal("At least one cluster required in config!") + log.Abort("Config Init: At least one cluster required in config. Exited with error.") } } } diff --git a/internal/repository/dbConnection.go b/internal/repository/dbConnection.go index 418eef9..0e3f29d 100644 --- a/internal/repository/dbConnection.go +++ b/internal/repository/dbConnection.go @@ -59,17 +59,15 @@ func Connect(driver string, db string) { } else { dbHandle, err = sqlx.Open("sqlite3", opts.URL) } - if err != nil { - log.Fatal(err) - } case "mysql": opts.URL += "?multiStatements=true" dbHandle, err = sqlx.Open("mysql", opts.URL) - if err != nil { - log.Fatalf("sqlx.Open() error: %v", err) - } default: - log.Fatalf("unsupported database driver: %s", driver) + log.Abortf("DB Connection: Unsupported database driver '%s'.\n", driver) + } + + if err != nil { + log.Abortf("DB Connection: Could not connect to '%s' database with sqlx.Open().\nError: %s\n", driver, err.Error()) } dbHandle.SetMaxOpenConns(opts.MaxOpenConnections) @@ -80,7 +78,7 @@ func Connect(driver string, db string) { dbConnInstance = &DBConnection{DB: dbHandle, Driver: driver} err = checkDBVersion(driver, dbHandle.DB) if err != nil { - log.Fatal(err) + log.Abortf("DB Connection: Failed DB version check.\nError: %s\n", err.Error()) } }) } diff --git a/internal/repository/migration.go b/internal/repository/migration.go index d32a624..0b2591e 100644 --- a/internal/repository/migration.go +++ b/internal/repository/migration.go @@ -54,7 +54,7 @@ func checkDBVersion(backend string, db *sql.DB) error { return err } default: - log.Fatalf("unsupported database backend: %s", backend) + log.Abortf("Migration: Unsupported database backend '%s'.\n", backend) } v, dirty, err := m.Version() @@ -102,7 +102,7 @@ func getMigrateInstance(backend string, db string) (m *migrate.Migrate, err erro return m, err } default: - log.Fatalf("unsupported database backend: %s", backend) + log.Abortf("Migration: Unsupported database backend '%s'.\n", backend) } return m, nil diff --git a/internal/repository/userConfig.go b/internal/repository/userConfig.go index d2ab1d2..5d43071 100644 --- a/internal/repository/userConfig.go +++ b/internal/repository/userConfig.go @@ -35,7 +35,7 @@ func GetUserCfgRepo() *UserCfgRepo { lookupConfigStmt, err := db.DB.Preparex(`SELECT confkey, value FROM configuration WHERE configuration.username = ?`) if err != nil { - log.Fatalf("db.DB.Preparex() error: %v", err) + log.Fatalf("User Config: Call 'db.DB.Preparex()' failed.\nError: %s\n", err.Error()) } userCfgRepoInstance = &UserCfgRepo{ diff --git a/internal/taskManager/taskManager.go b/internal/taskManager/taskManager.go index 101fc4a..2004e0d 100644 --- a/internal/taskManager/taskManager.go +++ b/internal/taskManager/taskManager.go @@ -40,7 +40,7 @@ func Start() { jobRepo = repository.GetJobRepository() s, err = gocron.NewScheduler() if err != nil { - log.Fatalf("Error while creating gocron scheduler: %s", err.Error()) + log.Abortf("Taskmanager Start: Could not create gocron scheduler.\nError: %s\n", err.Error()) } if config.Keys.StopJobsExceedingWalltime > 0 { diff --git a/tools/archive-manager/main.go b/tools/archive-manager/main.go index 1a80712..7c842ff 100644 --- a/tools/archive-manager/main.go +++ b/tools/archive-manager/main.go @@ -22,8 +22,7 @@ func parseDate(in string) int64 { if in != "" { t, err := time.ParseInLocation(shortForm, in, loc) if err != nil { - fmt.Printf("date parse error %v", err) - os.Exit(0) + log.Abortf("Archive Manager Main: Date parse failed with input: '%s'\nError: %s\n", in, err.Error()) } return t.Unix() } diff --git a/web/web.go b/web/web.go index 45d8646..f543e53 100644 --- a/web/web.go +++ b/web/web.go @@ -26,8 +26,7 @@ var frontendFiles embed.FS func ServeFiles() http.Handler { publicFiles, err := fs.Sub(frontendFiles, "frontend/public") if err != nil { - log.Fatalf("WEB/WEB > cannot find frontend public files") - panic(err) + log.Abortf("Serve Files: Could not find 'frontend/public' file directory.\nError: %s\n", err.Error()) } return http.FileServer(http.FS(publicFiles)) } @@ -75,8 +74,7 @@ func init() { templates[strings.TrimPrefix(path, "templates/")] = template.Must(template.Must(base.Clone()).ParseFS(templateFiles, path)) return nil }); err != nil { - log.Fatalf("WEB/WEB > cannot find frontend template files") - panic(err) + log.Abortf("Web init(): Could not find frontend template files.\nError: %s\n", err.Error()) } _ = base From 6640e93ce98884de56c965b7da66bb3f61be05aa Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 28 Feb 2025 15:12:42 +0100 Subject: [PATCH 07/82] edit new features for 1.4.3 releasenotes --- ReleaseNotes.md | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/ReleaseNotes.md b/ReleaseNotes.md index 0015727..ef1082f 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -22,20 +22,19 @@ For release specific notes visit the [ClusterCockpit Documentation](https://clus ## New features -- Tags have a scope now. Tags created by a basic user are only visible by that - user. Tags created by an admin/support role can be configured to be visible by - all users (global scope) or only be admin/support role. -- Re-sampling support for running (requires a recent `cc-metric-store`) and - archived jobs. This greatly speeds up loading of large or very long jobs. You - need to add the new configuration key `enable-resampling` to the `config.json` - file. -- For finished jobs a total job energy is shown in the job view. -- Continuous scrolling in job lists is default now. -- All database queries (especially for sqlite) were optimized resulting in - dramatically faster load times. -- A performance and energy footprint can be freely configured on a per - subcluster base. One can filter for footprint statistics for running and - finished jobs. +- Detailed Node List + - Adds new routes `/systems/list/$cluster` and `/systems/list/$cluster/$subcluster` + - Displays live, scoped metric data requested from the nodes indepenent of jobs +- Color Blind Mode + - Set on a per-user basis in options + - Applies to plot data, plot background color, statsseries colors, roofline timescale +- Histogram Bin Select in User-View + - Metric-Histograms: `10 Bins` now default, selectable options `20, 50, 100` + - Job-Duration-Histogram: `48h in 1h Bins` now default, selectable options: + - `60 minutes in 1 minute Bins` + - `12 hours in 10 minute Bins` + - `3 days in 6 hour Bins` + - `7 days in 12 hour Bins` ## Known issues From ec895e1d9e1b789e76f08da399922e7244b215d2 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 3 Mar 2025 09:36:37 +0100 Subject: [PATCH 08/82] Add fallback case to nodeInfo --- web/frontend/src/systems/nodelist/NodeInfo.svelte | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/web/frontend/src/systems/nodelist/NodeInfo.svelte b/web/frontend/src/systems/nodelist/NodeInfo.svelte index ad6c98e..6b14656 100644 --- a/web/frontend/src/systems/nodelist/NodeInfo.svelte +++ b/web/frontend/src/systems/nodelist/NodeInfo.svelte @@ -102,6 +102,19 @@ Shared + + {:else if nodeJobsData.jobs.count >= 1} + + + + + + Status + + + {:else} From c21d7cf101ad7e4a1e4d6780db519351c420b14a Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 3 Mar 2025 11:21:54 +0100 Subject: [PATCH 09/82] fix and review quick starttime select handling --- web/frontend/src/generic/Filters.svelte | 26 ++-- .../src/generic/filters/StartTime.svelte | 113 +++++++++++------- 2 files changed, 86 insertions(+), 53 deletions(-) diff --git a/web/frontend/src/generic/Filters.svelte b/web/frontend/src/generic/Filters.svelte index 481211b..4a9be3e 100644 --- a/web/frontend/src/generic/Filters.svelte +++ b/web/frontend/src/generic/Filters.svelte @@ -45,6 +45,14 @@ export let startTimeQuickSelect = false; export let matchedJobs = -2; + const startTimeSelectOptions = [ + { range: "", rangeLabel: "No Selection"}, + { range: "last6h", rangeLabel: "Last 6hrs"}, + { range: "last24h", rangeLabel: "Last 24hrs"}, + { range: "last7d", rangeLabel: "Last 7 days"}, + { range: "last30d", rangeLabel: "Last 30 days"} + ]; + let filters = { projectMatch: filterPresets.projectMatch || "contains", userMatch: filterPresets.userMatch || "contains", @@ -56,7 +64,7 @@ filterPresets.states || filterPresets.state ? [filterPresets.state].flat() : allJobStates, - startTime: filterPresets.startTime || { from: null, to: null }, + startTime: filterPresets.startTime || { from: null, to: null, range: ""}, tags: filterPresets.tags || [], duration: filterPresets.duration || { lessThan: null, @@ -268,16 +276,17 @@ {#if startTimeQuickSelect} Start Time Quick Selection - {#each [{ text: "Last 6hrs", range: "last6h" }, { text: "Last 24hrs", range: "last24h" }, { text: "Last 7 days", range: "last7d" }, { text: "Last 30 days", range: "last30d" }] as { text, range }} + {#each startTimeSelectOptions.filter((stso) => stso.range !== "") as { rangeLabel, range }} { + filters.startTime.from = null + filters.startTime.to = null filters.startTime.range = range; - filters.startTime.text = text; updateFilters(); }} > - {text} + {rangeLabel} {/each} {/if} @@ -316,7 +325,7 @@ {#if filters.startTime.range} (isStartTimeOpen = true)}> - {filters?.startTime?.text ? filters.startTime.text : filters.startTime.range } + {startTimeSelectOptions.find((stso) => stso.range === filters.startTime.range).rangeLabel } {/if} @@ -414,11 +423,8 @@ bind:from={filters.startTime.from} bind:to={filters.startTime.to} bind:range={filters.startTime.range} - on:set-filter={() => { - delete filters.startTime["text"]; - delete filters.startTime["range"]; - updateFilters(); - }} + {startTimeSelectOptions} + on:set-filter={() => updateFilters()} /> (isOpen = !isOpen)}> @@ -92,52 +89,82 @@ {#if range !== ""}

Current Range

- - - + + + {#each startTimeSelectOptions as { rangeLabel, range }} + {/if}

From

- + - +

To

- + - + - + {#if pendingRange !== ""} + + + {:else} + + {/if} From 3ab8973895441481c8ce5b2494b87a284d4bb7a5 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 3 Mar 2025 12:44:18 +0100 Subject: [PATCH 10/82] use extendedLegend in nodeList for all non-idle nodes - changed from "use for shared nodes only" --- web/frontend/src/systems/nodelist/NodeListRow.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/frontend/src/systems/nodelist/NodeListRow.svelte b/web/frontend/src/systems/nodelist/NodeListRow.svelte index a1e4a54..07e5556 100644 --- a/web/frontend/src/systems/nodelist/NodeListRow.svelte +++ b/web/frontend/src/systems/nodelist/NodeListRow.svelte @@ -98,8 +98,8 @@ let extendedLegendData = null; $: if ($nodeJobsData?.data) { - // Get Shared State of Node: Only Build extended Legend For Shared Nodes - if ($nodeJobsData.data.jobs.count >= 1 && !$nodeJobsData.data.jobs.items[0].exclusive) { + // Build Extended for allocated nodes [Commented: Only Build extended Legend For Shared Nodes] + if ($nodeJobsData.data.jobs.count >= 1) { // "&& !$nodeJobsData.data.jobs.items[0].exclusive)" const accSet = Array.from(new Set($nodeJobsData.data.jobs.items .map((i) => i.resources .filter((r) => r.hostname === nodeData.host) From 419bc2747b9e774b5404c34f59b17d3609fe5299 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 3 Mar 2025 16:53:19 +0100 Subject: [PATCH 11/82] fix nodeInfo null error --- web/frontend/src/systems/nodelist/NodeListRow.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/frontend/src/systems/nodelist/NodeListRow.svelte b/web/frontend/src/systems/nodelist/NodeListRow.svelte index 07e5556..5e6e4ac 100644 --- a/web/frontend/src/systems/nodelist/NodeListRow.svelte +++ b/web/frontend/src/systems/nodelist/NodeListRow.svelte @@ -105,7 +105,7 @@ .filter((r) => r.hostname === nodeData.host) .map((r) => r.accelerators) ) - )).flat(2) + )).flat(2).filter(a => a) // Last filter(): Exclude Null, Undefined and empty Str extendedLegendData = {} for (const accId of accSet) { From 5c9d4ffa9a1f70ea4ff3ad2c3185ef9c7cc87168 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 3 Mar 2025 17:00:33 +0100 Subject: [PATCH 12/82] clarify and simplyfy earlier change --- web/frontend/src/systems/nodelist/NodeListRow.svelte | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/frontend/src/systems/nodelist/NodeListRow.svelte b/web/frontend/src/systems/nodelist/NodeListRow.svelte index 5e6e4ac..5202573 100644 --- a/web/frontend/src/systems/nodelist/NodeListRow.svelte +++ b/web/frontend/src/systems/nodelist/NodeListRow.svelte @@ -102,10 +102,10 @@ if ($nodeJobsData.data.jobs.count >= 1) { // "&& !$nodeJobsData.data.jobs.items[0].exclusive)" const accSet = Array.from(new Set($nodeJobsData.data.jobs.items .map((i) => i.resources - .filter((r) => r.hostname === nodeData.host) - .map((r) => r.accelerators) + .filter((r) => (r.hostname === nodeData.host) && r?.accelerators) + .map((r) => r?.accelerators) ) - )).flat(2).filter(a => a) // Last filter(): Exclude Null, Undefined and empty Str + )).flat(2) extendedLegendData = {} for (const accId of accSet) { From fcc9e17664ecf5d05206bda34e054c6fc48abccb Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 3 Mar 2025 17:24:54 +0100 Subject: [PATCH 13/82] change: remove metrics from job view select if unavailable on subCLuster --- web/frontend/src/Job.root.svelte | 3 ++- web/frontend/src/generic/select/MetricSelection.svelte | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/web/frontend/src/Job.root.svelte b/web/frontend/src/Job.root.svelte index b641a43..f2df916 100644 --- a/web/frontend/src/Job.root.svelte +++ b/web/frontend/src/Job.root.svelte @@ -130,7 +130,7 @@ const pendingMetrics = [ ...(ccconfig[`job_view_selectedMetrics:${job.cluster}`] || $initq.data.globalMetrics.reduce((names, gm) => { - if (gm.availability.find((av) => av.cluster === job.cluster)) { + if (gm.availability.find((av) => av.cluster === job.cluster && av.subClusters.includes(job.subCluster))) { names.push(gm.name); } return names; @@ -434,6 +434,7 @@ {#if $initq.data} av.cluster === cluster)) allMetrics.add(gm.name); + if (subCluster == null) { + if (gm.availability.find((av) => av.cluster === cluster)) allMetrics.add(gm.name); + } else { + if (gm.availability.find((av) => av.cluster === cluster && av.subClusters.includes(subCluster))) allMetrics.add(gm.name); + } } } newMetricsOrder = [...allMetrics].filter((m) => !metrics.includes(m)); From e733688fd03b41ccc6bb6ffac47aa8c1abd69bad Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 3 Mar 2025 17:54:34 +0100 Subject: [PATCH 14/82] add new subCluster prop to statsTable metric select --- web/frontend/src/job/StatsTable.svelte | 1 + 1 file changed, 1 insertion(+) diff --git a/web/frontend/src/job/StatsTable.svelte b/web/frontend/src/job/StatsTable.svelte index 21d9b3b..b6b0f85 100644 --- a/web/frontend/src/job/StatsTable.svelte +++ b/web/frontend/src/job/StatsTable.svelte @@ -169,6 +169,7 @@ Date: Thu, 27 Feb 2025 15:11:07 +0100 Subject: [PATCH 15/82] allow /start_job/ with 0 second duration Apparently it is possible to get this for very short jobs. --- internal/api/rest.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/api/rest.go b/internal/api/rest.go index b76da0b..fd2f86d 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -1008,8 +1008,8 @@ func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Jo return } - if job == nil || job.StartTime.Unix() >= req.StopTime { - handleError(fmt.Errorf("jobId %d (id %d) on %s : stopTime %d must be larger than startTime %d", job.JobID, job.ID, job.Cluster, req.StopTime, job.StartTime.Unix()), http.StatusBadRequest, rw) + if job == nil || job.StartTime.Unix() > req.StopTime { + handleError(fmt.Errorf("jobId %d (id %d) on %s : stopTime %d must be larger/equal than startTime %d", job.JobID, job.ID, job.Cluster, req.StopTime, job.StartTime.Unix()), http.StatusBadRequest, rw) return } From 6454576417ca9048435390a6a3c30415d1a15951 Mon Sep 17 00:00:00 2001 From: Michael Panzlaff Date: Tue, 4 Mar 2025 17:39:38 +0100 Subject: [PATCH 16/82] add node_fail job state --- api/swagger.json | 6 ++++-- api/swagger.yaml | 2 ++ internal/api/docs.go | 6 ++++-- pkg/schema/job.go | 4 +++- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/api/swagger.json b/api/swagger.json index 51b22c8..9035beb 100644 --- a/api/swagger.json +++ b/api/swagger.json @@ -1786,7 +1786,8 @@ "stopped", "timeout", "preempted", - "out_of_memory" + "out_of_memory", + "node_fail" ], "x-enum-varnames": [ "JobStateRunning", @@ -1796,7 +1797,8 @@ "JobStateStopped", "JobStateTimeout", "JobStatePreempted", - "JobStateOutOfMemory" + "JobStateOutOfMemory", + "JobStateNodeFail" ] }, "schema.JobStatistics": { diff --git a/api/swagger.yaml b/api/swagger.yaml index f5f0081..20fa031 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -395,6 +395,7 @@ definitions: - timeout - preempted - out_of_memory + - node_fail type: string x-enum-varnames: - JobStateRunning @@ -405,6 +406,7 @@ definitions: - JobStateTimeout - JobStatePreempted - JobStateOutOfMemory + - JobStateNodeFail schema.JobStatistics: description: Specification for job metric statistics. properties: diff --git a/internal/api/docs.go b/internal/api/docs.go index 642003f..6f034b4 100644 --- a/internal/api/docs.go +++ b/internal/api/docs.go @@ -1792,7 +1792,8 @@ const docTemplate = `{ "stopped", "timeout", "preempted", - "out_of_memory" + "out_of_memory", + "node_fail" ], "x-enum-varnames": [ "JobStateRunning", @@ -1802,7 +1803,8 @@ const docTemplate = `{ "JobStateStopped", "JobStateTimeout", "JobStatePreempted", - "JobStateOutOfMemory" + "JobStateOutOfMemory", + "JobStateNodeFail" ] }, "schema.JobStatistics": { diff --git a/pkg/schema/job.go b/pkg/schema/job.go index 5e3110b..b6ac44d 100644 --- a/pkg/schema/job.go +++ b/pkg/schema/job.go @@ -143,6 +143,7 @@ const ( JobStateTimeout JobState = "timeout" JobStatePreempted JobState = "preempted" JobStateOutOfMemory JobState = "out_of_memory" + JobStateNodeFail JobState = "node_fail" ) func (e *JobState) UnmarshalGQL(v interface{}) error { @@ -171,5 +172,6 @@ func (e JobState) Valid() bool { e == JobStateStopped || e == JobStateTimeout || e == JobStatePreempted || - e == JobStateOutOfMemory + e == JobStateOutOfMemory || + e == JobStateNodeFail } From 65d2698af4a104fbd5ff1faf0f462e6a50b6a466 Mon Sep 17 00:00:00 2001 From: Michael Panzlaff Date: Tue, 4 Mar 2025 17:47:49 +0100 Subject: [PATCH 17/82] add node_fail state to database schema --- internal/repository/migrations/mysql/01_init-schema.up.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/repository/migrations/mysql/01_init-schema.up.sql b/internal/repository/migrations/mysql/01_init-schema.up.sql index 3a6930c..16f7627 100644 --- a/internal/repository/migrations/mysql/01_init-schema.up.sql +++ b/internal/repository/migrations/mysql/01_init-schema.up.sql @@ -13,7 +13,7 @@ CREATE TABLE IF NOT EXISTS job ( walltime INT NOT NULL DEFAULT 0, job_state VARCHAR(255) NOT NULL CHECK(job_state IN ('running', 'completed', 'failed', 'cancelled', - 'stopped', 'timeout', 'preempted', 'out_of_memory')), + 'stopped', 'timeout', 'preempted', 'out_of_memory', 'node_fail')), meta_data TEXT, -- JSON resources TEXT NOT NULL, -- JSON From d4336b0dcb4e054a39033fc681634c285d08d4d8 Mon Sep 17 00:00:00 2001 From: Michael Panzlaff Date: Tue, 4 Mar 2025 18:00:02 +0100 Subject: [PATCH 18/82] add missing node_fail to db constraints --- .../repository/migrations/sqlite3/04_add-constraints.up.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/repository/migrations/sqlite3/04_add-constraints.up.sql b/internal/repository/migrations/sqlite3/04_add-constraints.up.sql index 06b1a9b..a6898c3 100644 --- a/internal/repository/migrations/sqlite3/04_add-constraints.up.sql +++ b/internal/repository/migrations/sqlite3/04_add-constraints.up.sql @@ -11,7 +11,7 @@ array_job_id BIGINT, duration INT NOT NULL, walltime INT NOT NULL, job_state VARCHAR(255) NOT NULL -CHECK(job_state IN ('running', 'completed', 'failed', 'cancelled', 'stopped', 'timeout', 'preempted', 'out_of_memory')), +CHECK(job_state IN ('running', 'completed', 'failed', 'cancelled', 'stopped', 'timeout', 'preempted', 'out_of_memory', 'node_fail')), meta_data TEXT, -- JSON resources TEXT NOT NULL, -- JSON num_nodes INT NOT NULL, From 0a3e678329bc7162bffde549a2e85ac69b63e11b Mon Sep 17 00:00:00 2001 From: Michael Panzlaff Date: Tue, 4 Mar 2025 18:03:01 +0100 Subject: [PATCH 19/82] add more missing node_fail states --- api/swagger.json | 6 ++++-- api/swagger.yaml | 2 ++ internal/api/docs.go | 6 ++++-- pkg/schema/job.go | 2 +- pkg/schema/schemas/job-meta.schema.json | 1 + web/frontend/src/generic/filters/JobStates.svelte | 1 + 6 files changed, 13 insertions(+), 5 deletions(-) diff --git a/api/swagger.json b/api/swagger.json index 9035beb..5cd4a5e 100644 --- a/api/swagger.json +++ b/api/swagger.json @@ -1512,7 +1512,8 @@ "cancelled", "stopped", "timeout", - "out_of_memory" + "out_of_memory", + "node_fail" ], "allOf": [ { @@ -1670,7 +1671,8 @@ "cancelled", "stopped", "timeout", - "out_of_memory" + "out_of_memory", + "node_fail" ], "allOf": [ { diff --git a/api/swagger.yaml b/api/swagger.yaml index 20fa031..3f188c2 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -201,6 +201,7 @@ definitions: - stopped - timeout - out_of_memory + - node_fail example: completed metaData: additionalProperties: @@ -314,6 +315,7 @@ definitions: - stopped - timeout - out_of_memory + - node_fail example: completed metaData: additionalProperties: diff --git a/internal/api/docs.go b/internal/api/docs.go index 6f034b4..99a8a14 100644 --- a/internal/api/docs.go +++ b/internal/api/docs.go @@ -1518,7 +1518,8 @@ const docTemplate = `{ "cancelled", "stopped", "timeout", - "out_of_memory" + "out_of_memory", + "node_fail" ], "allOf": [ { @@ -1676,7 +1677,8 @@ const docTemplate = `{ "cancelled", "stopped", "timeout", - "out_of_memory" + "out_of_memory", + "node_fail" ], "allOf": [ { diff --git a/pkg/schema/job.go b/pkg/schema/job.go index b6ac44d..7a2d950 100644 --- a/pkg/schema/job.go +++ b/pkg/schema/job.go @@ -21,7 +21,7 @@ type BaseJob struct { Partition string `json:"partition,omitempty" db:"cluster_partition" example:"main"` Project string `json:"project" db:"project" example:"abcd200"` User string `json:"user" db:"hpc_user" example:"abcd100h"` - State JobState `json:"jobState" db:"job_state" example:"completed" enums:"completed,failed,cancelled,stopped,timeout,out_of_memory"` + State JobState `json:"jobState" db:"job_state" example:"completed" enums:"completed,failed,cancelled,stopped,timeout,out_of_memory,node_fail"` Tags []*Tag `json:"tags,omitempty"` RawEnergyFootprint []byte `json:"-" db:"energy_footprint"` RawFootprint []byte `json:"-" db:"footprint"` diff --git a/pkg/schema/schemas/job-meta.schema.json b/pkg/schema/schemas/job-meta.schema.json index db7475c..a12057b 100644 --- a/pkg/schema/schemas/job-meta.schema.json +++ b/pkg/schema/schemas/job-meta.schema.json @@ -76,6 +76,7 @@ "cancelled", "stopped", "out_of_memory", + "node_fail", "timeout" ] }, diff --git a/web/frontend/src/generic/filters/JobStates.svelte b/web/frontend/src/generic/filters/JobStates.svelte index d903abc..b9a747d 100644 --- a/web/frontend/src/generic/filters/JobStates.svelte +++ b/web/frontend/src/generic/filters/JobStates.svelte @@ -23,6 +23,7 @@ "timeout", "preempted", "out_of_memory", + "node_fail", ]; From a61ff915ac0517261b8ac2be3e3cc3b8e7f40e7c Mon Sep 17 00:00:00 2001 From: Michael Panzlaff Date: Tue, 4 Mar 2025 18:15:39 +0100 Subject: [PATCH 20/82] Revert "add more missing node_fail states" This reverts commit 0a3e678329bc7162bffde549a2e85ac69b63e11b. --- api/swagger.json | 6 ++---- api/swagger.yaml | 2 -- internal/api/docs.go | 6 ++---- pkg/schema/job.go | 2 +- pkg/schema/schemas/job-meta.schema.json | 1 - web/frontend/src/generic/filters/JobStates.svelte | 1 - 6 files changed, 5 insertions(+), 13 deletions(-) diff --git a/api/swagger.json b/api/swagger.json index 5cd4a5e..9035beb 100644 --- a/api/swagger.json +++ b/api/swagger.json @@ -1512,8 +1512,7 @@ "cancelled", "stopped", "timeout", - "out_of_memory", - "node_fail" + "out_of_memory" ], "allOf": [ { @@ -1671,8 +1670,7 @@ "cancelled", "stopped", "timeout", - "out_of_memory", - "node_fail" + "out_of_memory" ], "allOf": [ { diff --git a/api/swagger.yaml b/api/swagger.yaml index 3f188c2..20fa031 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -201,7 +201,6 @@ definitions: - stopped - timeout - out_of_memory - - node_fail example: completed metaData: additionalProperties: @@ -315,7 +314,6 @@ definitions: - stopped - timeout - out_of_memory - - node_fail example: completed metaData: additionalProperties: diff --git a/internal/api/docs.go b/internal/api/docs.go index 99a8a14..6f034b4 100644 --- a/internal/api/docs.go +++ b/internal/api/docs.go @@ -1518,8 +1518,7 @@ const docTemplate = `{ "cancelled", "stopped", "timeout", - "out_of_memory", - "node_fail" + "out_of_memory" ], "allOf": [ { @@ -1677,8 +1676,7 @@ const docTemplate = `{ "cancelled", "stopped", "timeout", - "out_of_memory", - "node_fail" + "out_of_memory" ], "allOf": [ { diff --git a/pkg/schema/job.go b/pkg/schema/job.go index 7a2d950..b6ac44d 100644 --- a/pkg/schema/job.go +++ b/pkg/schema/job.go @@ -21,7 +21,7 @@ type BaseJob struct { Partition string `json:"partition,omitempty" db:"cluster_partition" example:"main"` Project string `json:"project" db:"project" example:"abcd200"` User string `json:"user" db:"hpc_user" example:"abcd100h"` - State JobState `json:"jobState" db:"job_state" example:"completed" enums:"completed,failed,cancelled,stopped,timeout,out_of_memory,node_fail"` + State JobState `json:"jobState" db:"job_state" example:"completed" enums:"completed,failed,cancelled,stopped,timeout,out_of_memory"` Tags []*Tag `json:"tags,omitempty"` RawEnergyFootprint []byte `json:"-" db:"energy_footprint"` RawFootprint []byte `json:"-" db:"footprint"` diff --git a/pkg/schema/schemas/job-meta.schema.json b/pkg/schema/schemas/job-meta.schema.json index a12057b..db7475c 100644 --- a/pkg/schema/schemas/job-meta.schema.json +++ b/pkg/schema/schemas/job-meta.schema.json @@ -76,7 +76,6 @@ "cancelled", "stopped", "out_of_memory", - "node_fail", "timeout" ] }, diff --git a/web/frontend/src/generic/filters/JobStates.svelte b/web/frontend/src/generic/filters/JobStates.svelte index b9a747d..d903abc 100644 --- a/web/frontend/src/generic/filters/JobStates.svelte +++ b/web/frontend/src/generic/filters/JobStates.svelte @@ -23,7 +23,6 @@ "timeout", "preempted", "out_of_memory", - "node_fail", ]; From aa3fe2b8726634800a36d6dc4153ab6c7c9f93f9 Mon Sep 17 00:00:00 2001 From: Michael Panzlaff Date: Tue, 4 Mar 2025 18:15:46 +0100 Subject: [PATCH 21/82] Revert "add missing node_fail to db constraints" This reverts commit d4336b0dcb4e054a39033fc681634c285d08d4d8. --- .../repository/migrations/sqlite3/04_add-constraints.up.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/repository/migrations/sqlite3/04_add-constraints.up.sql b/internal/repository/migrations/sqlite3/04_add-constraints.up.sql index a6898c3..06b1a9b 100644 --- a/internal/repository/migrations/sqlite3/04_add-constraints.up.sql +++ b/internal/repository/migrations/sqlite3/04_add-constraints.up.sql @@ -11,7 +11,7 @@ array_job_id BIGINT, duration INT NOT NULL, walltime INT NOT NULL, job_state VARCHAR(255) NOT NULL -CHECK(job_state IN ('running', 'completed', 'failed', 'cancelled', 'stopped', 'timeout', 'preempted', 'out_of_memory', 'node_fail')), +CHECK(job_state IN ('running', 'completed', 'failed', 'cancelled', 'stopped', 'timeout', 'preempted', 'out_of_memory')), meta_data TEXT, -- JSON resources TEXT NOT NULL, -- JSON num_nodes INT NOT NULL, From bd93b8be8efd2440d3eab6d5c43e9c4e7d4c164b Mon Sep 17 00:00:00 2001 From: Michael Panzlaff Date: Tue, 4 Mar 2025 18:15:53 +0100 Subject: [PATCH 22/82] Revert "add node_fail state to database schema" This reverts commit 65d2698af4a104fbd5ff1faf0f462e6a50b6a466. --- internal/repository/migrations/mysql/01_init-schema.up.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/repository/migrations/mysql/01_init-schema.up.sql b/internal/repository/migrations/mysql/01_init-schema.up.sql index 16f7627..3a6930c 100644 --- a/internal/repository/migrations/mysql/01_init-schema.up.sql +++ b/internal/repository/migrations/mysql/01_init-schema.up.sql @@ -13,7 +13,7 @@ CREATE TABLE IF NOT EXISTS job ( walltime INT NOT NULL DEFAULT 0, job_state VARCHAR(255) NOT NULL CHECK(job_state IN ('running', 'completed', 'failed', 'cancelled', - 'stopped', 'timeout', 'preempted', 'out_of_memory', 'node_fail')), + 'stopped', 'timeout', 'preempted', 'out_of_memory')), meta_data TEXT, -- JSON resources TEXT NOT NULL, -- JSON From 4b2d7068b334c99bca3b77cc6a34371d5cb4416e Mon Sep 17 00:00:00 2001 From: Michael Panzlaff Date: Tue, 4 Mar 2025 18:16:02 +0100 Subject: [PATCH 23/82] Revert "add node_fail job state" This reverts commit 6454576417ca9048435390a6a3c30415d1a15951. --- api/swagger.json | 6 ++---- api/swagger.yaml | 2 -- internal/api/docs.go | 6 ++---- pkg/schema/job.go | 4 +--- 4 files changed, 5 insertions(+), 13 deletions(-) diff --git a/api/swagger.json b/api/swagger.json index 9035beb..51b22c8 100644 --- a/api/swagger.json +++ b/api/swagger.json @@ -1786,8 +1786,7 @@ "stopped", "timeout", "preempted", - "out_of_memory", - "node_fail" + "out_of_memory" ], "x-enum-varnames": [ "JobStateRunning", @@ -1797,8 +1796,7 @@ "JobStateStopped", "JobStateTimeout", "JobStatePreempted", - "JobStateOutOfMemory", - "JobStateNodeFail" + "JobStateOutOfMemory" ] }, "schema.JobStatistics": { diff --git a/api/swagger.yaml b/api/swagger.yaml index 20fa031..f5f0081 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -395,7 +395,6 @@ definitions: - timeout - preempted - out_of_memory - - node_fail type: string x-enum-varnames: - JobStateRunning @@ -406,7 +405,6 @@ definitions: - JobStateTimeout - JobStatePreempted - JobStateOutOfMemory - - JobStateNodeFail schema.JobStatistics: description: Specification for job metric statistics. properties: diff --git a/internal/api/docs.go b/internal/api/docs.go index 6f034b4..642003f 100644 --- a/internal/api/docs.go +++ b/internal/api/docs.go @@ -1792,8 +1792,7 @@ const docTemplate = `{ "stopped", "timeout", "preempted", - "out_of_memory", - "node_fail" + "out_of_memory" ], "x-enum-varnames": [ "JobStateRunning", @@ -1803,8 +1802,7 @@ const docTemplate = `{ "JobStateStopped", "JobStateTimeout", "JobStatePreempted", - "JobStateOutOfMemory", - "JobStateNodeFail" + "JobStateOutOfMemory" ] }, "schema.JobStatistics": { diff --git a/pkg/schema/job.go b/pkg/schema/job.go index b6ac44d..5e3110b 100644 --- a/pkg/schema/job.go +++ b/pkg/schema/job.go @@ -143,7 +143,6 @@ const ( JobStateTimeout JobState = "timeout" JobStatePreempted JobState = "preempted" JobStateOutOfMemory JobState = "out_of_memory" - JobStateNodeFail JobState = "node_fail" ) func (e *JobState) UnmarshalGQL(v interface{}) error { @@ -172,6 +171,5 @@ func (e JobState) Valid() bool { e == JobStateStopped || e == JobStateTimeout || e == JobStatePreempted || - e == JobStateOutOfMemory || - e == JobStateNodeFail + e == JobStateOutOfMemory } From 2b56b40e6d2b69d49f666f0753e131f34a13aa83 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 6 Mar 2025 12:46:25 +0100 Subject: [PATCH 24/82] Review energyFootprint calculation, fix missing numNodes factor, add log --- internal/importer/handleImport.go | 24 ++++++++++++++++-------- internal/importer/initDB.go | 24 ++++++++++++++++-------- internal/repository/job.go | 24 +++++++++++++++--------- 3 files changed, 47 insertions(+), 25 deletions(-) diff --git a/internal/importer/handleImport.go b/internal/importer/handleImport.go index 01773a5..623291c 100644 --- a/internal/importer/handleImport.go +++ b/internal/importer/handleImport.go @@ -96,27 +96,35 @@ func HandleImportFlag(flag string) error { } job.EnergyFootprint = make(map[string]float64) - var totalEnergy float64 - var energy float64 + // Total Job Energy Outside Loop + totalEnergy := 0.0 for _, fp := range sc.EnergyFootprint { + // Always Init Metric Energy Inside Loop + metricEnergy := 0.0 if i, err := archive.MetricIndex(sc.MetricConfig, fp); err == nil { // Note: For DB data, calculate and save as kWh - // Energy: Power (in Watts) * Time (in Seconds) if sc.MetricConfig[i].Energy == "energy" { // this metric has energy as unit (Joules) + log.Warnf("Update EnergyFootprint for Job %d and Metric %s on cluster %s: Set to 'energy' in cluster.json: Not implemented, will return 0.0", job.JobID, job.Cluster, fp) + // FIXME: Needs sum as stats type } else if sc.MetricConfig[i].Energy == "power" { // this metric has power as unit (Watt) - // Unit: ( W * s ) / 3600 / 1000 = kWh ; Rounded to 2 nearest digits - energy = math.Round(((repository.LoadJobStat(&job, fp, "avg")*float64(job.Duration))/3600/1000)*100) / 100 + // Energy: Power (in Watts) * Time (in Seconds) + // Unit: (W * (s / 3600)) / 1000 = kWh + // Round 2 Digits: round(Energy * 100) / 100 + // Here: (All-Node Metric Average * Number of Nodes) * (Job Duration in Seconds / 3600) / 1000 + // Note: Shared Jobs handled correctly since "Node Average" is based on partial resources, while "numNodes" factor is 1 + rawEnergy := ((repository.LoadJobStat(&job, fp, "avg") * float64(job.NumNodes)) * (float64(job.Duration) / 3600.0)) / 1000.0 + metricEnergy = math.Round(rawEnergy*100.0) / 100.0 } } else { log.Warnf("Error while collecting energy metric %s for job, DB ID '%v', return '0.0'", fp, job.ID) } - job.EnergyFootprint[fp] = energy - totalEnergy += energy + job.EnergyFootprint[fp] = metricEnergy + totalEnergy += metricEnergy } - job.Energy = (math.Round(totalEnergy*100) / 100) + job.Energy = (math.Round(totalEnergy*100.0) / 100.0) if job.RawEnergyFootprint, err = json.Marshal(job.EnergyFootprint); err != nil { log.Warnf("Error while marshaling energy footprint for job INTO BYTES, DB ID '%v'", job.ID) return err diff --git a/internal/importer/initDB.go b/internal/importer/initDB.go index fa2ee6e..9a2ccdf 100644 --- a/internal/importer/initDB.go +++ b/internal/importer/initDB.go @@ -93,27 +93,35 @@ func InitDB() error { } job.EnergyFootprint = make(map[string]float64) - var totalEnergy float64 - var energy float64 + // Total Job Energy Outside Loop + totalEnergy := 0.0 for _, fp := range sc.EnergyFootprint { + // Always Init Metric Energy Inside Loop + metricEnergy := 0.0 if i, err := archive.MetricIndex(sc.MetricConfig, fp); err == nil { // Note: For DB data, calculate and save as kWh - // Energy: Power (in Watts) * Time (in Seconds) if sc.MetricConfig[i].Energy == "energy" { // this metric has energy as unit (Joules) + log.Warnf("Update EnergyFootprint for Job %d and Metric %s on cluster %s: Set to 'energy' in cluster.json: Not implemented, will return 0.0", jobMeta.JobID, jobMeta.Cluster, fp) + // FIXME: Needs sum as stats type } else if sc.MetricConfig[i].Energy == "power" { // this metric has power as unit (Watt) - // Unit: ( W * s ) / 3600 / 1000 = kWh ; Rounded to 2 nearest digits - energy = math.Round(((repository.LoadJobStat(jobMeta, fp, "avg")*float64(jobMeta.Duration))/3600/1000)*100) / 100 + // Energy: Power (in Watts) * Time (in Seconds) + // Unit: (W * (s / 3600)) / 1000 = kWh + // Round 2 Digits: round(Energy * 100) / 100 + // Here: (All-Node Metric Average * Number of Nodes) * (Job Duration in Seconds / 3600) / 1000 + // Note: Shared Jobs handled correctly since "Node Average" is based on partial resources, while "numNodes" factor is 1 + rawEnergy := ((repository.LoadJobStat(jobMeta, fp, "avg") * float64(jobMeta.NumNodes)) * (float64(jobMeta.Duration) / 3600.0)) / 1000.0 + metricEnergy = math.Round(rawEnergy*100.0) / 100.0 } } else { log.Warnf("Error while collecting energy metric %s for job, DB ID '%v', return '0.0'", fp, jobMeta.ID) } - job.EnergyFootprint[fp] = energy - totalEnergy += energy + job.EnergyFootprint[fp] = metricEnergy + totalEnergy += metricEnergy } - job.Energy = (math.Round(totalEnergy*100) / 100) + job.Energy = (math.Round(totalEnergy*100.0) / 100.0) if job.RawEnergyFootprint, err = json.Marshal(job.EnergyFootprint); err != nil { log.Warnf("Error while marshaling energy footprint for job INTO BYTES, DB ID '%v'", jobMeta.ID) return err diff --git a/internal/repository/job.go b/internal/repository/job.go index 020c3c2..84de6f7 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -590,28 +590,34 @@ func (r *JobRepository) UpdateEnergy( return stmt, err } energyFootprint := make(map[string]float64) - var totalEnergy float64 - var energy float64 + // Total Job Energy Outside Loop + totalEnergy := 0.0 for _, fp := range sc.EnergyFootprint { + // Always Init Metric Energy Inside Loop + metricEnergy := 0.0 if i, err := archive.MetricIndex(sc.MetricConfig, fp); err == nil { // Note: For DB data, calculate and save as kWh if sc.MetricConfig[i].Energy == "energy" { // this metric has energy as unit (Joules or Wh) + log.Warnf("Update EnergyFootprint for Job %d and Metric %s on cluster %s: Set to 'energy' in cluster.json: Not implemented, will return 0.0", jobMeta.JobID, jobMeta.Cluster, fp) // FIXME: Needs sum as stats type } else if sc.MetricConfig[i].Energy == "power" { // this metric has power as unit (Watt) // Energy: Power (in Watts) * Time (in Seconds) - // Unit: (( W * s ) / 3600) / 1000 = kWh ; Rounded to 2 nearest digits: (Energy * 100) / 100 - // Here: All-Node Metric Average * Number of Nodes * Job Runtime + // Unit: (W * (s / 3600)) / 1000 = kWh + // Round 2 Digits: round(Energy * 100) / 100 + // Here: (All-Node Metric Average * Number of Nodes) * (Job Duration in Seconds / 3600) / 1000 // Note: Shared Jobs handled correctly since "Node Average" is based on partial resources, while "numNodes" factor is 1 - metricNodeSum := LoadJobStat(jobMeta, fp, "avg") * float64(jobMeta.NumNodes) * float64(jobMeta.Duration) - energy = math.Round(((metricNodeSum/3600)/1000)*100) / 100 + rawEnergy := ((LoadJobStat(jobMeta, fp, "avg") * float64(jobMeta.NumNodes)) * (float64(jobMeta.Duration) / 3600.0)) / 1000.0 + metricEnergy = math.Round(rawEnergy*100.0) / 100.0 } } else { log.Warnf("Error while collecting energy metric %s for job, DB ID '%v', return '0.0'", fp, jobMeta.ID) } - energyFootprint[fp] = energy - totalEnergy += energy + energyFootprint[fp] = metricEnergy + totalEnergy += metricEnergy + + // log.Infof("Metric %s Average %f -> %f kWh | Job %d Total -> %f kWh", fp, LoadJobStat(jobMeta, fp, "avg"), energy, jobMeta.JobID, totalEnergy) } var rawFootprint []byte @@ -620,7 +626,7 @@ func (r *JobRepository) UpdateEnergy( return stmt, err } - return stmt.Set("energy_footprint", string(rawFootprint)).Set("energy", (math.Round(totalEnergy*100) / 100)), nil + return stmt.Set("energy_footprint", string(rawFootprint)).Set("energy", (math.Round(totalEnergy*100.0) / 100.0)), nil } func (r *JobRepository) UpdateFootprint( From d0af933b350d3e50cc64c648b5fbfd2fd4d1a0cf Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 6 Mar 2025 15:39:15 +0100 Subject: [PATCH 25/82] feat: add subCluster level frontend keys for metric selections - applies to jobView and nodeList --- web/frontend/src/Job.root.svelte | 25 +++++++---- web/frontend/src/Jobs.root.svelte | 2 +- web/frontend/src/Systems.root.svelte | 10 +++-- web/frontend/src/User.root.svelte | 2 +- .../src/generic/select/MetricSelection.svelte | 42 ++++++++++++------- web/frontend/src/job/StatsTable.svelte | 7 ++-- web/frontend/src/systems/NodeList.svelte | 16 +++---- 7 files changed, 65 insertions(+), 39 deletions(-) diff --git a/web/frontend/src/Job.root.svelte b/web/frontend/src/Job.root.svelte index f2df916..6980230 100644 --- a/web/frontend/src/Job.root.svelte +++ b/web/frontend/src/Job.root.svelte @@ -128,15 +128,24 @@ if (!job) return; const pendingMetrics = [ - ...(ccconfig[`job_view_selectedMetrics:${job.cluster}`] || - $initq.data.globalMetrics.reduce((names, gm) => { - if (gm.availability.find((av) => av.cluster === job.cluster && av.subClusters.includes(job.subCluster))) { - names.push(gm.name); - } - return names; - }, []) + ...( + ( + ccconfig[`job_view_selectedMetrics:${job.cluster}:${job.subCluster}`] || + ccconfig[`job_view_selectedMetrics:${job.cluster}`] + ) || + $initq.data.globalMetrics + .reduce((names, gm) => { + if (gm.availability.find((av) => av.cluster === job.cluster && av.subClusters.includes(job.subCluster))) { + names.push(gm.name); + } + return names; + }, []) ), - ...(ccconfig[`job_view_nodestats_selectedMetrics:${job.cluster}`] || + ...( + ( + ccconfig[`job_view_nodestats_selectedMetrics:${job.cluster}:${job.subCluster}`] || + ccconfig[`job_view_nodestats_selectedMetrics:${job.cluster}`] + ) || ccconfig[`job_view_nodestats_selectedMetrics`] ), ]; diff --git a/web/frontend/src/Jobs.root.svelte b/web/frontend/src/Jobs.root.svelte index df928d0..7faa8b8 100644 --- a/web/frontend/src/Jobs.root.svelte +++ b/web/frontend/src/Jobs.root.svelte @@ -137,5 +137,5 @@ bind:metrics bind:isOpen={isMetricsSelectionOpen} bind:showFootprint - footprintSelect={true} + footprintSelect /> diff --git a/web/frontend/src/Systems.root.svelte b/web/frontend/src/Systems.root.svelte index 8089bbe..1589cac 100644 --- a/web/frontend/src/Systems.root.svelte +++ b/web/frontend/src/Systems.root.svelte @@ -29,8 +29,8 @@ import Refresher from "./generic/helper/Refresher.svelte"; export let displayType; - export let cluster; - export let subCluster = ""; + export let cluster = null; + export let subCluster = null; export let from = null; export let to = null; @@ -60,7 +60,10 @@ let hostnameFilter = ""; let pendingHostnameFilter = ""; let selectedMetric = ccconfig.system_view_selectedMetric || ""; - let selectedMetrics = ccconfig[`node_list_selectedMetrics:${cluster}`] || [ccconfig.system_view_selectedMetric]; + let selectedMetrics = ( + ccconfig[`node_list_selectedMetrics:${cluster}:${subCluster}`] || + ccconfig[`node_list_selectedMetrics:${cluster}`] + ) || [ccconfig.system_view_selectedMetric]; let isMetricsSelectionOpen = false; /* @@ -191,6 +194,7 @@ av.cluster === cluster)) allMetrics.add(gm.name); } else { if (gm.availability.find((av) => av.cluster === cluster && av.subClusters.includes(subCluster))) allMetrics.add(gm.name); @@ -67,7 +67,7 @@ function printAvailability(metric, cluster) { const avail = globalMetrics.find((gm) => gm.name === metric)?.availability - if (cluster == null) { + if (!cluster) { return avail.map((av) => av.cluster).join(',') } else { return avail.find((av) => av.cluster === cluster).subClusters.join(',') @@ -112,10 +112,17 @@ metrics = newMetricsOrder.filter((m) => unorderedMetrics.includes(m)); isOpen = false; - showFootprint = !!pendingShowFootprint; + let configKey; + if (cluster && subCluster) { + configKey = `${configName}:${cluster}:${subCluster}`; + } else if (cluster && !subCluster) { + configKey = `${configName}:${cluster}`; + } else { + configKey = `${configName}`; + } updateConfigurationMutation({ - name: cluster == null ? configName : `${configName}:${cluster}`, + name: configKey, value: JSON.stringify(metrics), }).subscribe((res) => { if (res.fetching === false && res.error) { @@ -123,17 +130,20 @@ } }); - updateConfigurationMutation({ - name: - cluster == null - ? "plot_list_showFootprint" - : `plot_list_showFootprint:${cluster}`, - value: JSON.stringify(showFootprint), - }).subscribe((res) => { - if (res.fetching === false && res.error) { - throw res.error; - } - }); + if (footprintSelect) { + showFootprint = !!pendingShowFootprint; + updateConfigurationMutation({ + name: + !cluster + ? "plot_list_showFootprint" + : `plot_list_showFootprint:${cluster}`, + value: JSON.stringify(showFootprint), + }).subscribe((res) => { + if (res.fetching === false && res.error) { + throw res.error; + } + }); + }; dispatch('update-metrics', metrics); } diff --git a/web/frontend/src/job/StatsTable.svelte b/web/frontend/src/job/StatsTable.svelte index b6b0f85..c8f12f2 100644 --- a/web/frontend/src/job/StatsTable.svelte +++ b/web/frontend/src/job/StatsTable.svelte @@ -37,9 +37,10 @@ sorting = {}, isMetricSelectionOpen = false, availableMetrics = new Set(), - selectedMetrics = - getContext("cc-config")[`job_view_nodestats_selectedMetrics:${job.cluster}`] || - getContext("cc-config")["job_view_nodestats_selectedMetrics"]; + selectedMetrics = ( + getContext("cc-config")[`job_view_nodestats_selectedMetrics:${job.cluster}:${job.subCluster}`] || + getContext("cc-config")[`job_view_nodestats_selectedMetrics:${job.cluster}`] + ) || getContext("cc-config")["job_view_nodestats_selectedMetrics"]; for (let metric of sortedJobMetrics) { // Not Exclusive or Multi-Node: get maxScope directly (mostly: node) diff --git a/web/frontend/src/systems/NodeList.svelte b/web/frontend/src/systems/NodeList.svelte index ad64a1f..ca22d57 100644 --- a/web/frontend/src/systems/NodeList.svelte +++ b/web/frontend/src/systems/NodeList.svelte @@ -217,13 +217,15 @@
-

- Loading nodes {nodes.length + 1} to - { matchedNodes - ? `${(nodes.length + paging.itemsPerPage) > matchedNodes ? matchedNodes : (nodes.length + paging.itemsPerPage)} of ${matchedNodes} total` - : (nodes.length + paging.itemsPerPage) - } -

+ {#if !usePaging} +

+ Loading nodes {nodes.length + 1} to + { matchedNodes + ? `${(nodes.length + paging.itemsPerPage) > matchedNodes ? matchedNodes : (nodes.length + paging.itemsPerPage)} of ${matchedNodes} total` + : (nodes.length + paging.itemsPerPage) + } +

+ {/if}
From 16db9bd1a29382a1bc556d1f31c1ef1c4352673d Mon Sep 17 00:00:00 2001 From: exterr2f Date: Mon, 10 Mar 2025 08:15:42 +0100 Subject: [PATCH 26/82] Fix node filter: Use EXISTS with Eq for exact match and LIKE for Contains --- internal/repository/jobFind.go | 4 ++-- internal/repository/jobQuery.go | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/internal/repository/jobFind.go b/internal/repository/jobFind.go index 0354df0..ea5e1e9 100644 --- a/internal/repository/jobFind.go +++ b/internal/repository/jobFind.go @@ -194,11 +194,11 @@ func (r *JobRepository) FindConcurrentJobs( queryRunning := query.Where("job.job_state = ?").Where("(job.start_time BETWEEN ? AND ? OR job.start_time < ?)", "running", startTimeTail, stopTimeTail, startTime) - queryRunning = queryRunning.Where("job.resources LIKE ?", fmt.Sprint("%", hostname, "%")) + queryRunning = queryRunning.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, '$.hostname') = ?)", hostname) query = query.Where("job.job_state != ?").Where("((job.start_time BETWEEN ? AND ?) OR (job.start_time + job.duration) BETWEEN ? AND ? OR (job.start_time < ?) AND (job.start_time + job.duration) > ?)", "running", startTimeTail, stopTimeTail, startTimeFront, stopTimeTail, startTime, stopTime) - query = query.Where("job.resources LIKE ?", fmt.Sprint("%", hostname, "%")) + query = query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, '$.hostname') = ?)", hostname) rows, err := query.RunWith(r.stmtCache).Query() if err != nil { diff --git a/internal/repository/jobQuery.go b/internal/repository/jobQuery.go index b43b569..7d1d9eb 100644 --- a/internal/repository/jobQuery.go +++ b/internal/repository/jobQuery.go @@ -194,7 +194,14 @@ func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.Select query = buildIntCondition("job.num_hwthreads", filter.NumHWThreads, query) } if filter.Node != nil { - query = buildStringCondition("job.resources", filter.Node, query) + log.Infof("Applying node filter: %v", filter.Node) + if filter.Node.Eq != nil { + query = query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, '$.hostname') = ?)", *filter.Node.Eq) + } else if filter.Node.Contains != nil { + query = query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, '$.hostname') LIKE ?)", "%"+*filter.Node.Contains+"%") + } else { + query = buildStringCondition("job.resources", filter.Node, query) + } } if filter.Energy != nil { query = buildFloatCondition("job.energy", filter.Energy, query) From f5f36427a45d082cb92df72a79566ea8efbe75d0 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 13 Mar 2025 17:33:55 +0100 Subject: [PATCH 27/82] split statsTable data from jobMetrics query, initial commit - mainly backend changes - statstable changes only for prototyping --- api/schema.graphqls | 39 +- internal/graph/generated/generated.go | 1326 +++++++++++++---- internal/graph/model/models_gen.go | 22 +- internal/graph/schema.resolvers.go | 56 +- internal/metricDataDispatcher/dataLoader.go | 34 +- internal/metricdata/cc-metric-store.go | 93 +- internal/metricdata/influxdb-v2.go | 12 + internal/metricdata/metricdata.go | 5 +- internal/metricdata/prometheus.go | 14 +- internal/metricdata/utils.go | 12 +- pkg/archive/archive.go | 20 +- pkg/archive/fsBackend.go | 46 + pkg/archive/json.go | 37 + pkg/schema/metrics.go | 7 + web/frontend/src/Job.root.svelte | 44 +- web/frontend/src/job/Metric.svelte | 5 - web/frontend/src/job/StatsTable.svelte | 95 +- web/frontend/src/job/StatsTableEntry.svelte | 18 +- .../job/jobsummary/JobFootprintPolar.svelte | 12 +- 19 files changed, 1471 insertions(+), 426 deletions(-) diff --git a/api/schema.graphqls b/api/schema.graphqls index 9385a6f..ed8843c 100644 --- a/api/schema.graphqls +++ b/api/schema.graphqls @@ -137,11 +137,6 @@ type JobMetricWithName { metric: JobMetric! } -type JobMetricStatWithName { - name: String! - stats: MetricStatistics! -} - type JobMetric { unit: Unit timestep: Int! @@ -156,6 +151,30 @@ type Series { data: [NullableFloat!]! } +type StatsSeries { + mean: [NullableFloat!]! + median: [NullableFloat!]! + min: [NullableFloat!]! + max: [NullableFloat!]! +} + +type JobStatsWithScope { + name: String! + scope: MetricScope! + stats: [ScopedStats!]! +} + +type ScopedStats { + hostname: String! + id: String + data: MetricStatistics! +} + +type JobStats { + name: String! + stats: MetricStatistics! +} + type Unit { base: String! prefix: String @@ -167,13 +186,6 @@ type MetricStatistics { max: Float! } -type StatsSeries { - mean: [NullableFloat!]! - median: [NullableFloat!]! - min: [NullableFloat!]! - max: [NullableFloat!]! -} - type MetricFootprints { metric: String! data: [NullableFloat!]! @@ -247,7 +259,8 @@ type Query { job(id: ID!): Job jobMetrics(id: ID!, metrics: [String!], scopes: [MetricScope!], resolution: Int): [JobMetricWithName!]! - jobMetricStats(id: ID!, metrics: [String!]): [JobMetricStatWithName!]! + jobStats(id: ID!, metrics: [String!]): [JobStats!]! + scopedJobStats(id: ID!, metrics: [String!], scopes: [MetricScope!]): [JobStatsWithScope!]! jobsFootprints(filter: [JobFilter!], metrics: [String!]!): Footprints jobs(filter: [JobFilter!], page: PageRequest, order: OrderByInput): JobResultList! diff --git a/internal/graph/generated/generated.go b/internal/graph/generated/generated.go index b4c6e19..e5c9ca2 100644 --- a/internal/graph/generated/generated.go +++ b/internal/graph/generated/generated.go @@ -156,11 +156,6 @@ type ComplexityRoot struct { Unit func(childComplexity int) int } - JobMetricStatWithName struct { - Name func(childComplexity int) int - Stats func(childComplexity int) int - } - JobMetricWithName struct { Metric func(childComplexity int) int Name func(childComplexity int) int @@ -175,6 +170,17 @@ type ComplexityRoot struct { Offset func(childComplexity int) int } + JobStats struct { + Name func(childComplexity int) int + Stats func(childComplexity int) int + } + + JobStatsWithScope struct { + Name func(childComplexity int) int + Scope func(childComplexity int) int + Stats func(childComplexity int) int + } + JobsStatistics struct { HistDuration func(childComplexity int) int HistMetrics func(childComplexity int) int @@ -268,14 +274,15 @@ type ComplexityRoot struct { Clusters func(childComplexity int) int GlobalMetrics func(childComplexity int) int Job func(childComplexity int, id string) int - JobMetricStats func(childComplexity int, id string, metrics []string) int JobMetrics func(childComplexity int, id string, metrics []string, scopes []schema.MetricScope, resolution *int) int + JobStats func(childComplexity int, id string, metrics []string) int Jobs func(childComplexity int, filter []*model.JobFilter, page *model.PageRequest, order *model.OrderByInput) int JobsFootprints func(childComplexity int, filter []*model.JobFilter, metrics []string) int JobsStatistics func(childComplexity int, filter []*model.JobFilter, metrics []string, page *model.PageRequest, sortBy *model.SortByAggregate, groupBy *model.Aggregate, numDurationBins *string, numMetricBins *int) int NodeMetrics func(childComplexity int, cluster string, nodes []string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time) int NodeMetricsList func(childComplexity int, cluster string, subCluster string, nodeFilter string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time, page *model.PageRequest, resolution *int) int RooflineHeatmap func(childComplexity int, filter []*model.JobFilter, rows int, cols int, minX float64, minY float64, maxX float64, maxY float64) int + ScopedJobStats func(childComplexity int, id string, metrics []string, scopes []schema.MetricScope) int Tags func(childComplexity int) int User func(childComplexity int, username string) int } @@ -287,6 +294,12 @@ type ComplexityRoot struct { Hostname func(childComplexity int) int } + ScopedStats struct { + Data func(childComplexity int) int + Hostname func(childComplexity int) int + ID func(childComplexity int) int + } + Series struct { Data func(childComplexity int) int Hostname func(childComplexity int) int @@ -396,7 +409,8 @@ type QueryResolver interface { AllocatedNodes(ctx context.Context, cluster string) ([]*model.Count, error) Job(ctx context.Context, id string) (*schema.Job, error) JobMetrics(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope, resolution *int) ([]*model.JobMetricWithName, error) - JobMetricStats(ctx context.Context, id string, metrics []string) ([]*model.JobMetricStatWithName, error) + JobStats(ctx context.Context, id string, metrics []string) ([]*model.JobStats, error) + ScopedJobStats(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope) ([]*model.JobStatsWithScope, error) JobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) Jobs(ctx context.Context, filter []*model.JobFilter, page *model.PageRequest, order *model.OrderByInput) (*model.JobResultList, error) JobsStatistics(ctx context.Context, filter []*model.JobFilter, metrics []string, page *model.PageRequest, sortBy *model.SortByAggregate, groupBy *model.Aggregate, numDurationBins *string, numMetricBins *int) ([]*model.JobsStatistics, error) @@ -861,20 +875,6 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.JobMetric.Unit(childComplexity), true - case "JobMetricStatWithName.name": - if e.complexity.JobMetricStatWithName.Name == nil { - break - } - - return e.complexity.JobMetricStatWithName.Name(childComplexity), true - - case "JobMetricStatWithName.stats": - if e.complexity.JobMetricStatWithName.Stats == nil { - break - } - - return e.complexity.JobMetricStatWithName.Stats(childComplexity), true - case "JobMetricWithName.metric": if e.complexity.JobMetricWithName.Metric == nil { break @@ -931,6 +931,41 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.JobResultList.Offset(childComplexity), true + case "JobStats.name": + if e.complexity.JobStats.Name == nil { + break + } + + return e.complexity.JobStats.Name(childComplexity), true + + case "JobStats.stats": + if e.complexity.JobStats.Stats == nil { + break + } + + return e.complexity.JobStats.Stats(childComplexity), true + + case "JobStatsWithScope.name": + if e.complexity.JobStatsWithScope.Name == nil { + break + } + + return e.complexity.JobStatsWithScope.Name(childComplexity), true + + case "JobStatsWithScope.scope": + if e.complexity.JobStatsWithScope.Scope == nil { + break + } + + return e.complexity.JobStatsWithScope.Scope(childComplexity), true + + case "JobStatsWithScope.stats": + if e.complexity.JobStatsWithScope.Stats == nil { + break + } + + return e.complexity.JobStatsWithScope.Stats(childComplexity), true + case "JobsStatistics.histDuration": if e.complexity.JobsStatistics.HistDuration == nil { break @@ -1400,18 +1435,6 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Query.Job(childComplexity, args["id"].(string)), true - case "Query.jobMetricStats": - if e.complexity.Query.JobMetricStats == nil { - break - } - - args, err := ec.field_Query_jobMetricStats_args(context.TODO(), rawArgs) - if err != nil { - return 0, false - } - - return e.complexity.Query.JobMetricStats(childComplexity, args["id"].(string), args["metrics"].([]string)), true - case "Query.jobMetrics": if e.complexity.Query.JobMetrics == nil { break @@ -1424,6 +1447,18 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Query.JobMetrics(childComplexity, args["id"].(string), args["metrics"].([]string), args["scopes"].([]schema.MetricScope), args["resolution"].(*int)), true + case "Query.jobStats": + if e.complexity.Query.JobStats == nil { + break + } + + args, err := ec.field_Query_jobStats_args(context.TODO(), rawArgs) + if err != nil { + return 0, false + } + + return e.complexity.Query.JobStats(childComplexity, args["id"].(string), args["metrics"].([]string)), true + case "Query.jobs": if e.complexity.Query.Jobs == nil { break @@ -1496,6 +1531,18 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Query.RooflineHeatmap(childComplexity, args["filter"].([]*model.JobFilter), args["rows"].(int), args["cols"].(int), args["minX"].(float64), args["minY"].(float64), args["maxX"].(float64), args["maxY"].(float64)), true + case "Query.scopedJobStats": + if e.complexity.Query.ScopedJobStats == nil { + break + } + + args, err := ec.field_Query_scopedJobStats_args(context.TODO(), rawArgs) + if err != nil { + return 0, false + } + + return e.complexity.Query.ScopedJobStats(childComplexity, args["id"].(string), args["metrics"].([]string), args["scopes"].([]schema.MetricScope)), true + case "Query.tags": if e.complexity.Query.Tags == nil { break @@ -1543,6 +1590,27 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Resource.Hostname(childComplexity), true + case "ScopedStats.data": + if e.complexity.ScopedStats.Data == nil { + break + } + + return e.complexity.ScopedStats.Data(childComplexity), true + + case "ScopedStats.hostname": + if e.complexity.ScopedStats.Hostname == nil { + break + } + + return e.complexity.ScopedStats.Hostname(childComplexity), true + + case "ScopedStats.id": + if e.complexity.ScopedStats.ID == nil { + break + } + + return e.complexity.ScopedStats.ID(childComplexity), true + case "Series.data": if e.complexity.Series.Data == nil { break @@ -2131,11 +2199,6 @@ type JobMetricWithName { metric: JobMetric! } -type JobMetricStatWithName { - name: String! - stats: MetricStatistics! -} - type JobMetric { unit: Unit timestep: Int! @@ -2150,6 +2213,30 @@ type Series { data: [NullableFloat!]! } +type StatsSeries { + mean: [NullableFloat!]! + median: [NullableFloat!]! + min: [NullableFloat!]! + max: [NullableFloat!]! +} + +type JobStatsWithScope { + name: String! + scope: MetricScope! + stats: [ScopedStats!]! +} + +type ScopedStats { + hostname: String! + id: String + data: MetricStatistics! +} + +type JobStats { + name: String! + stats: MetricStatistics! +} + type Unit { base: String! prefix: String @@ -2161,13 +2248,6 @@ type MetricStatistics { max: Float! } -type StatsSeries { - mean: [NullableFloat!]! - median: [NullableFloat!]! - min: [NullableFloat!]! - max: [NullableFloat!]! -} - type MetricFootprints { metric: String! data: [NullableFloat!]! @@ -2241,7 +2321,8 @@ type Query { job(id: ID!): Job jobMetrics(id: ID!, metrics: [String!], scopes: [MetricScope!], resolution: Int): [JobMetricWithName!]! - jobMetricStats(id: ID!, metrics: [String!]): [JobMetricStatWithName!]! + jobStats(id: ID!, metrics: [String!]): [JobStats!]! + scopedJobStats(id: ID!, metrics: [String!], scopes: [MetricScope!]): [JobStatsWithScope!]! jobsFootprints(filter: [JobFilter!], metrics: [String!]!): Footprints jobs(filter: [JobFilter!], page: PageRequest, order: OrderByInput): JobResultList! @@ -2694,57 +2775,6 @@ func (ec *executionContext) field_Query_allocatedNodes_argsCluster( return zeroVal, nil } -func (ec *executionContext) field_Query_jobMetricStats_args(ctx context.Context, rawArgs map[string]any) (map[string]any, error) { - var err error - args := map[string]any{} - arg0, err := ec.field_Query_jobMetricStats_argsID(ctx, rawArgs) - if err != nil { - return nil, err - } - args["id"] = arg0 - arg1, err := ec.field_Query_jobMetricStats_argsMetrics(ctx, rawArgs) - if err != nil { - return nil, err - } - args["metrics"] = arg1 - return args, nil -} -func (ec *executionContext) field_Query_jobMetricStats_argsID( - ctx context.Context, - rawArgs map[string]any, -) (string, error) { - if _, ok := rawArgs["id"]; !ok { - var zeroVal string - return zeroVal, nil - } - - ctx = graphql.WithPathContext(ctx, graphql.NewPathWithField("id")) - if tmp, ok := rawArgs["id"]; ok { - return ec.unmarshalNID2string(ctx, tmp) - } - - var zeroVal string - return zeroVal, nil -} - -func (ec *executionContext) field_Query_jobMetricStats_argsMetrics( - ctx context.Context, - rawArgs map[string]any, -) ([]string, error) { - if _, ok := rawArgs["metrics"]; !ok { - var zeroVal []string - return zeroVal, nil - } - - ctx = graphql.WithPathContext(ctx, graphql.NewPathWithField("metrics")) - if tmp, ok := rawArgs["metrics"]; ok { - return ec.unmarshalOString2ᚕstringᚄ(ctx, tmp) - } - - var zeroVal []string - return zeroVal, nil -} - func (ec *executionContext) field_Query_jobMetrics_args(ctx context.Context, rawArgs map[string]any) (map[string]any, error) { var err error args := map[string]any{} @@ -2842,6 +2872,57 @@ func (ec *executionContext) field_Query_jobMetrics_argsResolution( return zeroVal, nil } +func (ec *executionContext) field_Query_jobStats_args(ctx context.Context, rawArgs map[string]any) (map[string]any, error) { + var err error + args := map[string]any{} + arg0, err := ec.field_Query_jobStats_argsID(ctx, rawArgs) + if err != nil { + return nil, err + } + args["id"] = arg0 + arg1, err := ec.field_Query_jobStats_argsMetrics(ctx, rawArgs) + if err != nil { + return nil, err + } + args["metrics"] = arg1 + return args, nil +} +func (ec *executionContext) field_Query_jobStats_argsID( + ctx context.Context, + rawArgs map[string]any, +) (string, error) { + if _, ok := rawArgs["id"]; !ok { + var zeroVal string + return zeroVal, nil + } + + ctx = graphql.WithPathContext(ctx, graphql.NewPathWithField("id")) + if tmp, ok := rawArgs["id"]; ok { + return ec.unmarshalNID2string(ctx, tmp) + } + + var zeroVal string + return zeroVal, nil +} + +func (ec *executionContext) field_Query_jobStats_argsMetrics( + ctx context.Context, + rawArgs map[string]any, +) ([]string, error) { + if _, ok := rawArgs["metrics"]; !ok { + var zeroVal []string + return zeroVal, nil + } + + ctx = graphql.WithPathContext(ctx, graphql.NewPathWithField("metrics")) + if tmp, ok := rawArgs["metrics"]; ok { + return ec.unmarshalOString2ᚕstringᚄ(ctx, tmp) + } + + var zeroVal []string + return zeroVal, nil +} + func (ec *executionContext) field_Query_job_args(ctx context.Context, rawArgs map[string]any) (map[string]any, error) { var err error args := map[string]any{} @@ -3682,6 +3763,80 @@ func (ec *executionContext) field_Query_rooflineHeatmap_argsMaxY( return zeroVal, nil } +func (ec *executionContext) field_Query_scopedJobStats_args(ctx context.Context, rawArgs map[string]any) (map[string]any, error) { + var err error + args := map[string]any{} + arg0, err := ec.field_Query_scopedJobStats_argsID(ctx, rawArgs) + if err != nil { + return nil, err + } + args["id"] = arg0 + arg1, err := ec.field_Query_scopedJobStats_argsMetrics(ctx, rawArgs) + if err != nil { + return nil, err + } + args["metrics"] = arg1 + arg2, err := ec.field_Query_scopedJobStats_argsScopes(ctx, rawArgs) + if err != nil { + return nil, err + } + args["scopes"] = arg2 + return args, nil +} +func (ec *executionContext) field_Query_scopedJobStats_argsID( + ctx context.Context, + rawArgs map[string]any, +) (string, error) { + if _, ok := rawArgs["id"]; !ok { + var zeroVal string + return zeroVal, nil + } + + ctx = graphql.WithPathContext(ctx, graphql.NewPathWithField("id")) + if tmp, ok := rawArgs["id"]; ok { + return ec.unmarshalNID2string(ctx, tmp) + } + + var zeroVal string + return zeroVal, nil +} + +func (ec *executionContext) field_Query_scopedJobStats_argsMetrics( + ctx context.Context, + rawArgs map[string]any, +) ([]string, error) { + if _, ok := rawArgs["metrics"]; !ok { + var zeroVal []string + return zeroVal, nil + } + + ctx = graphql.WithPathContext(ctx, graphql.NewPathWithField("metrics")) + if tmp, ok := rawArgs["metrics"]; ok { + return ec.unmarshalOString2ᚕstringᚄ(ctx, tmp) + } + + var zeroVal []string + return zeroVal, nil +} + +func (ec *executionContext) field_Query_scopedJobStats_argsScopes( + ctx context.Context, + rawArgs map[string]any, +) ([]schema.MetricScope, error) { + if _, ok := rawArgs["scopes"]; !ok { + var zeroVal []schema.MetricScope + return zeroVal, nil + } + + ctx = graphql.WithPathContext(ctx, graphql.NewPathWithField("scopes")) + if tmp, ok := rawArgs["scopes"]; ok { + return ec.unmarshalOMetricScope2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋpkgᚋschemaᚐMetricScopeᚄ(ctx, tmp) + } + + var zeroVal []schema.MetricScope + return zeroVal, nil +} + func (ec *executionContext) field_Query_user_args(ctx context.Context, rawArgs map[string]any) (map[string]any, error) { var err error args := map[string]any{} @@ -6663,102 +6818,6 @@ func (ec *executionContext) fieldContext_JobMetric_statisticsSeries(_ context.Co return fc, nil } -func (ec *executionContext) _JobMetricStatWithName_name(ctx context.Context, field graphql.CollectedField, obj *model.JobMetricStatWithName) (ret graphql.Marshaler) { - fc, err := ec.fieldContext_JobMetricStatWithName_name(ctx, field) - if err != nil { - return graphql.Null - } - ctx = graphql.WithFieldContext(ctx, fc) - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = graphql.Null - } - }() - resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { - ctx = rctx // use context from middleware stack in children - return obj.Name, nil - }) - if err != nil { - ec.Error(ctx, err) - return graphql.Null - } - if resTmp == nil { - if !graphql.HasFieldError(ctx, fc) { - ec.Errorf(ctx, "must not be null") - } - return graphql.Null - } - res := resTmp.(string) - fc.Result = res - return ec.marshalNString2string(ctx, field.Selections, res) -} - -func (ec *executionContext) fieldContext_JobMetricStatWithName_name(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { - fc = &graphql.FieldContext{ - Object: "JobMetricStatWithName", - Field: field, - IsMethod: false, - IsResolver: false, - Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { - return nil, errors.New("field of type String does not have child fields") - }, - } - return fc, nil -} - -func (ec *executionContext) _JobMetricStatWithName_stats(ctx context.Context, field graphql.CollectedField, obj *model.JobMetricStatWithName) (ret graphql.Marshaler) { - fc, err := ec.fieldContext_JobMetricStatWithName_stats(ctx, field) - if err != nil { - return graphql.Null - } - ctx = graphql.WithFieldContext(ctx, fc) - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = graphql.Null - } - }() - resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { - ctx = rctx // use context from middleware stack in children - return obj.Stats, nil - }) - if err != nil { - ec.Error(ctx, err) - return graphql.Null - } - if resTmp == nil { - if !graphql.HasFieldError(ctx, fc) { - ec.Errorf(ctx, "must not be null") - } - return graphql.Null - } - res := resTmp.(*schema.MetricStatistics) - fc.Result = res - return ec.marshalNMetricStatistics2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋpkgᚋschemaᚐMetricStatistics(ctx, field.Selections, res) -} - -func (ec *executionContext) fieldContext_JobMetricStatWithName_stats(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { - fc = &graphql.FieldContext{ - Object: "JobMetricStatWithName", - Field: field, - IsMethod: false, - IsResolver: false, - Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { - switch field.Name { - case "avg": - return ec.fieldContext_MetricStatistics_avg(ctx, field) - case "min": - return ec.fieldContext_MetricStatistics_min(ctx, field) - case "max": - return ec.fieldContext_MetricStatistics_max(ctx, field) - } - return nil, fmt.Errorf("no field named %q was found under type MetricStatistics", field.Name) - }, - } - return fc, nil -} - func (ec *executionContext) _JobMetricWithName_name(ctx context.Context, field graphql.CollectedField, obj *model.JobMetricWithName) (ret graphql.Marshaler) { fc, err := ec.fieldContext_JobMetricWithName_name(ctx, field) if err != nil { @@ -7163,6 +7222,242 @@ func (ec *executionContext) fieldContext_JobResultList_hasNextPage(_ context.Con return fc, nil } +func (ec *executionContext) _JobStats_name(ctx context.Context, field graphql.CollectedField, obj *model.JobStats) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_JobStats_name(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.Name, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(string) + fc.Result = res + return ec.marshalNString2string(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_JobStats_name(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "JobStats", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type String does not have child fields") + }, + } + return fc, nil +} + +func (ec *executionContext) _JobStats_stats(ctx context.Context, field graphql.CollectedField, obj *model.JobStats) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_JobStats_stats(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.Stats, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(*schema.MetricStatistics) + fc.Result = res + return ec.marshalNMetricStatistics2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋpkgᚋschemaᚐMetricStatistics(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_JobStats_stats(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "JobStats", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "avg": + return ec.fieldContext_MetricStatistics_avg(ctx, field) + case "min": + return ec.fieldContext_MetricStatistics_min(ctx, field) + case "max": + return ec.fieldContext_MetricStatistics_max(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type MetricStatistics", field.Name) + }, + } + return fc, nil +} + +func (ec *executionContext) _JobStatsWithScope_name(ctx context.Context, field graphql.CollectedField, obj *model.JobStatsWithScope) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_JobStatsWithScope_name(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.Name, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(string) + fc.Result = res + return ec.marshalNString2string(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_JobStatsWithScope_name(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "JobStatsWithScope", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type String does not have child fields") + }, + } + return fc, nil +} + +func (ec *executionContext) _JobStatsWithScope_scope(ctx context.Context, field graphql.CollectedField, obj *model.JobStatsWithScope) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_JobStatsWithScope_scope(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.Scope, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(schema.MetricScope) + fc.Result = res + return ec.marshalNMetricScope2githubᚗcomᚋClusterCockpitᚋccᚑbackendᚋpkgᚋschemaᚐMetricScope(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_JobStatsWithScope_scope(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "JobStatsWithScope", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type MetricScope does not have child fields") + }, + } + return fc, nil +} + +func (ec *executionContext) _JobStatsWithScope_stats(ctx context.Context, field graphql.CollectedField, obj *model.JobStatsWithScope) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_JobStatsWithScope_stats(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.Stats, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.([]*model.ScopedStats) + fc.Result = res + return ec.marshalNScopedStats2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐScopedStatsᚄ(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_JobStatsWithScope_stats(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "JobStatsWithScope", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "hostname": + return ec.fieldContext_ScopedStats_hostname(ctx, field) + case "id": + return ec.fieldContext_ScopedStats_id(ctx, field) + case "data": + return ec.fieldContext_ScopedStats_data(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type ScopedStats", field.Name) + }, + } + return fc, nil +} + func (ec *executionContext) _JobsStatistics_id(ctx context.Context, field graphql.CollectedField, obj *model.JobsStatistics) (ret graphql.Marshaler) { fc, err := ec.fieldContext_JobsStatistics_id(ctx, field) if err != nil { @@ -10296,8 +10591,8 @@ func (ec *executionContext) fieldContext_Query_jobMetrics(ctx context.Context, f return fc, nil } -func (ec *executionContext) _Query_jobMetricStats(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { - fc, err := ec.fieldContext_Query_jobMetricStats(ctx, field) +func (ec *executionContext) _Query_jobStats(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_Query_jobStats(ctx, field) if err != nil { return graphql.Null } @@ -10310,7 +10605,7 @@ func (ec *executionContext) _Query_jobMetricStats(ctx context.Context, field gra }() resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { ctx = rctx // use context from middleware stack in children - return ec.resolvers.Query().JobMetricStats(rctx, fc.Args["id"].(string), fc.Args["metrics"].([]string)) + return ec.resolvers.Query().JobStats(rctx, fc.Args["id"].(string), fc.Args["metrics"].([]string)) }) if err != nil { ec.Error(ctx, err) @@ -10322,12 +10617,12 @@ func (ec *executionContext) _Query_jobMetricStats(ctx context.Context, field gra } return graphql.Null } - res := resTmp.([]*model.JobMetricStatWithName) + res := resTmp.([]*model.JobStats) fc.Result = res - return ec.marshalNJobMetricStatWithName2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobMetricStatWithNameᚄ(ctx, field.Selections, res) + return ec.marshalNJobStats2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobStatsᚄ(ctx, field.Selections, res) } -func (ec *executionContext) fieldContext_Query_jobMetricStats(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { +func (ec *executionContext) fieldContext_Query_jobStats(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { fc = &graphql.FieldContext{ Object: "Query", Field: field, @@ -10336,11 +10631,11 @@ func (ec *executionContext) fieldContext_Query_jobMetricStats(ctx context.Contex Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { switch field.Name { case "name": - return ec.fieldContext_JobMetricStatWithName_name(ctx, field) + return ec.fieldContext_JobStats_name(ctx, field) case "stats": - return ec.fieldContext_JobMetricStatWithName_stats(ctx, field) + return ec.fieldContext_JobStats_stats(ctx, field) } - return nil, fmt.Errorf("no field named %q was found under type JobMetricStatWithName", field.Name) + return nil, fmt.Errorf("no field named %q was found under type JobStats", field.Name) }, } defer func() { @@ -10350,7 +10645,70 @@ func (ec *executionContext) fieldContext_Query_jobMetricStats(ctx context.Contex } }() ctx = graphql.WithFieldContext(ctx, fc) - if fc.Args, err = ec.field_Query_jobMetricStats_args(ctx, field.ArgumentMap(ec.Variables)); err != nil { + if fc.Args, err = ec.field_Query_jobStats_args(ctx, field.ArgumentMap(ec.Variables)); err != nil { + ec.Error(ctx, err) + return fc, err + } + return fc, nil +} + +func (ec *executionContext) _Query_scopedJobStats(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_Query_scopedJobStats(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return ec.resolvers.Query().ScopedJobStats(rctx, fc.Args["id"].(string), fc.Args["metrics"].([]string), fc.Args["scopes"].([]schema.MetricScope)) + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.([]*model.JobStatsWithScope) + fc.Result = res + return ec.marshalNJobStatsWithScope2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobStatsWithScopeᚄ(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_Query_scopedJobStats(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "Query", + Field: field, + IsMethod: true, + IsResolver: true, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "name": + return ec.fieldContext_JobStatsWithScope_name(ctx, field) + case "scope": + return ec.fieldContext_JobStatsWithScope_scope(ctx, field) + case "stats": + return ec.fieldContext_JobStatsWithScope_stats(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type JobStatsWithScope", field.Name) + }, + } + defer func() { + if r := recover(); r != nil { + err = ec.Recover(ctx, r) + ec.Error(ctx, err) + } + }() + ctx = graphql.WithFieldContext(ctx, fc) + if fc.Args, err = ec.field_Query_scopedJobStats_args(ctx, field.ArgumentMap(ec.Variables)); err != nil { ec.Error(ctx, err) return fc, err } @@ -11058,6 +11416,143 @@ func (ec *executionContext) fieldContext_Resource_configuration(_ context.Contex return fc, nil } +func (ec *executionContext) _ScopedStats_hostname(ctx context.Context, field graphql.CollectedField, obj *model.ScopedStats) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_ScopedStats_hostname(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.Hostname, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(string) + fc.Result = res + return ec.marshalNString2string(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_ScopedStats_hostname(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "ScopedStats", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type String does not have child fields") + }, + } + return fc, nil +} + +func (ec *executionContext) _ScopedStats_id(ctx context.Context, field graphql.CollectedField, obj *model.ScopedStats) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_ScopedStats_id(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.ID, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + return graphql.Null + } + res := resTmp.(*string) + fc.Result = res + return ec.marshalOString2ᚖstring(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_ScopedStats_id(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "ScopedStats", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type String does not have child fields") + }, + } + return fc, nil +} + +func (ec *executionContext) _ScopedStats_data(ctx context.Context, field graphql.CollectedField, obj *model.ScopedStats) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_ScopedStats_data(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.Data, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(*schema.MetricStatistics) + fc.Result = res + return ec.marshalNMetricStatistics2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋpkgᚋschemaᚐMetricStatistics(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_ScopedStats_data(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "ScopedStats", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "avg": + return ec.fieldContext_MetricStatistics_avg(ctx, field) + case "min": + return ec.fieldContext_MetricStatistics_min(ctx, field) + case "max": + return ec.fieldContext_MetricStatistics_max(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type MetricStatistics", field.Name) + }, + } + return fc, nil +} + func (ec *executionContext) _Series_hostname(ctx context.Context, field graphql.CollectedField, obj *schema.Series) (ret graphql.Marshaler) { fc, err := ec.fieldContext_Series_hostname(ctx, field) if err != nil { @@ -16569,50 +17064,6 @@ func (ec *executionContext) _JobMetric(ctx context.Context, sel ast.SelectionSet return out } -var jobMetricStatWithNameImplementors = []string{"JobMetricStatWithName"} - -func (ec *executionContext) _JobMetricStatWithName(ctx context.Context, sel ast.SelectionSet, obj *model.JobMetricStatWithName) graphql.Marshaler { - fields := graphql.CollectFields(ec.OperationContext, sel, jobMetricStatWithNameImplementors) - - out := graphql.NewFieldSet(fields) - deferred := make(map[string]*graphql.FieldSet) - for i, field := range fields { - switch field.Name { - case "__typename": - out.Values[i] = graphql.MarshalString("JobMetricStatWithName") - case "name": - out.Values[i] = ec._JobMetricStatWithName_name(ctx, field, obj) - if out.Values[i] == graphql.Null { - out.Invalids++ - } - case "stats": - out.Values[i] = ec._JobMetricStatWithName_stats(ctx, field, obj) - if out.Values[i] == graphql.Null { - out.Invalids++ - } - default: - panic("unknown field " + strconv.Quote(field.Name)) - } - } - out.Dispatch(ctx) - if out.Invalids > 0 { - return graphql.Null - } - - atomic.AddInt32(&ec.deferred, int32(len(deferred))) - - for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ - Label: label, - Path: graphql.GetPath(ctx), - FieldSet: dfs, - Context: ctx, - }) - } - - return out -} - var jobMetricWithNameImplementors = []string{"JobMetricWithName"} func (ec *executionContext) _JobMetricWithName(ctx context.Context, sel ast.SelectionSet, obj *model.JobMetricWithName) graphql.Marshaler { @@ -16709,6 +17160,99 @@ func (ec *executionContext) _JobResultList(ctx context.Context, sel ast.Selectio return out } +var jobStatsImplementors = []string{"JobStats"} + +func (ec *executionContext) _JobStats(ctx context.Context, sel ast.SelectionSet, obj *model.JobStats) graphql.Marshaler { + fields := graphql.CollectFields(ec.OperationContext, sel, jobStatsImplementors) + + out := graphql.NewFieldSet(fields) + deferred := make(map[string]*graphql.FieldSet) + for i, field := range fields { + switch field.Name { + case "__typename": + out.Values[i] = graphql.MarshalString("JobStats") + case "name": + out.Values[i] = ec._JobStats_name(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + case "stats": + out.Values[i] = ec._JobStats_stats(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + default: + panic("unknown field " + strconv.Quote(field.Name)) + } + } + out.Dispatch(ctx) + if out.Invalids > 0 { + return graphql.Null + } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + + return out +} + +var jobStatsWithScopeImplementors = []string{"JobStatsWithScope"} + +func (ec *executionContext) _JobStatsWithScope(ctx context.Context, sel ast.SelectionSet, obj *model.JobStatsWithScope) graphql.Marshaler { + fields := graphql.CollectFields(ec.OperationContext, sel, jobStatsWithScopeImplementors) + + out := graphql.NewFieldSet(fields) + deferred := make(map[string]*graphql.FieldSet) + for i, field := range fields { + switch field.Name { + case "__typename": + out.Values[i] = graphql.MarshalString("JobStatsWithScope") + case "name": + out.Values[i] = ec._JobStatsWithScope_name(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + case "scope": + out.Values[i] = ec._JobStatsWithScope_scope(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + case "stats": + out.Values[i] = ec._JobStatsWithScope_stats(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + default: + panic("unknown field " + strconv.Quote(field.Name)) + } + } + out.Dispatch(ctx) + if out.Invalids > 0 { + return graphql.Null + } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + + return out +} + var jobsStatisticsImplementors = []string{"JobsStatistics"} func (ec *executionContext) _JobsStatistics(ctx context.Context, sel ast.SelectionSet, obj *model.JobsStatistics) graphql.Marshaler { @@ -17513,7 +18057,7 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr } out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return rrm(innerCtx) }) - case "jobMetricStats": + case "jobStats": field := field innerFunc := func(ctx context.Context, fs *graphql.FieldSet) (res graphql.Marshaler) { @@ -17522,7 +18066,29 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr ec.Error(ctx, ec.Recover(ctx, r)) } }() - res = ec._Query_jobMetricStats(ctx, field) + res = ec._Query_jobStats(ctx, field) + if res == graphql.Null { + atomic.AddUint32(&fs.Invalids, 1) + } + return res + } + + rrm := func(ctx context.Context) graphql.Marshaler { + return ec.OperationContext.RootResolverMiddleware(ctx, + func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) + } + + out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return rrm(innerCtx) }) + case "scopedJobStats": + field := field + + innerFunc := func(ctx context.Context, fs *graphql.FieldSet) (res graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + } + }() + res = ec._Query_scopedJobStats(ctx, field) if res == graphql.Null { atomic.AddUint32(&fs.Invalids, 1) } @@ -17740,6 +18306,52 @@ func (ec *executionContext) _Resource(ctx context.Context, sel ast.SelectionSet, return out } +var scopedStatsImplementors = []string{"ScopedStats"} + +func (ec *executionContext) _ScopedStats(ctx context.Context, sel ast.SelectionSet, obj *model.ScopedStats) graphql.Marshaler { + fields := graphql.CollectFields(ec.OperationContext, sel, scopedStatsImplementors) + + out := graphql.NewFieldSet(fields) + deferred := make(map[string]*graphql.FieldSet) + for i, field := range fields { + switch field.Name { + case "__typename": + out.Values[i] = graphql.MarshalString("ScopedStats") + case "hostname": + out.Values[i] = ec._ScopedStats_hostname(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + case "id": + out.Values[i] = ec._ScopedStats_id(ctx, field, obj) + case "data": + out.Values[i] = ec._ScopedStats_data(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + default: + panic("unknown field " + strconv.Quote(field.Name)) + } + } + out.Dispatch(ctx) + if out.Invalids > 0 { + return graphql.Null + } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + + return out +} + var seriesImplementors = []string{"Series"} func (ec *executionContext) _Series(ctx context.Context, sel ast.SelectionSet, obj *schema.Series) graphql.Marshaler { @@ -19346,60 +19958,6 @@ func (ec *executionContext) marshalNJobMetric2ᚖgithubᚗcomᚋClusterCockpit return ec._JobMetric(ctx, sel, v) } -func (ec *executionContext) marshalNJobMetricStatWithName2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobMetricStatWithNameᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.JobMetricStatWithName) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNJobMetricStatWithName2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobMetricStatWithName(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() - - for _, e := range ret { - if e == graphql.Null { - return graphql.Null - } - } - - return ret -} - -func (ec *executionContext) marshalNJobMetricStatWithName2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobMetricStatWithName(ctx context.Context, sel ast.SelectionSet, v *model.JobMetricStatWithName) graphql.Marshaler { - if v == nil { - if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { - ec.Errorf(ctx, "the requested element is null which the schema does not allow") - } - return graphql.Null - } - return ec._JobMetricStatWithName(ctx, sel, v) -} - func (ec *executionContext) marshalNJobMetricWithName2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobMetricWithNameᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.JobMetricWithName) graphql.Marshaler { ret := make(graphql.Array, len(v)) var wg sync.WaitGroup @@ -19478,6 +20036,114 @@ func (ec *executionContext) marshalNJobState2githubᚗcomᚋClusterCockpitᚋcc return v } +func (ec *executionContext) marshalNJobStats2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobStatsᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.JobStats) graphql.Marshaler { + ret := make(graphql.Array, len(v)) + var wg sync.WaitGroup + isLen1 := len(v) == 1 + if !isLen1 { + wg.Add(len(v)) + } + for i := range v { + i := i + fc := &graphql.FieldContext{ + Index: &i, + Result: &v[i], + } + ctx := graphql.WithFieldContext(ctx, fc) + f := func(i int) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = nil + } + }() + if !isLen1 { + defer wg.Done() + } + ret[i] = ec.marshalNJobStats2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobStats(ctx, sel, v[i]) + } + if isLen1 { + f(i) + } else { + go f(i) + } + + } + wg.Wait() + + for _, e := range ret { + if e == graphql.Null { + return graphql.Null + } + } + + return ret +} + +func (ec *executionContext) marshalNJobStats2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobStats(ctx context.Context, sel ast.SelectionSet, v *model.JobStats) graphql.Marshaler { + if v == nil { + if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { + ec.Errorf(ctx, "the requested element is null which the schema does not allow") + } + return graphql.Null + } + return ec._JobStats(ctx, sel, v) +} + +func (ec *executionContext) marshalNJobStatsWithScope2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobStatsWithScopeᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.JobStatsWithScope) graphql.Marshaler { + ret := make(graphql.Array, len(v)) + var wg sync.WaitGroup + isLen1 := len(v) == 1 + if !isLen1 { + wg.Add(len(v)) + } + for i := range v { + i := i + fc := &graphql.FieldContext{ + Index: &i, + Result: &v[i], + } + ctx := graphql.WithFieldContext(ctx, fc) + f := func(i int) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = nil + } + }() + if !isLen1 { + defer wg.Done() + } + ret[i] = ec.marshalNJobStatsWithScope2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobStatsWithScope(ctx, sel, v[i]) + } + if isLen1 { + f(i) + } else { + go f(i) + } + + } + wg.Wait() + + for _, e := range ret { + if e == graphql.Null { + return graphql.Null + } + } + + return ret +} + +func (ec *executionContext) marshalNJobStatsWithScope2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobStatsWithScope(ctx context.Context, sel ast.SelectionSet, v *model.JobStatsWithScope) graphql.Marshaler { + if v == nil { + if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { + ec.Errorf(ctx, "the requested element is null which the schema does not allow") + } + return graphql.Null + } + return ec._JobStatsWithScope(ctx, sel, v) +} + func (ec *executionContext) marshalNJobsStatistics2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobsStatisticsᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.JobsStatistics) graphql.Marshaler { ret := make(graphql.Array, len(v)) var wg sync.WaitGroup @@ -19891,6 +20557,60 @@ func (ec *executionContext) marshalNResource2ᚖgithubᚗcomᚋClusterCockpitᚋ return ec._Resource(ctx, sel, v) } +func (ec *executionContext) marshalNScopedStats2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐScopedStatsᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.ScopedStats) graphql.Marshaler { + ret := make(graphql.Array, len(v)) + var wg sync.WaitGroup + isLen1 := len(v) == 1 + if !isLen1 { + wg.Add(len(v)) + } + for i := range v { + i := i + fc := &graphql.FieldContext{ + Index: &i, + Result: &v[i], + } + ctx := graphql.WithFieldContext(ctx, fc) + f := func(i int) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = nil + } + }() + if !isLen1 { + defer wg.Done() + } + ret[i] = ec.marshalNScopedStats2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐScopedStats(ctx, sel, v[i]) + } + if isLen1 { + f(i) + } else { + go f(i) + } + + } + wg.Wait() + + for _, e := range ret { + if e == graphql.Null { + return graphql.Null + } + } + + return ret +} + +func (ec *executionContext) marshalNScopedStats2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐScopedStats(ctx context.Context, sel ast.SelectionSet, v *model.ScopedStats) graphql.Marshaler { + if v == nil { + if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { + ec.Errorf(ctx, "the requested element is null which the schema does not allow") + } + return graphql.Null + } + return ec._ScopedStats(ctx, sel, v) +} + func (ec *executionContext) marshalNSeries2githubᚗcomᚋClusterCockpitᚋccᚑbackendᚋpkgᚋschemaᚐSeries(ctx context.Context, sel ast.SelectionSet, v schema.Series) graphql.Marshaler { return ec._Series(ctx, sel, &v) } diff --git a/internal/graph/model/models_gen.go b/internal/graph/model/models_gen.go index d83a318..43c4e37 100644 --- a/internal/graph/model/models_gen.go +++ b/internal/graph/model/models_gen.go @@ -81,11 +81,6 @@ type JobLinkResultList struct { Count *int `json:"count,omitempty"` } -type JobMetricStatWithName struct { - Name string `json:"name"` - Stats *schema.MetricStatistics `json:"stats"` -} - type JobMetricWithName struct { Name string `json:"name"` Scope schema.MetricScope `json:"scope"` @@ -100,6 +95,17 @@ type JobResultList struct { HasNextPage *bool `json:"hasNextPage,omitempty"` } +type JobStats struct { + Name string `json:"name"` + Stats *schema.MetricStatistics `json:"stats"` +} + +type JobStatsWithScope struct { + Name string `json:"name"` + Scope schema.MetricScope `json:"scope"` + Stats []*ScopedStats `json:"stats"` +} + type JobsStatistics struct { ID string `json:"id"` Name string `json:"name"` @@ -173,6 +179,12 @@ type PageRequest struct { Page int `json:"page"` } +type ScopedStats struct { + Hostname string `json:"hostname"` + ID *string `json:"id,omitempty"` + Data *schema.MetricStatistics `json:"data"` +} + type StringInput struct { Eq *string `json:"eq,omitempty"` Neq *string `json:"neq,omitempty"` diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index ce1384b..1565c7e 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -301,24 +301,23 @@ func (r *queryResolver) JobMetrics(ctx context.Context, id string, metrics []str return res, err } -// JobMetricStats is the resolver for the jobMetricStats field. -func (r *queryResolver) JobMetricStats(ctx context.Context, id string, metrics []string) ([]*model.JobMetricStatWithName, error) { - +// JobMetricStats is the resolver for the jobStats field. +func (r *queryResolver) JobStats(ctx context.Context, id string, metrics []string) ([]*model.JobStats, error) { job, err := r.Query().Job(ctx, id) if err != nil { - log.Warn("Error while querying job for metrics") + log.Warnf("Error while querying job %s for metrics", id) return nil, err } - data, err := metricDataDispatcher.LoadStatData(job, metrics, ctx) + data, err := metricDataDispatcher.LoadJobStats(job, metrics, ctx) if err != nil { - log.Warn("Error while loading job stat data") + log.Warnf("Error while loading job stat data for job id %s", id) return nil, err } - res := []*model.JobMetricStatWithName{} + res := []*model.JobStats{} for name, md := range data { - res = append(res, &model.JobMetricStatWithName{ + res = append(res, &model.JobStats{ Name: name, Stats: &md, }) @@ -327,6 +326,47 @@ func (r *queryResolver) JobMetricStats(ctx context.Context, id string, metrics [ return res, err } +// JobStats is the resolver for the scopedJobStats field. +func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope) ([]*model.JobStatsWithScope, error) { + job, err := r.Query().Job(ctx, id) + if err != nil { + log.Warnf("Error while querying job %s for metrics", id) + return nil, err + } + + data, err := metricDataDispatcher.LoadScopedJobStats(job, metrics, scopes, ctx) + if err != nil { + log.Warnf("Error while loading scoped job stat data for job id %s", id) + return nil, err + } + + res := make([]*model.JobStatsWithScope, 0) + for name, scoped := range data { + for scope, stats := range scoped { + // log.Debugf("HANDLE >>>>> %s @ %s -> First Array Value %#v", name, scope, *stats[0]) + + mdlStats := make([]*model.ScopedStats, 0) + for _, stat := range stats { + // log.Debugf("CONVERT >>>>> >>>>> %s -> %v -> %#v", stat.Hostname, stat.Id, stat.Data) + mdlStats = append(mdlStats, &model.ScopedStats{ + Hostname: stat.Hostname, + ID: stat.Id, + Data: stat.Data, + }) + } + + // log.Debugf("APPEND >>>>> >>>>> %#v", mdlStats) + res = append(res, &model.JobStatsWithScope{ + Name: name, + Scope: scope, + Stats: mdlStats, + }) + } + } + + return res, nil +} + // JobsFootprints is the resolver for the jobsFootprints field. func (r *queryResolver) JobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) { // NOTE: Legacy Naming! This resolver is for normalized histograms in analysis view only - *Not* related to DB "footprint" column! diff --git a/internal/metricDataDispatcher/dataLoader.go b/internal/metricDataDispatcher/dataLoader.go index f3f60b4..c6cecd8 100644 --- a/internal/metricDataDispatcher/dataLoader.go +++ b/internal/metricDataDispatcher/dataLoader.go @@ -224,8 +224,34 @@ func LoadAverages( return nil } -// Used for polar plots in frontend -func LoadStatData( +// Used for statsTable in frontend: Return scoped statistics by metric. +func LoadScopedJobStats( + job *schema.Job, + metrics []string, + scopes []schema.MetricScope, + ctx context.Context, +) (schema.ScopedJobStats, error) { + + if job.State != schema.JobStateRunning && !config.Keys.DisableArchive { + return archive.LoadScopedStatsFromArchive(job, metrics, scopes) + } + + repo, err := metricdata.GetMetricDataRepo(job.Cluster) + if err != nil { + return nil, fmt.Errorf("job %d: no metric data repository configured for '%s'", job.JobID, job.Cluster) + } + + scopedStats, err := repo.LoadScopedStats(job, metrics, scopes, ctx) + if err != nil { + log.Errorf("error while loading scoped statistics for job %d (User %s, Project %s)", job.JobID, job.User, job.Project) + return nil, err + } + + return scopedStats, nil +} + +// Used for polar plots in frontend: Aggregates statistics for all nodes to single values for job per metric. +func LoadJobStats( job *schema.Job, metrics []string, ctx context.Context, @@ -237,12 +263,12 @@ func LoadStatData( data := make(map[string]schema.MetricStatistics, len(metrics)) repo, err := metricdata.GetMetricDataRepo(job.Cluster) if err != nil { - return data, fmt.Errorf("METRICDATA/METRICDATA > no metric data repository configured for '%s'", job.Cluster) + return data, fmt.Errorf("job %d: no metric data repository configured for '%s'", job.JobID, job.Cluster) } stats, err := repo.LoadStats(job, metrics, ctx) if err != nil { - log.Errorf("Error while loading statistics for job %v (User %v, Project %v)", job.JobID, job.User, job.Project) + log.Errorf("error while loading statistics for job %d (User %s, Project %s)", job.JobID, job.User, job.Project) return data, err } diff --git a/internal/metricdata/cc-metric-store.go b/internal/metricdata/cc-metric-store.go index 2b92fbb..6635299 100644 --- a/internal/metricdata/cc-metric-store.go +++ b/internal/metricdata/cc-metric-store.go @@ -618,7 +618,98 @@ func (ccms *CCMetricStore) LoadStats( return stats, nil } -// TODO: Support sub-node-scope metrics! For this, the partition of a node needs to be known! +// Scoped Stats: Basically Load Data without resolution and data query flag? +func (ccms *CCMetricStore) LoadScopedStats( + job *schema.Job, + metrics []string, + scopes []schema.MetricScope, + ctx context.Context, +) (schema.ScopedJobStats, error) { + queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, 0) + if err != nil { + log.Warn("Error while building queries") + return nil, err + } + + req := ApiQueryRequest{ + Cluster: job.Cluster, + From: job.StartTime.Unix(), + To: job.StartTime.Add(time.Duration(job.Duration) * time.Second).Unix(), + Queries: queries, + WithStats: true, + WithData: false, + } + + resBody, err := ccms.doRequest(ctx, &req) + if err != nil { + log.Error("Error while performing request") + return nil, err + } + + var errors []string + scopedJobStats := make(schema.ScopedJobStats) + + for i, row := range resBody.Results { + query := req.Queries[i] + metric := ccms.toLocalName(query.Metric) + scope := assignedScope[i] + + if _, ok := scopedJobStats[metric]; !ok { + scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats) + } + + if _, ok := scopedJobStats[metric][scope]; !ok { + scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0) + } + + for ndx, res := range row { + if res.Error != nil { + /* Build list for "partial errors", if any */ + errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error)) + continue + } + + id := (*string)(nil) + if query.Type != nil { + id = new(string) + *id = query.TypeIds[ndx] + } + + if res.Avg.IsNaN() || res.Min.IsNaN() || res.Max.IsNaN() { + // "schema.Float()" because regular float64 can not be JSONed when NaN. + res.Avg = schema.Float(0) + res.Min = schema.Float(0) + res.Max = schema.Float(0) + } + + scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{ + Hostname: query.Hostname, + Id: id, + Data: &schema.MetricStatistics{ + Avg: float64(res.Avg), + Min: float64(res.Min), + Max: float64(res.Max), + }, + }) + } + + // So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty + if len(scopedJobStats[metric][scope]) == 0 { + delete(scopedJobStats[metric], scope) + if len(scopedJobStats[metric]) == 0 { + delete(scopedJobStats, metric) + } + } + } + + if len(errors) != 0 { + /* Returns list for "partial errors" */ + return scopedJobStats, fmt.Errorf("METRICDATA/CCMS > Errors: %s", strings.Join(errors, ", ")) + } + return scopedJobStats, nil +} + +// TODO: Support sub-node-scope metrics! For this, the partition of a node needs to be known! - Todo Outdated with NodeListData? func (ccms *CCMetricStore) LoadNodeData( cluster string, metrics, nodes []string, diff --git a/internal/metricdata/influxdb-v2.go b/internal/metricdata/influxdb-v2.go index 79c2d4a..2a943b6 100644 --- a/internal/metricdata/influxdb-v2.go +++ b/internal/metricdata/influxdb-v2.go @@ -301,6 +301,18 @@ func (idb *InfluxDBv2DataRepository) LoadStats( return stats, nil } +func (idb *InfluxDBv2DataRepository) LoadScopedStats( + job *schema.Job, + metrics []string, + scopes []schema.MetricScope, + ctx context.Context) (schema.ScopedJobStats, error) { + + // TODO : Implement to be used in JobView Stats Table + log.Infof("LoadScopedStats unimplemented for InfluxDBv2DataRepository, Args: Job-ID %d, metrics %v, scopes %v", job.JobID, metrics, scopes) + + return nil, errors.New("METRICDATA/INFLUXV2 > unimplemented for InfluxDBv2DataRepository") +} + func (idb *InfluxDBv2DataRepository) LoadNodeData( cluster string, metrics, nodes []string, diff --git a/internal/metricdata/metricdata.go b/internal/metricdata/metricdata.go index 0fe94d1..f30d837 100644 --- a/internal/metricdata/metricdata.go +++ b/internal/metricdata/metricdata.go @@ -24,9 +24,12 @@ type MetricDataRepository interface { // Return the JobData for the given job, only with the requested metrics. LoadData(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) - // Return a map of metrics to a map of nodes to the metric statistics of the job. node scope assumed for now. + // Return a map of metrics to a map of nodes to the metric statistics of the job. node scope only. LoadStats(job *schema.Job, metrics []string, ctx context.Context) (map[string]map[string]schema.MetricStatistics, error) + // Return a map of metrics to a map of scopes to the scoped metric statistics of the job. + LoadScopedStats(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context) (schema.ScopedJobStats, error) + // Return a map of hosts to a map of metrics at the requested scopes (currently only node) for that node. LoadNodeData(cluster string, metrics, nodes []string, scopes []schema.MetricScope, from, to time.Time, ctx context.Context) (map[string]map[string][]*schema.JobMetric, error) diff --git a/internal/metricdata/prometheus.go b/internal/metricdata/prometheus.go index cd849ce..fe829c0 100644 --- a/internal/metricdata/prometheus.go +++ b/internal/metricdata/prometheus.go @@ -448,6 +448,18 @@ func (pdb *PrometheusDataRepository) LoadNodeData( return data, nil } +func (pdb *PrometheusDataRepository) LoadScopedStats( + job *schema.Job, + metrics []string, + scopes []schema.MetricScope, + ctx context.Context) (schema.ScopedJobStats, error) { + + // TODO : Implement to be used in Job-View StatsTable + log.Infof("LoadScopedStats unimplemented for PrometheusDataRepository, Args: job-id %v, metrics %v, scopes %v", job.JobID, metrics, scopes) + + return nil, errors.New("METRICDATA/PROMETHEUS > unimplemented for PrometheusDataRepository") +} + func (pdb *PrometheusDataRepository) LoadNodeListData( cluster, subCluster, nodeFilter string, metrics []string, @@ -463,5 +475,5 @@ func (pdb *PrometheusDataRepository) LoadNodeListData( // TODO : Implement to be used in NodeList-View log.Infof("LoadNodeListData unimplemented for PrometheusDataRepository, Args: cluster %s, metrics %v, nodeFilter %v, scopes %v", cluster, metrics, nodeFilter, scopes) - return nil, totalNodes, hasNextPage, errors.New("METRICDATA/INFLUXV2 > unimplemented for PrometheusDataRepository") + return nil, totalNodes, hasNextPage, errors.New("METRICDATA/PROMETHEUS > unimplemented for PrometheusDataRepository") } diff --git a/internal/metricdata/utils.go b/internal/metricdata/utils.go index 48dd237..aa7bde1 100644 --- a/internal/metricdata/utils.go +++ b/internal/metricdata/utils.go @@ -36,7 +36,17 @@ func (tmdr *TestMetricDataRepository) LoadData( func (tmdr *TestMetricDataRepository) LoadStats( job *schema.Job, - metrics []string, ctx context.Context) (map[string]map[string]schema.MetricStatistics, error) { + metrics []string, + ctx context.Context) (map[string]map[string]schema.MetricStatistics, error) { + + panic("TODO") +} + +func (tmdr *TestMetricDataRepository) LoadScopedStats( + job *schema.Job, + metrics []string, + scopes []schema.MetricScope, + ctx context.Context) (schema.ScopedJobStats, error) { panic("TODO") } diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go index 2eabb52..002fd5e 100644 --- a/pkg/archive/archive.go +++ b/pkg/archive/archive.go @@ -27,6 +27,8 @@ type ArchiveBackend interface { LoadJobData(job *schema.Job) (schema.JobData, error) + LoadJobStats(job *schema.Job) (schema.ScopedJobStats, error) + LoadClusterCfg(name string) (*schema.Cluster, error) StoreJobMeta(jobMeta *schema.JobMeta) error @@ -125,7 +127,7 @@ func LoadAveragesFromArchive( return nil } -// Helper to metricdataloader.LoadStatData(). +// Helper to metricdataloader.LoadJobStats(). func LoadStatsFromArchive( job *schema.Job, metrics []string, @@ -154,6 +156,22 @@ func LoadStatsFromArchive( return data, nil } +// Helper to metricdataloader.LoadScopedJobStats(). +func LoadScopedStatsFromArchive( + job *schema.Job, + metrics []string, + scopes []schema.MetricScope, +) (schema.ScopedJobStats, error) { + + data, err := ar.LoadJobStats(job) + if err != nil { + log.Warn("Error while loading job metadata from archiveBackend") + return nil, err + } + + return data, nil +} + func GetStatistics(job *schema.Job) (map[string]schema.JobStatistics, error) { metaFile, err := ar.LoadJobMeta(job) if err != nil { diff --git a/pkg/archive/fsBackend.go b/pkg/archive/fsBackend.go index 8a43748..711b1f5 100644 --- a/pkg/archive/fsBackend.go +++ b/pkg/archive/fsBackend.go @@ -115,6 +115,40 @@ func loadJobData(filename string, isCompressed bool) (schema.JobData, error) { } } +func loadJobStats(filename string, isCompressed bool) (schema.ScopedJobStats, error) { + f, err := os.Open(filename) + + if err != nil { + log.Errorf("fsBackend LoadJobStats()- %v", err) + return nil, err + } + defer f.Close() + + if isCompressed { + r, err := gzip.NewReader(f) + if err != nil { + log.Errorf(" %v", err) + return nil, err + } + defer r.Close() + + if config.Keys.Validate { + if err := schema.Validate(schema.Data, r); err != nil { + return nil, fmt.Errorf("validate job data: %v", err) + } + } + + return DecodeJobStats(r, filename) + } else { + if config.Keys.Validate { + if err := schema.Validate(schema.Data, bufio.NewReader(f)); err != nil { + return nil, fmt.Errorf("validate job data: %v", err) + } + } + return DecodeJobStats(bufio.NewReader(f), filename) + } +} + func (fsa *FsArchive) Init(rawConfig json.RawMessage) (uint64, error) { var config FsArchiveConfig @@ -389,6 +423,18 @@ func (fsa *FsArchive) LoadJobData(job *schema.Job) (schema.JobData, error) { return loadJobData(filename, isCompressed) } +func (fsa *FsArchive) LoadJobStats(job *schema.Job) (schema.ScopedJobStats, error) { + var isCompressed bool = true + filename := getPath(job, fsa.path, "data.json.gz") + + if !util.CheckFileExists(filename) { + filename = getPath(job, fsa.path, "data.json") + isCompressed = false + } + + return loadJobStats(filename, isCompressed) +} + func (fsa *FsArchive) LoadJobMeta(job *schema.Job) (*schema.JobMeta, error) { filename := getPath(job, fsa.path, "meta.json") return loadJobMeta(filename) diff --git a/pkg/archive/json.go b/pkg/archive/json.go index 1219658..5201b74 100644 --- a/pkg/archive/json.go +++ b/pkg/archive/json.go @@ -32,6 +32,43 @@ func DecodeJobData(r io.Reader, k string) (schema.JobData, error) { return data.(schema.JobData), nil } +func DecodeJobStats(r io.Reader, k string) (schema.ScopedJobStats, error) { + jobData, err := DecodeJobData(r, k) + // Convert schema.JobData to schema.ScopedJobStats + if jobData != nil { + scopedJobStats := make(schema.ScopedJobStats) + for metric, metricData := range jobData { + if _, ok := scopedJobStats[metric]; !ok { + scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats) + } + + for scope, jobMetric := range metricData { + if _, ok := scopedJobStats[metric][scope]; !ok { + scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0) + } + + for _, series := range jobMetric.Series { + scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{ + Hostname: series.Hostname, + Id: series.Id, + Data: &series.Statistics, + }) + } + + // So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty + if len(scopedJobStats[metric][scope]) == 0 { + delete(scopedJobStats[metric], scope) + if len(scopedJobStats[metric]) == 0 { + delete(scopedJobStats, metric) + } + } + } + } + return scopedJobStats, nil + } + return nil, err +} + func DecodeJobMeta(r io.Reader) (*schema.JobMeta, error) { var d schema.JobMeta if err := json.NewDecoder(r).Decode(&d); err != nil { diff --git a/pkg/schema/metrics.go b/pkg/schema/metrics.go index ffac21b..fbb85e4 100644 --- a/pkg/schema/metrics.go +++ b/pkg/schema/metrics.go @@ -15,6 +15,7 @@ import ( ) type JobData map[string]map[MetricScope]*JobMetric +type ScopedJobStats map[string]map[MetricScope][]*ScopedStats type JobMetric struct { StatisticsSeries *StatsSeries `json:"statisticsSeries,omitempty"` @@ -30,6 +31,12 @@ type Series struct { Statistics MetricStatistics `json:"statistics"` } +type ScopedStats struct { + Hostname string `json:"hostname"` + Id *string `json:"id,omitempty"` + Data *MetricStatistics `json:"data"` +} + type MetricStatistics struct { Avg float64 `json:"avg"` Min float64 `json:"min"` diff --git a/web/frontend/src/Job.root.svelte b/web/frontend/src/Job.root.svelte index 6980230..2fe5bc4 100644 --- a/web/frontend/src/Job.root.svelte +++ b/web/frontend/src/Job.root.svelte @@ -127,28 +127,17 @@ let job = $initq.data.job; if (!job) return; - const pendingMetrics = [ - ...( - ( - ccconfig[`job_view_selectedMetrics:${job.cluster}:${job.subCluster}`] || - ccconfig[`job_view_selectedMetrics:${job.cluster}`] - ) || - $initq.data.globalMetrics - .reduce((names, gm) => { - if (gm.availability.find((av) => av.cluster === job.cluster && av.subClusters.includes(job.subCluster))) { - names.push(gm.name); - } - return names; - }, []) - ), - ...( - ( - ccconfig[`job_view_nodestats_selectedMetrics:${job.cluster}:${job.subCluster}`] || - ccconfig[`job_view_nodestats_selectedMetrics:${job.cluster}`] - ) || - ccconfig[`job_view_nodestats_selectedMetrics`] - ), - ]; + const pendingMetrics = ( + ccconfig[`job_view_selectedMetrics:${job.cluster}:${job.subCluster}`] || + ccconfig[`job_view_selectedMetrics:${job.cluster}`] + ) || + $initq.data.globalMetrics + .reduce((names, gm) => { + if (gm.availability.find((av) => av.cluster === job.cluster && av.subClusters.includes(job.subCluster))) { + names.push(gm.name); + } + return names; + }, []) // Select default Scopes to load: Check before if any metric has accelerator scope by default const accScopeDefault = [...pendingMetrics].some(function (m) { @@ -343,7 +332,6 @@ {#if item.data} statsTable.moreLoaded(detail)} job={$initq.data.job} metricName={item.metric} metricUnit={$initq.data.globalMetrics.find((gm) => gm.name == item.metric)?.unit} @@ -404,15 +392,7 @@ class="overflow-x-auto" active={!somethingMissing} > - {#if $jobMetrics?.data?.jobMetrics} - {#key $jobMetrics.data.jobMetrics} - - {/key} - {/if} +
diff --git a/web/frontend/src/job/Metric.svelte b/web/frontend/src/job/Metric.svelte index bcfa4fd..b68ef47 100644 --- a/web/frontend/src/job/Metric.svelte +++ b/web/frontend/src/job/Metric.svelte @@ -150,11 +150,6 @@ // On additional scope request if (selectedScope == "load-all") { - // Push scope to statsTable (Needs to be in this case, else newly selected 'Metric.svelte' renders cause statsTable race condition) - const statsTableData = $metricData.data.singleUpdate.filter((x) => x.scope !== "node") - if (statsTableData.length > 0) { - dispatch("more-loaded", statsTableData); - } // Set selected scope to min of returned scopes selectedScope = minScope(scopes) nodeOnly = (selectedScope == "node") // "node" still only scope after load-all diff --git a/web/frontend/src/job/StatsTable.svelte b/web/frontend/src/job/StatsTable.svelte index c8f12f2..159d24b 100644 --- a/web/frontend/src/job/StatsTable.svelte +++ b/web/frontend/src/job/StatsTable.svelte @@ -3,13 +3,14 @@ Properties: - `job Object`: The job object - - `jobMetrics [Object]`: The jobs metricdata - - Exported: - - `moreLoaded`: Adds additional scopes requested from Metric.svelte in Job-View --> diff --git a/web/frontend/src/job/StatsTableEntry.svelte b/web/frontend/src/job/StatsTableEntry.svelte index 9504a63..dc2f628 100644 --- a/web/frontend/src/job/StatsTableEntry.svelte +++ b/web/frontend/src/job/StatsTableEntry.svelte @@ -37,8 +37,8 @@ return s.dir != "up" ? a[field] - b[field] : b[field] - a[field]; } else { return s.dir != "up" - ? a.statistics[field] - b.statistics[field] - : b.statistics[field] - a.statistics[field]; + ? a.data[field] - b.data[field] + : b.data[field] - a.data[field]; } }); } @@ -52,7 +52,7 @@ $: series = jobMetrics .find((jm) => jm.name == metric && jm.scope == scope) - ?.metric.series.filter((s) => s.hostname == host && s.statistics != null) + ?.stats.filter((s) => s.hostname == host && s.data != null) ?.sort(compareNumbers); @@ -60,13 +60,13 @@ No data {:else if series.length == 1 && scope == "node"} - {series[0].statistics.min} + {series[0].data.min} - {series[0].statistics.avg} + {series[0].data.avg} - {series[0].statistics.max} + {series[0].data.max} {:else} @@ -86,9 +86,9 @@ {#each series as s, i} {s.id ?? i} - {s.statistics.min} - {s.statistics.avg} - {s.statistics.max} + {s.data.min} + {s.data.avg} + {s.data.max} {/each} diff --git a/web/frontend/src/job/jobsummary/JobFootprintPolar.svelte b/web/frontend/src/job/jobsummary/JobFootprintPolar.svelte index cf90408..fe6693b 100644 --- a/web/frontend/src/job/jobsummary/JobFootprintPolar.svelte +++ b/web/frontend/src/job/jobsummary/JobFootprintPolar.svelte @@ -40,14 +40,14 @@ const client = getContextClient(); const polarQuery = gql` query ($dbid: ID!, $selectedMetrics: [String!]!) { - jobMetricStats(id: $dbid, metrics: $selectedMetrics) { + jobStats(id: $dbid, metrics: $selectedMetrics) { name stats { - min - avg - max - } + min + avg + max } + } } `; @@ -66,7 +66,7 @@ {:else} {/if} \ No newline at end of file From 6a7546c43b7e833ed32054b51390a94607deb356 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Fri, 14 Mar 2025 10:03:53 +0100 Subject: [PATCH 28/82] Clarify header for breaking changes --- ReleaseNotes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ReleaseNotes.md b/ReleaseNotes.md index ef1082f..f047abf 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -6,7 +6,7 @@ This is a bug fix release of `cc-backend`, the API backend and frontend implementation of ClusterCockpit. For release specific notes visit the [ClusterCockpit Documentation](https://clusterockpit.org/docs/release/). -## Breaking changes +## Breaking changes for minor release 1.4.x - You need to perform a database migration. Depending on your database size the migration might require several hours! From 25aaf55b93a13d853c6ef57d08ccc22488ddaf84 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Fri, 14 Mar 2025 10:06:25 +0100 Subject: [PATCH 29/82] Add feature to Releasenotes --- ReleaseNotes.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ReleaseNotes.md b/ReleaseNotes.md index f047abf..beb8ee1 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -28,6 +28,8 @@ For release specific notes visit the [ClusterCockpit Documentation](https://clus - Color Blind Mode - Set on a per-user basis in options - Applies to plot data, plot background color, statsseries colors, roofline timescale +- Job-View metric selection is now persisted based on the jobs subcluster. +Helpful for heterogeneous subcluster configurations. - Histogram Bin Select in User-View - Metric-Histograms: `10 Bins` now default, selectable options `20, 50, 100` - Job-Duration-Histogram: `48h in 1h Bins` now default, selectable options: From 33c6cdb9feaa3bca13571c58bedc541550f1ff28 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Fri, 14 Mar 2025 10:52:27 +0100 Subject: [PATCH 30/82] Update test workflow --- .github/workflows/Release.yml | 331 ---------------------------------- .github/workflows/test.yml | 2 +- 2 files changed, 1 insertion(+), 332 deletions(-) delete mode 100644 .github/workflows/Release.yml diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml deleted file mode 100644 index 8fc8755..0000000 --- a/.github/workflows/Release.yml +++ /dev/null @@ -1,331 +0,0 @@ -# See: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions - -# Workflow name -name: Release - -# Run on tag push -on: - push: - tags: - - '**' - -jobs: - - # - # Build on AlmaLinux 8.5 using golang-1.18.2 - # - AlmaLinux-RPM-build: - runs-on: ubuntu-latest - # See: https://hub.docker.com/_/almalinux - container: almalinux:8.5 - # The job outputs link to the outputs of the 'rpmrename' step - # Only job outputs can be used in child jobs - outputs: - rpm : ${{steps.rpmrename.outputs.RPM}} - srpm : ${{steps.rpmrename.outputs.SRPM}} - steps: - - # Use dnf to install development packages - - name: Install development packages - run: | - dnf --assumeyes group install "Development Tools" "RPM Development Tools" - dnf --assumeyes install wget openssl-devel diffutils delve which npm - dnf --assumeyes install 'dnf-command(builddep)' - - # Checkout git repository and submodules - # fetch-depth must be 0 to use git describe - # See: https://github.com/marketplace/actions/checkout - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: recursive - fetch-depth: 0 - - # Use dnf to install build dependencies - - name: Install build dependencies - run: | - wget -q http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \ - http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-bin-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \ - http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-src-1.18.2-1.module_el8.7.0+1173+5d37c0fd.noarch.rpm \ - http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/go-toolset-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm - rpm -i go*.rpm - npm install --global yarn rollup svelte rollup-plugin-svelte - #dnf --assumeyes builddep build/package/cc-backend.spec - - - name: RPM build ClusterCockpit - id: rpmbuild - run: make RPM - - # AlmaLinux 8.5 is a derivate of RedHat Enterprise Linux 8 (UBI8), - # so the created RPM both contain the substring 'el8' in the RPM file names - # This step replaces the substring 'el8' to 'alma85'. It uses the move operation - # because it is unclear whether the default AlmaLinux 8.5 container contains the - # 'rename' command. This way we also get the new names for output. - - name: Rename RPMs (s/el8/alma85/) - id: rpmrename - run: | - OLD_RPM="${{steps.rpmbuild.outputs.RPM}}" - OLD_SRPM="${{steps.rpmbuild.outputs.SRPM}}" - NEW_RPM="${OLD_RPM/el8/alma85}" - NEW_SRPM=${OLD_SRPM/el8/alma85} - mv "${OLD_RPM}" "${NEW_RPM}" - mv "${OLD_SRPM}" "${NEW_SRPM}" - echo "::set-output name=SRPM::${NEW_SRPM}" - echo "::set-output name=RPM::${NEW_RPM}" - - # See: https://github.com/actions/upload-artifact - - name: Save RPM as artifact - uses: actions/upload-artifact@v2 - with: - name: cc-backend RPM for AlmaLinux 8.5 - path: ${{ steps.rpmrename.outputs.RPM }} - - name: Save SRPM as artifact - uses: actions/upload-artifact@v2 - with: - name: cc-backend SRPM for AlmaLinux 8.5 - path: ${{ steps.rpmrename.outputs.SRPM }} - - # - # Build on UBI 8 using golang-1.18.2 - # - UBI-8-RPM-build: - runs-on: ubuntu-latest - # See: https://catalog.redhat.com/software/containers/ubi8/ubi/5c359854d70cc534b3a3784e?container-tabs=gti - container: registry.access.redhat.com/ubi8/ubi:8.5-226.1645809065 - # The job outputs link to the outputs of the 'rpmbuild' step - outputs: - rpm : ${{steps.rpmbuild.outputs.RPM}} - srpm : ${{steps.rpmbuild.outputs.SRPM}} - steps: - - # Use dnf to install development packages - - name: Install development packages - run: dnf --assumeyes --disableplugin=subscription-manager install rpm-build go-srpm-macros rpm-build-libs rpm-libs gcc make python38 git wget openssl-devel diffutils delve which - - # Checkout git repository and submodules - # fetch-depth must be 0 to use git describe - # See: https://github.com/marketplace/actions/checkout - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: recursive - fetch-depth: 0 - - # Use dnf to install build dependencies - - name: Install build dependencies - run: | - wget -q http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \ - http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-bin-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \ - http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-src-1.18.2-1.module_el8.7.0+1173+5d37c0fd.noarch.rpm \ - http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/go-toolset-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm - rpm -i go*.rpm - dnf --assumeyes --disableplugin=subscription-manager install npm - npm install --global yarn rollup svelte rollup-plugin-svelte - #dnf --assumeyes builddep build/package/cc-backend.spec - - - name: RPM build ClusterCockpit - id: rpmbuild - run: make RPM - - # See: https://github.com/actions/upload-artifact - - name: Save RPM as artifact - uses: actions/upload-artifact@v2 - with: - name: cc-backend RPM for UBI 8 - path: ${{ steps.rpmbuild.outputs.RPM }} - - name: Save SRPM as artifact - uses: actions/upload-artifact@v2 - with: - name: cc-backend SRPM for UBI 8 - path: ${{ steps.rpmbuild.outputs.SRPM }} - - # - # Build on Ubuntu 20.04 using official go 1.19.1 package - # - Ubuntu-focal-build: - runs-on: ubuntu-latest - container: ubuntu:20.04 - # The job outputs link to the outputs of the 'debrename' step - # Only job outputs can be used in child jobs - outputs: - deb : ${{steps.debrename.outputs.DEB}} - steps: - # Use apt to install development packages - - name: Install development packages - run: | - apt update && apt --assume-yes upgrade - apt --assume-yes install build-essential sed git wget bash - apt --assume-yes install npm - npm install --global yarn rollup svelte rollup-plugin-svelte - # Checkout git repository and submodules - # fetch-depth must be 0 to use git describe - # See: https://github.com/marketplace/actions/checkout - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: recursive - fetch-depth: 0 - # Use official golang package - - name: Install Golang - run: | - wget -q https://go.dev/dl/go1.19.1.linux-amd64.tar.gz - tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz - export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH - go version - - name: DEB build ClusterCockpit - id: dpkg-build - run: | - ls -la - pwd - env - export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH - git config --global --add safe.directory $(pwd) - make DEB - - name: Rename DEB (add '_ubuntu20.04') - id: debrename - run: | - OLD_DEB_NAME=$(echo "${{steps.dpkg-build.outputs.DEB}}" | rev | cut -d '.' -f 2- | rev) - NEW_DEB_FILE="${OLD_DEB_NAME}_ubuntu20.04.deb" - mv "${{steps.dpkg-build.outputs.DEB}}" "${NEW_DEB_FILE}" - echo "::set-output name=DEB::${NEW_DEB_FILE}" - # See: https://github.com/actions/upload-artifact - - name: Save DEB as artifact - uses: actions/upload-artifact@v2 - with: - name: cc-backend DEB for Ubuntu 20.04 - path: ${{ steps.debrename.outputs.DEB }} - - # - # Build on Ubuntu 20.04 using official go 1.19.1 package - # - Ubuntu-jammy-build: - runs-on: ubuntu-latest - container: ubuntu:22.04 - # The job outputs link to the outputs of the 'debrename' step - # Only job outputs can be used in child jobs - outputs: - deb : ${{steps.debrename.outputs.DEB}} - steps: - # Use apt to install development packages - - name: Install development packages - run: | - apt update && apt --assume-yes upgrade - apt --assume-yes install build-essential sed git wget bash npm - npm install --global yarn rollup svelte rollup-plugin-svelte - # Checkout git repository and submodules - # fetch-depth must be 0 to use git describe - # See: https://github.com/marketplace/actions/checkout - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: recursive - fetch-depth: 0 - # Use official golang package - - name: Install Golang - run: | - wget -q https://go.dev/dl/go1.19.1.linux-amd64.tar.gz - tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz - export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH - go version - - name: DEB build ClusterCockpit - id: dpkg-build - run: | - ls -la - pwd - env - export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH - git config --global --add safe.directory $(pwd) - make DEB - - name: Rename DEB (add '_ubuntu22.04') - id: debrename - run: | - OLD_DEB_NAME=$(echo "${{steps.dpkg-build.outputs.DEB}}" | rev | cut -d '.' -f 2- | rev) - NEW_DEB_FILE="${OLD_DEB_NAME}_ubuntu22.04.deb" - mv "${{steps.dpkg-build.outputs.DEB}}" "${NEW_DEB_FILE}" - echo "::set-output name=DEB::${NEW_DEB_FILE}" - # See: https://github.com/actions/upload-artifact - - name: Save DEB as artifact - uses: actions/upload-artifact@v2 - with: - name: cc-backend DEB for Ubuntu 22.04 - path: ${{ steps.debrename.outputs.DEB }} - - # - # Create release with fresh RPMs - # - Release: - runs-on: ubuntu-latest - # We need the RPMs, so add dependency - needs: [AlmaLinux-RPM-build, UBI-8-RPM-build, Ubuntu-focal-build, Ubuntu-jammy-build] - - steps: - # See: https://github.com/actions/download-artifact - - name: Download AlmaLinux 8.5 RPM - uses: actions/download-artifact@v2 - with: - name: cc-backend RPM for AlmaLinux 8.5 - - name: Download AlmaLinux 8.5 SRPM - uses: actions/download-artifact@v2 - with: - name: cc-backend SRPM for AlmaLinux 8.5 - - - name: Download UBI 8 RPM - uses: actions/download-artifact@v2 - with: - name: cc-backend RPM for UBI 8 - - name: Download UBI 8 SRPM - uses: actions/download-artifact@v2 - with: - name: cc-backend SRPM for UBI 8 - - - name: Download Ubuntu 20.04 DEB - uses: actions/download-artifact@v2 - with: - name: cc-backend DEB for Ubuntu 20.04 - - - name: Download Ubuntu 22.04 DEB - uses: actions/download-artifact@v2 - with: - name: cc-backend DEB for Ubuntu 22.04 - - # The download actions do not publish the name of the downloaded file, - # so we re-use the job outputs of the parent jobs. The files are all - # downloaded to the current folder. - # The gh-release action afterwards does not accept file lists but all - # files have to be listed at 'files'. The step creates one output per - # RPM package (2 per distro) - - name: Set RPM variables - id: files - run: | - ALMA_85_RPM=$(basename "${{ needs.AlmaLinux-RPM-build.outputs.rpm}}") - ALMA_85_SRPM=$(basename "${{ needs.AlmaLinux-RPM-build.outputs.srpm}}") - UBI_8_RPM=$(basename "${{ needs.UBI-8-RPM-build.outputs.rpm}}") - UBI_8_SRPM=$(basename "${{ needs.UBI-8-RPM-build.outputs.srpm}}") - U_2004_DEB=$(basename "${{ needs.Ubuntu-focal-build.outputs.deb}}") - U_2204_DEB=$(basename "${{ needs.Ubuntu-jammy-build.outputs.deb}}") - echo "ALMA_85_RPM::${ALMA_85_RPM}" - echo "ALMA_85_SRPM::${ALMA_85_SRPM}" - echo "UBI_8_RPM::${UBI_8_RPM}" - echo "UBI_8_SRPM::${UBI_8_SRPM}" - echo "U_2004_DEB::${U_2004_DEB}" - echo "U_2204_DEB::${U_2204_DEB}" - echo "::set-output name=ALMA_85_RPM::${ALMA_85_RPM}" - echo "::set-output name=ALMA_85_SRPM::${ALMA_85_SRPM}" - echo "::set-output name=UBI_8_RPM::${UBI_8_RPM}" - echo "::set-output name=UBI_8_SRPM::${UBI_8_SRPM}" - echo "::set-output name=U_2004_DEB::${U_2004_DEB}" - echo "::set-output name=U_2204_DEB::${U_2204_DEB}" - - # See: https://github.com/softprops/action-gh-release - - name: Release - uses: softprops/action-gh-release@v1 - if: startsWith(github.ref, 'refs/tags/') - with: - name: cc-backend-${{github.ref_name}} - files: | - ${{ steps.files.outputs.ALMA_85_RPM }} - ${{ steps.files.outputs.ALMA_85_SRPM }} - ${{ steps.files.outputs.UBI_8_RPM }} - ${{ steps.files.outputs.UBI_8_SRPM }} - ${{ steps.files.outputs.U_2004_DEB }} - ${{ steps.files.outputs.U_2204_DEB }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e4aa02b..a8a7429 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: - name: Install Go uses: actions/setup-go@v4 with: - go-version: 1.22.x + go-version: 1.24.x - name: Checkout code uses: actions/checkout@v3 - name: Build, Vet & Test From 8da2fc30c39d2ce4b3c7c2b532b480851f431d89 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 14 Mar 2025 16:36:31 +0100 Subject: [PATCH 31/82] split statsTable data from jobMetrics query, frontend refactor --- internal/graph/schema.resolvers.go | 3 - web/frontend/src/Job.root.svelte | 18 +- web/frontend/src/job/StatsTab.svelte | 145 +++++++++++++ web/frontend/src/job/StatsTable.svelte | 201 ------------------ .../src/job/statstab/StatsTable.svelte | 139 ++++++++++++ .../job/{ => statstab}/StatsTableEntry.svelte | 50 ++--- 6 files changed, 314 insertions(+), 242 deletions(-) create mode 100644 web/frontend/src/job/StatsTab.svelte delete mode 100644 web/frontend/src/job/StatsTable.svelte create mode 100644 web/frontend/src/job/statstab/StatsTable.svelte rename web/frontend/src/job/{ => statstab}/StatsTableEntry.svelte (67%) diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index 1565c7e..a470807 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -343,11 +343,9 @@ func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics [ res := make([]*model.JobStatsWithScope, 0) for name, scoped := range data { for scope, stats := range scoped { - // log.Debugf("HANDLE >>>>> %s @ %s -> First Array Value %#v", name, scope, *stats[0]) mdlStats := make([]*model.ScopedStats, 0) for _, stat := range stats { - // log.Debugf("CONVERT >>>>> >>>>> %s -> %v -> %#v", stat.Hostname, stat.Id, stat.Data) mdlStats = append(mdlStats, &model.ScopedStats{ Hostname: stat.Hostname, ID: stat.Id, @@ -355,7 +353,6 @@ func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics [ }) } - // log.Debugf("APPEND >>>>> >>>>> %#v", mdlStats) res = append(res, &model.JobStatsWithScope{ Name: name, Scope: scope, diff --git a/web/frontend/src/Job.root.svelte b/web/frontend/src/Job.root.svelte index 2fe5bc4..c2748e6 100644 --- a/web/frontend/src/Job.root.svelte +++ b/web/frontend/src/Job.root.svelte @@ -40,7 +40,7 @@ import JobRoofline from "./job/JobRoofline.svelte"; import EnergySummary from "./job/EnergySummary.svelte"; import PlotGrid from "./generic/PlotGrid.svelte"; - import StatsTable from "./job/StatsTable.svelte"; + import StatsTab from "./job/StatsTab.svelte"; export let dbid; export let username; @@ -53,10 +53,8 @@ let isMetricsSelectionOpen = false, selectedMetrics = [], - selectedScopes = []; - - let plots = {}, - statsTable + selectedScopes = [], + plots = {}; let availableMetrics = new Set(), missingMetrics = [], @@ -386,14 +384,8 @@
{/if} - - - + +
{#if $initq.data.job.metaData?.jobScript} diff --git a/web/frontend/src/job/StatsTab.svelte b/web/frontend/src/job/StatsTab.svelte new file mode 100644 index 0000000..b7647b5 --- /dev/null +++ b/web/frontend/src/job/StatsTab.svelte @@ -0,0 +1,145 @@ + + + + + + + + + {#if job.numNodes > 1} + + {/if} + + +
+ + {#if $scopedStats.fetching} + + + + + + {:else if $scopedStats.error} + + + {$scopedStats.error.message} + + + {:else} + r.hostname).sort()} + data={$scopedStats?.data?.scopedJobStats} + {selectedMetrics} + /> + {/if} + + + diff --git a/web/frontend/src/job/StatsTable.svelte b/web/frontend/src/job/StatsTable.svelte deleted file mode 100644 index 159d24b..0000000 --- a/web/frontend/src/job/StatsTable.svelte +++ /dev/null @@ -1,201 +0,0 @@ - - - - - - - - - -
- - - - - - {/each} - - - - - {#each selectedMetrics as metric} - {#if selectedScopes[metric] != "node"} - - {/if} - {#each ["min", "avg", "max"] as stat} - - {/each} - {/each} - - - - {#each hosts as host (host)} - - - {#each selectedMetrics as metric (metric)} - - {/each} - - {/each} - -
- {#each selectedMetrics as metric} - - - - - {metric} - - - {#each scopesForMetric(metric, jobMetrics) as scope} - - {/each} - - -
NodeId sortBy(metric, stat)}> - {stat} - {#if selectedScopes[metric] == "node"} - - {/if} -
{host}
- - diff --git a/web/frontend/src/job/statstab/StatsTable.svelte b/web/frontend/src/job/statstab/StatsTable.svelte new file mode 100644 index 0000000..4adb4bd --- /dev/null +++ b/web/frontend/src/job/statstab/StatsTable.svelte @@ -0,0 +1,139 @@ + + + + + + + + + + {/each} + + + + + {#each selectedMetrics as metric} + {#if selectedScopes[metric] != "node"} + + {/if} + {#each ["min", "avg", "max"] as stat} + + {/each} + {/each} + + + + {#each hosts as host (host)} + + + {#each selectedMetrics as metric (metric)} + + {/each} + + {/each} + +
+ {#each selectedMetrics as metric} + + + + + {metric} + + + {#each (availableScopes[metric] || []) as scope} + + {/each} + + +
NodeId sortBy(metric, stat)}> + {stat} + {#if selectedScopes[metric] == "node"} + + {/if} +
{host}
\ No newline at end of file diff --git a/web/frontend/src/job/StatsTableEntry.svelte b/web/frontend/src/job/statstab/StatsTableEntry.svelte similarity index 67% rename from web/frontend/src/job/StatsTableEntry.svelte rename to web/frontend/src/job/statstab/StatsTableEntry.svelte index dc2f628..b39eacb 100644 --- a/web/frontend/src/job/StatsTableEntry.svelte +++ b/web/frontend/src/job/statstab/StatsTableEntry.svelte @@ -1,11 +1,11 @@ -{#if series == null || series.length == 0} +{#if stats == null || stats.length == 0} No data -{:else if series.length == 1 && scope == "node"} +{:else if stats.length == 1 && scope == "node"} - {series[0].data.min} + {stats[0].data.min} - {series[0].data.avg} + {stats[0].data.avg} - {series[0].data.max} + {stats[0].data.max} {:else} @@ -76,14 +76,14 @@ sortByField(field)}> Sort {/each} - {#each series as s, i} + {#each stats as s, i} {s.id ?? i} {s.data.min} From 0144ad43f57c6e3c903a7a9f7152cd8617920e59 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 17 Mar 2025 11:03:51 +0100 Subject: [PATCH 32/82] Implement NodeListData and ScopedStats for Prometheus Backend --- internal/metricdata/prometheus.go | 167 ++++++++++++++++++++++++++++-- 1 file changed, 161 insertions(+), 6 deletions(-) diff --git a/internal/metricdata/prometheus.go b/internal/metricdata/prometheus.go index fe829c0..d16501e 100644 --- a/internal/metricdata/prometheus.go +++ b/internal/metricdata/prometheus.go @@ -448,18 +448,51 @@ func (pdb *PrometheusDataRepository) LoadNodeData( return data, nil } +// Implemented by NHR@FAU; Used in Job-View StatsTable func (pdb *PrometheusDataRepository) LoadScopedStats( job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context) (schema.ScopedJobStats, error) { - // TODO : Implement to be used in Job-View StatsTable - log.Infof("LoadScopedStats unimplemented for PrometheusDataRepository, Args: job-id %v, metrics %v, scopes %v", job.JobID, metrics, scopes) + // Assumption: pdb.loadData() only returns series node-scope - use node scope for statsTable + scopedJobStats := make(schema.ScopedJobStats) + data, err := pdb.LoadData(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, ctx, 0 /*resolution here*/) + if err != nil { + log.Warn("Error while loading job for scopedJobStats") + return nil, err + } - return nil, errors.New("METRICDATA/PROMETHEUS > unimplemented for PrometheusDataRepository") + for metric, metricData := range data { + for _, scope := range scopes { + if scope != schema.MetricScopeNode { + logOnce.Do(func() { + log.Infof("Note: Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope) + }) + continue + } + + if _, ok := scopedJobStats[metric]; !ok { + scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats) + } + + if _, ok := scopedJobStats[metric][scope]; !ok { + scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0) + } + + for _, series := range metricData[scope].Series { + scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{ + Hostname: series.Hostname, + Data: &series.Statistics, + }) + } + } + } + + return scopedJobStats, nil } +// Implemented by NHR@FAU; Used in NodeList-View func (pdb *PrometheusDataRepository) LoadNodeListData( cluster, subCluster, nodeFilter string, metrics []string, @@ -470,10 +503,132 @@ func (pdb *PrometheusDataRepository) LoadNodeListData( ctx context.Context, ) (map[string]schema.JobData, int, bool, error) { + // Assumption: pdb.loadData() only returns series node-scope - use node scope for NodeList + + // 0) Init additional vars var totalNodes int = 0 var hasNextPage bool = false - // TODO : Implement to be used in NodeList-View - log.Infof("LoadNodeListData unimplemented for PrometheusDataRepository, Args: cluster %s, metrics %v, nodeFilter %v, scopes %v", cluster, metrics, nodeFilter, scopes) - return nil, totalNodes, hasNextPage, errors.New("METRICDATA/PROMETHEUS > unimplemented for PrometheusDataRepository") + // 1) Get list of all nodes + var nodes []string + if subCluster != "" { + scNodes := archive.NodeLists[cluster][subCluster] + nodes = scNodes.PrintList() + } else { + subClusterNodeLists := archive.NodeLists[cluster] + for _, nodeList := range subClusterNodeLists { + nodes = append(nodes, nodeList.PrintList()...) + } + } + + // 2) Filter nodes + if nodeFilter != "" { + filteredNodes := []string{} + for _, node := range nodes { + if strings.Contains(node, nodeFilter) { + filteredNodes = append(filteredNodes, node) + } + } + nodes = filteredNodes + } + + // 2.1) Count total nodes && Sort nodes -> Sorting invalidated after return ... + totalNodes = len(nodes) + sort.Strings(nodes) + + // 3) Apply paging + if len(nodes) > page.ItemsPerPage { + start := (page.Page - 1) * page.ItemsPerPage + end := start + page.ItemsPerPage + if end > len(nodes) { + end = len(nodes) + hasNextPage = false + } else { + hasNextPage = true + } + nodes = nodes[start:end] + } + + // 4) Fetch Data, based on pdb.LoadNodeData() + + t0 := time.Now() + // Map of hosts of jobData + data := make(map[string]schema.JobData) + + // query db for each metric + // TODO: scopes seems to be always empty + if len(scopes) == 0 || !contains(scopes, schema.MetricScopeNode) { + scopes = append(scopes, schema.MetricScopeNode) + } + + for _, scope := range scopes { + if scope != schema.MetricScopeNode { + logOnce.Do(func() { + log.Infof("Note: Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope) + }) + continue + } + + for _, metric := range metrics { + metricConfig := archive.GetMetricConfig(cluster, metric) + if metricConfig == nil { + log.Warnf("Error in LoadNodeListData: Metric %s for cluster %s not configured", metric, cluster) + return nil, totalNodes, hasNextPage, errors.New("Prometheus config error") + } + query, err := pdb.FormatQuery(metric, scope, nodes, cluster) + if err != nil { + log.Warn("Error while formatting prometheus query") + return nil, totalNodes, hasNextPage, err + } + + // ranged query over all nodes + r := promv1.Range{ + Start: from, + End: to, + Step: time.Duration(metricConfig.Timestep * 1e9), + } + result, warnings, err := pdb.queryClient.QueryRange(ctx, query, r) + if err != nil { + log.Errorf("Prometheus query error in LoadNodeData: %v\n", err) + return nil, totalNodes, hasNextPage, errors.New("Prometheus query error") + } + if len(warnings) > 0 { + log.Warnf("Warnings: %v\n", warnings) + } + + step := int64(metricConfig.Timestep) + steps := int64(to.Sub(from).Seconds()) / step + + // iter rows of host, metric, values + for _, row := range result.(promm.Matrix) { + hostname := strings.TrimSuffix(string(row.Metric["exported_instance"]), pdb.suffix) + + hostdata, ok := data[hostname] + if !ok { + hostdata = make(schema.JobData) + data[hostname] = hostdata + } + + metricdata, ok := hostdata[metric] + if !ok { + metricdata = make(map[schema.MetricScope]*schema.JobMetric) + data[hostname][metric] = metricdata + } + + // output per host, metric and scope + scopeData, ok := metricdata[scope] + if !ok { + scopeData = &schema.JobMetric{ + Unit: metricConfig.Unit, + Timestep: metricConfig.Timestep, + Series: []schema.Series{pdb.RowToSeries(from, step, steps, row)}, + } + data[hostname][metric][scope] = scopeData + } + } + } + } + t1 := time.Since(t0) + log.Debugf("LoadNodeListData of %v nodes took %s", len(data), t1) + return data, totalNodes, hasNextPage, nil } From 93040d46296eb16f4a9c02bef220e33f3abefdd0 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 17 Mar 2025 15:25:33 +0100 Subject: [PATCH 33/82] IMplement LoadNode Data, LoadNodeListData, LoadScopedStats for influxDB2 backend - Untested - Only Node Scope --- internal/metricdata/influxdb-v2.go | 250 +++++++++++++++++++++++++++-- 1 file changed, 240 insertions(+), 10 deletions(-) diff --git a/internal/metricdata/influxdb-v2.go b/internal/metricdata/influxdb-v2.go index 2a943b6..c53dad3 100644 --- a/internal/metricdata/influxdb-v2.go +++ b/internal/metricdata/influxdb-v2.go @@ -10,6 +10,8 @@ import ( "encoding/json" "errors" "fmt" + "math" + "sort" "strings" "time" @@ -64,6 +66,8 @@ func (idb *InfluxDBv2DataRepository) LoadData( ctx context.Context, resolution int) (schema.JobData, error) { + log.Infof("InfluxDB 2 Backend: Resolution Scaling not Implemented, will return default timestep. Requested Resolution %d", resolution) + measurementsConds := make([]string, 0, len(metrics)) for _, m := range metrics { measurementsConds = append(measurementsConds, fmt.Sprintf(`r["_measurement"] == "%s"`, m)) @@ -86,7 +90,7 @@ func (idb *InfluxDBv2DataRepository) LoadData( query := "" switch scope { case "node": - // Get Finest Granularity, Groupy By Measurement and Hostname (== Metric / Node), Calculate Mean for 60s windows + // Get Finest Granularity, Groupy By Measurement and Hostname (== Metric / Node), Calculate Mean for 60s windows <-- Resolution could be added here? // log.Info("Scope 'node' requested. ") query = fmt.Sprintf(` from(bucket: "%s") @@ -116,6 +120,12 @@ func (idb *InfluxDBv2DataRepository) LoadData( // idb.bucket, // idb.formatTime(job.StartTime), idb.formatTime(idb.epochToTime(job.StartTimeUnix + int64(job.Duration) + int64(1) )), // measurementsCond, hostsCond) + case "hwthread": + log.Info(" Scope 'hwthread' requested, but not yet supported: Will return 'node' scope only. ") + continue + case "accelerator": + log.Info(" Scope 'accelerator' requested, but not yet supported: Will return 'node' scope only. ") + continue default: log.Infof("Unknown scope '%s' requested: Will return 'node' scope.", scope) continue @@ -173,6 +183,11 @@ func (idb *InfluxDBv2DataRepository) LoadData( } case "socket": continue + case "accelerator": + continue + case "hwthread": + // See below @ core + continue case "core": continue // Include Series.Id in hostSeries @@ -301,18 +316,53 @@ func (idb *InfluxDBv2DataRepository) LoadStats( return stats, nil } +// Used in Job-View StatsTable +// UNTESTED func (idb *InfluxDBv2DataRepository) LoadScopedStats( job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context) (schema.ScopedJobStats, error) { - // TODO : Implement to be used in JobView Stats Table - log.Infof("LoadScopedStats unimplemented for InfluxDBv2DataRepository, Args: Job-ID %d, metrics %v, scopes %v", job.JobID, metrics, scopes) + // Assumption: idb.loadData() only returns series node-scope - use node scope for statsTable + scopedJobStats := make(schema.ScopedJobStats) + data, err := idb.LoadData(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, ctx, 0 /*resolution here*/) + if err != nil { + log.Warn("Error while loading job for scopedJobStats") + return nil, err + } - return nil, errors.New("METRICDATA/INFLUXV2 > unimplemented for InfluxDBv2DataRepository") + for metric, metricData := range data { + for _, scope := range scopes { + if scope != schema.MetricScopeNode { + logOnce.Do(func() { + log.Infof("Note: Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope) + }) + continue + } + + if _, ok := scopedJobStats[metric]; !ok { + scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats) + } + + if _, ok := scopedJobStats[metric][scope]; !ok { + scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0) + } + + for _, series := range metricData[scope].Series { + scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{ + Hostname: series.Hostname, + Data: &series.Statistics, + }) + } + } + } + + return scopedJobStats, nil } +// Used in Systems-View @ Node-Overview +// UNTESTED func (idb *InfluxDBv2DataRepository) LoadNodeData( cluster string, metrics, nodes []string, @@ -320,12 +370,123 @@ func (idb *InfluxDBv2DataRepository) LoadNodeData( from, to time.Time, ctx context.Context) (map[string]map[string][]*schema.JobMetric, error) { - // TODO : Implement to be used in Analysis- und System/Node-View - log.Infof("LoadNodeData unimplemented for InfluxDBv2DataRepository, Args: cluster %s, metrics %v, nodes %v, scopes %v", cluster, metrics, nodes, scopes) + // Note: scopes[] Array will be ignored, only return node scope - return nil, errors.New("METRICDATA/INFLUXV2 > unimplemented for InfluxDBv2DataRepository") + // CONVERT ARGS TO INFLUX + measurementsConds := make([]string, 0) + for _, m := range metrics { + measurementsConds = append(measurementsConds, fmt.Sprintf(`r["_measurement"] == "%s"`, m)) + } + measurementsCond := strings.Join(measurementsConds, " or ") + + hostsConds := make([]string, 0) + if nodes == nil { + var allNodes []string + subClusterNodeLists := archive.NodeLists[cluster] + for _, nodeList := range subClusterNodeLists { + allNodes = append(nodes, nodeList.PrintList()...) + } + for _, node := range allNodes { + nodes = append(nodes, node) + hostsConds = append(hostsConds, fmt.Sprintf(`r["hostname"] == "%s"`, node)) + } + } else { + for _, node := range nodes { + hostsConds = append(hostsConds, fmt.Sprintf(`r["hostname"] == "%s"`, node)) + } + } + hostsCond := strings.Join(hostsConds, " or ") + + // BUILD AND PERFORM QUERY + query := fmt.Sprintf(` + from(bucket: "%s") + |> range(start: %s, stop: %s) + |> filter(fn: (r) => (%s) and (%s) ) + |> drop(columns: ["_start", "_stop"]) + |> group(columns: ["hostname", "_measurement"]) + |> aggregateWindow(every: 60s, fn: mean) + |> drop(columns: ["_time"])`, + idb.bucket, + idb.formatTime(from), idb.formatTime(to), + measurementsCond, hostsCond) + + rows, err := idb.queryClient.Query(ctx, query) + if err != nil { + log.Error("Error while performing query") + return nil, err + } + + // HANDLE QUERY RETURN + // Collect Float Arrays for Node@Metric -> No Scope Handling! + influxData := make(map[string]map[string][]schema.Float) + for rows.Next() { + row := rows.Record() + host, field := row.ValueByKey("hostname").(string), row.Measurement() + + influxHostData, ok := influxData[host] + if !ok { + influxHostData = make(map[string][]schema.Float) + influxData[host] = influxHostData + } + + influxFieldData, ok := influxData[host][field] + if !ok { + influxFieldData = make([]schema.Float, 0) + influxData[host][field] = influxFieldData + } + + val, ok := row.Value().(float64) + if ok { + influxData[host][field] = append(influxData[host][field], schema.Float(val)) + } else { + influxData[host][field] = append(influxData[host][field], schema.Float(0)) + } + } + + // BUILD FUNCTION RETURN + data := make(map[string]map[string][]*schema.JobMetric) + for node, metricData := range influxData { + + nodeData, ok := data[node] + if !ok { + nodeData = make(map[string][]*schema.JobMetric) + data[node] = nodeData + } + + for metric, floatArray := range metricData { + avg, min, max := 0.0, 0.0, 0.0 + for _, val := range floatArray { + avg += float64(val) + min = math.Min(min, float64(val)) + max = math.Max(max, float64(val)) + } + + stats := schema.MetricStatistics{ + Avg: (math.Round((avg/float64(len(floatArray)))*100) / 100), + Min: (math.Round(min*100) / 100), + Max: (math.Round(max*100) / 100), + } + + mc := archive.GetMetricConfig(cluster, metric) + nodeData[metric] = append(nodeData[metric], &schema.JobMetric{ + Unit: mc.Unit, + Timestep: mc.Timestep, + Series: []schema.Series{ + { + Hostname: node, + Statistics: stats, + Data: floatArray, + }, + }, + }) + } + } + + return data, nil } +// Used in Systems-View @ Node-List +// UNTESTED func (idb *InfluxDBv2DataRepository) LoadNodeListData( cluster, subCluster, nodeFilter string, metrics []string, @@ -336,10 +497,79 @@ func (idb *InfluxDBv2DataRepository) LoadNodeListData( ctx context.Context, ) (map[string]schema.JobData, int, bool, error) { + // Assumption: idb.loadData() only returns series node-scope - use node scope for NodeList + + // 0) Init additional vars var totalNodes int = 0 var hasNextPage bool = false - // TODO : Implement to be used in NodeList-View - log.Infof("LoadNodeListData unimplemented for InfluxDBv2DataRepository, Args: cluster %s, metrics %v, nodeFilter %v, scopes %v", cluster, metrics, nodeFilter, scopes) - return nil, totalNodes, hasNextPage, errors.New("METRICDATA/INFLUXV2 > unimplemented for InfluxDBv2DataRepository") + // 1) Get list of all nodes + var nodes []string + if subCluster != "" { + scNodes := archive.NodeLists[cluster][subCluster] + nodes = scNodes.PrintList() + } else { + subClusterNodeLists := archive.NodeLists[cluster] + for _, nodeList := range subClusterNodeLists { + nodes = append(nodes, nodeList.PrintList()...) + } + } + + // 2) Filter nodes + if nodeFilter != "" { + filteredNodes := []string{} + for _, node := range nodes { + if strings.Contains(node, nodeFilter) { + filteredNodes = append(filteredNodes, node) + } + } + nodes = filteredNodes + } + + // 2.1) Count total nodes && Sort nodes -> Sorting invalidated after return ... + totalNodes = len(nodes) + sort.Strings(nodes) + + // 3) Apply paging + if len(nodes) > page.ItemsPerPage { + start := (page.Page - 1) * page.ItemsPerPage + end := start + page.ItemsPerPage + if end > len(nodes) { + end = len(nodes) + hasNextPage = false + } else { + hasNextPage = true + } + nodes = nodes[start:end] + } + + // 4) Fetch And Convert Data, use idb.LoadNodeData() for query + + rawNodeData, err := idb.LoadNodeData(cluster, metrics, nodes, scopes, from, to, ctx) + if err != nil { + log.Error(fmt.Sprintf("Error while loading influx nodeData for nodeListData %#v\n", err)) + return nil, totalNodes, hasNextPage, err + } + + data := make(map[string]schema.JobData) + for node, nodeData := range rawNodeData { + // Init Nested Map Data Structures If Not Found + hostData, ok := data[node] + if !ok { + hostData = make(schema.JobData) + data[node] = hostData + } + + for metric, nodeMetricData := range nodeData { + metricData, ok := hostData[metric] + if !ok { + metricData = make(map[schema.MetricScope]*schema.JobMetric) + data[node][metric] = metricData + } + + data[node][metric][schema.MetricScopeNode] = nodeMetricData[0] // Only Node Scope Returned from loadNodeData + } + } + + return data, totalNodes, hasNextPage, nil } From 9ed64e0388eb948bd8ad3b1dc07f14feb771a367 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 17 Mar 2025 17:39:17 +0100 Subject: [PATCH 34/82] Review logging, comment cleanup --- internal/graph/schema.resolvers.go | 12 ++--- internal/metricdata/cc-metric-store.go | 48 ++++++++----------- pkg/archive/archive.go | 14 +++--- .../src/job/statstab/StatsTable.svelte | 4 +- 4 files changed, 35 insertions(+), 43 deletions(-) diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index a470807..b5966c7 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -301,17 +301,17 @@ func (r *queryResolver) JobMetrics(ctx context.Context, id string, metrics []str return res, err } -// JobMetricStats is the resolver for the jobStats field. +// JobStats is the resolver for the jobStats field. func (r *queryResolver) JobStats(ctx context.Context, id string, metrics []string) ([]*model.JobStats, error) { job, err := r.Query().Job(ctx, id) if err != nil { - log.Warnf("Error while querying job %s for metrics", id) + log.Warnf("Error while querying job %s for metadata", id) return nil, err } data, err := metricDataDispatcher.LoadJobStats(job, metrics, ctx) if err != nil { - log.Warnf("Error while loading job stat data for job id %s", id) + log.Warnf("Error while loading jobStats data for job id %s", id) return nil, err } @@ -326,17 +326,17 @@ func (r *queryResolver) JobStats(ctx context.Context, id string, metrics []strin return res, err } -// JobStats is the resolver for the scopedJobStats field. +// ScopedJobStats is the resolver for the scopedJobStats field. func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope) ([]*model.JobStatsWithScope, error) { job, err := r.Query().Job(ctx, id) if err != nil { - log.Warnf("Error while querying job %s for metrics", id) + log.Warnf("Error while querying job %s for metadata", id) return nil, err } data, err := metricDataDispatcher.LoadScopedJobStats(job, metrics, scopes, ctx) if err != nil { - log.Warnf("Error while loading scoped job stat data for job id %s", id) + log.Warnf("Error while loading scopedJobStats data for job id %s", id) return nil, err } diff --git a/internal/metricdata/cc-metric-store.go b/internal/metricdata/cc-metric-store.go index 6635299..9516e2b 100644 --- a/internal/metricdata/cc-metric-store.go +++ b/internal/metricdata/cc-metric-store.go @@ -129,13 +129,13 @@ func (ccms *CCMetricStore) doRequest( ) (*ApiQueryResponse, error) { buf := &bytes.Buffer{} if err := json.NewEncoder(buf).Encode(body); err != nil { - log.Warn("Error while encoding request body") + log.Errorf("Error while encoding request body: %s", err.Error()) return nil, err } req, err := http.NewRequestWithContext(ctx, http.MethodGet, ccms.queryEndpoint, buf) if err != nil { - log.Warn("Error while building request body") + log.Errorf("Error while building request body: %s", err.Error()) return nil, err } if ccms.jwt != "" { @@ -151,7 +151,7 @@ func (ccms *CCMetricStore) doRequest( res, err := ccms.client.Do(req) if err != nil { - log.Error("Error while performing request") + log.Errorf("Error while performing request: %s", err.Error()) return nil, err } @@ -161,7 +161,7 @@ func (ccms *CCMetricStore) doRequest( var resBody ApiQueryResponse if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&resBody); err != nil { - log.Warn("Error while decoding result body") + log.Errorf("Error while decoding result body: %s", err.Error()) return nil, err } @@ -177,7 +177,7 @@ func (ccms *CCMetricStore) LoadData( ) (schema.JobData, error) { queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, resolution) if err != nil { - log.Warn("Error while building queries") + log.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error()) return nil, err } @@ -192,7 +192,7 @@ func (ccms *CCMetricStore) LoadData( resBody, err := ccms.doRequest(ctx, &req) if err != nil { - log.Error("Error while performing request") + log.Errorf("Error while performing request: %s", err.Error()) return nil, err } @@ -557,16 +557,9 @@ func (ccms *CCMetricStore) LoadStats( ctx context.Context, ) (map[string]map[string]schema.MetricStatistics, error) { - // metricConfigs := archive.GetCluster(job.Cluster).MetricConfig - // resolution := 9000 - - // for _, mc := range metricConfigs { - // resolution = min(resolution, mc.Timestep) - // } - queries, _, err := ccms.buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, 0) // #166 Add scope shere for analysis view accelerator normalization? if err != nil { - log.Warn("Error while building query") + log.Errorf("Error while building queries for jobId %d, Metrics %v: %s", job.JobID, metrics, err.Error()) return nil, err } @@ -581,7 +574,7 @@ func (ccms *CCMetricStore) LoadStats( resBody, err := ccms.doRequest(ctx, &req) if err != nil { - log.Error("Error while performing request") + log.Errorf("Error while performing request: %s", err.Error()) return nil, err } @@ -591,9 +584,8 @@ func (ccms *CCMetricStore) LoadStats( metric := ccms.toLocalName(query.Metric) data := res[0] if data.Error != nil { - log.Infof("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error) + log.Errorf("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error) continue - // return nil, fmt.Errorf("METRICDATA/CCMS > fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error) } metricdata, ok := stats[metric] @@ -603,9 +595,8 @@ func (ccms *CCMetricStore) LoadStats( } if data.Avg.IsNaN() || data.Min.IsNaN() || data.Max.IsNaN() { - log.Infof("fetching %s for node %s failed: one of avg/min/max is NaN", metric, query.Hostname) + log.Warnf("fetching %s for node %s failed: one of avg/min/max is NaN", metric, query.Hostname) continue - // return nil, fmt.Errorf("METRICDATA/CCMS > fetching %s for node %s failed: %s", metric, query.Hostname, "avg/min/max is NaN") } metricdata[query.Hostname] = schema.MetricStatistics{ @@ -618,7 +609,7 @@ func (ccms *CCMetricStore) LoadStats( return stats, nil } -// Scoped Stats: Basically Load Data without resolution and data query flag? +// Used for Job-View Statistics Table func (ccms *CCMetricStore) LoadScopedStats( job *schema.Job, metrics []string, @@ -627,7 +618,7 @@ func (ccms *CCMetricStore) LoadScopedStats( ) (schema.ScopedJobStats, error) { queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, 0) if err != nil { - log.Warn("Error while building queries") + log.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error()) return nil, err } @@ -642,7 +633,7 @@ func (ccms *CCMetricStore) LoadScopedStats( resBody, err := ccms.doRequest(ctx, &req) if err != nil { - log.Error("Error while performing request") + log.Errorf("Error while performing request: %s", err.Error()) return nil, err } @@ -709,7 +700,7 @@ func (ccms *CCMetricStore) LoadScopedStats( return scopedJobStats, nil } -// TODO: Support sub-node-scope metrics! For this, the partition of a node needs to be known! - Todo Outdated with NodeListData? +// Used for Systems-View Node-Overview func (ccms *CCMetricStore) LoadNodeData( cluster string, metrics, nodes []string, @@ -743,7 +734,7 @@ func (ccms *CCMetricStore) LoadNodeData( resBody, err := ccms.doRequest(ctx, &req) if err != nil { - log.Error(fmt.Sprintf("Error while performing request %#v\n", err)) + log.Errorf("Error while performing request: %s", err.Error()) return nil, err } @@ -801,6 +792,7 @@ func (ccms *CCMetricStore) LoadNodeData( return data, nil } +// Used for Systems-View Node-List func (ccms *CCMetricStore) LoadNodeListData( cluster, subCluster, nodeFilter string, metrics []string, @@ -859,7 +851,7 @@ func (ccms *CCMetricStore) LoadNodeListData( queries, assignedScope, err := ccms.buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, resolution) if err != nil { - log.Warn("Error while building queries") + log.Errorf("Error while building node queries for Cluster %s, SubCLuster %s, Metrics %v, Scopes %v: %s", cluster, subCluster, metrics, scopes, err.Error()) return nil, totalNodes, hasNextPage, err } @@ -874,7 +866,7 @@ func (ccms *CCMetricStore) LoadNodeListData( resBody, err := ccms.doRequest(ctx, &req) if err != nil { - log.Error(fmt.Sprintf("Error while performing request %#v\n", err)) + log.Errorf("Error while performing request: %s", err.Error()) return nil, totalNodes, hasNextPage, err } @@ -979,7 +971,7 @@ func (ccms *CCMetricStore) buildNodeQueries( if subCluster != "" { subClusterTopol, scterr = archive.GetSubCluster(cluster, subCluster) if scterr != nil { - // TODO: Log + log.Errorf("could not load cluster %s subCluster %s topology: %s", cluster, subCluster, scterr.Error()) return nil, nil, scterr } } @@ -989,7 +981,7 @@ func (ccms *CCMetricStore) buildNodeQueries( mc := archive.GetMetricConfig(cluster, metric) if mc == nil { // return nil, fmt.Errorf("METRICDATA/CCMS > metric '%s' is not specified for cluster '%s'", metric, cluster) - log.Infof("metric '%s' is not specified for cluster '%s'", metric, cluster) + log.Warnf("metric '%s' is not specified for cluster '%s'", metric, cluster) continue } diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go index 002fd5e..cd457eb 100644 --- a/pkg/archive/archive.go +++ b/pkg/archive/archive.go @@ -89,7 +89,7 @@ func Init(rawConfig json.RawMessage, disableArchive bool) error { var version uint64 version, err = ar.Init(rawConfig) if err != nil { - log.Error("Error while initializing archiveBackend") + log.Errorf("Error while initializing archiveBackend: %s", err.Error()) return } log.Infof("Load archive version %d", version) @@ -112,7 +112,7 @@ func LoadAveragesFromArchive( ) error { metaFile, err := ar.LoadJobMeta(job) if err != nil { - log.Warn("Error while loading job metadata from archiveBackend") + log.Errorf("Error while loading job metadata from archiveBackend: %s", err.Error()) return err } @@ -135,7 +135,7 @@ func LoadStatsFromArchive( data := make(map[string]schema.MetricStatistics, len(metrics)) metaFile, err := ar.LoadJobMeta(job) if err != nil { - log.Warn("Error while loading job metadata from archiveBackend") + log.Errorf("Error while loading job metadata from archiveBackend: %s", err.Error()) return data, err } @@ -165,7 +165,7 @@ func LoadScopedStatsFromArchive( data, err := ar.LoadJobStats(job) if err != nil { - log.Warn("Error while loading job metadata from archiveBackend") + log.Errorf("Error while loading job stats from archiveBackend: %s", err.Error()) return nil, err } @@ -175,7 +175,7 @@ func LoadScopedStatsFromArchive( func GetStatistics(job *schema.Job) (map[string]schema.JobStatistics, error) { metaFile, err := ar.LoadJobMeta(job) if err != nil { - log.Warn("Error while loading job metadata from archiveBackend") + log.Errorf("Error while loading job metadata from archiveBackend: %s", err.Error()) return nil, err } @@ -191,7 +191,7 @@ func UpdateMetadata(job *schema.Job, metadata map[string]string) error { jobMeta, err := ar.LoadJobMeta(job) if err != nil { - log.Warn("Error while loading job metadata from archiveBackend") + log.Errorf("Error while loading job metadata from archiveBackend: %s", err.Error()) return err } @@ -211,7 +211,7 @@ func UpdateTags(job *schema.Job, tags []*schema.Tag) error { jobMeta, err := ar.LoadJobMeta(job) if err != nil { - log.Warn("Error while loading job metadata from archiveBackend") + log.Errorf("Error while loading job metadata from archiveBackend: %s", err.Error()) return err } diff --git a/web/frontend/src/job/statstab/StatsTable.svelte b/web/frontend/src/job/statstab/StatsTable.svelte index 4adb4bd..2ed2f28 100644 --- a/web/frontend/src/job/statstab/StatsTable.svelte +++ b/web/frontend/src/job/statstab/StatsTable.svelte @@ -2,8 +2,8 @@ @component Job-View subcomponent; display table of metric data statistics with selectable scopes Properties: - - `job Object`: The job object - - `clusters Object`: The clusters object + - `data Object`: The data object + - `selectedMetrics [String]`: The selected metrics - `hosts [String]`: The list of hostnames of this job --> From c53f5eb144ec559cde19440748ca6886ad54f2a3 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 18 Mar 2025 18:01:37 +0100 Subject: [PATCH 35/82] fix: always return hasNextPage boolean to frontend - removes dependency on uiDefaults setting --- internal/graph/schema.resolvers.go | 44 ++++++++++++++---------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index ce1384b..7d3de90 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -354,30 +354,28 @@ func (r *queryResolver) Jobs(ctx context.Context, filter []*model.JobFilter, pag return nil, err } - if !config.Keys.UiDefaults["job_list_usePaging"].(bool) { - hasNextPage := false - // page.Page += 1 : Simple, but expensive - // Example Page 4 @ 10 IpP : Does item 41 exist? - // Minimal Page 41 @ 1 IpP : If len(result) is 1, Page 5 @ 10 IpP exists. - nextPage := &model.PageRequest{ - ItemsPerPage: 1, - Page: ((page.Page * page.ItemsPerPage) + 1), - } - - nextJobs, err := r.Repo.QueryJobs(ctx, filter, nextPage, order) - if err != nil { - log.Warn("Error while querying next jobs") - return nil, err - } - - if len(nextJobs) == 1 { - hasNextPage = true - } - - return &model.JobResultList{Items: jobs, Count: &count, HasNextPage: &hasNextPage}, nil - } else { - return &model.JobResultList{Items: jobs, Count: &count}, nil + // Note: Even if App-Default 'config.Keys.UiDefaults["job_list_usePaging"]' is set, always return hasNextPage boolean. + // Users can decide in frontend to use continuous scroll, even if app-default is paging! + /* + Example Page 4 @ 10 IpP : Does item 41 exist? + Minimal Page 41 @ 1 IpP : If len(result) is 1, Page 5 @ 10 IpP exists. + */ + nextPage := &model.PageRequest{ + ItemsPerPage: 1, + Page: ((page.Page * page.ItemsPerPage) + 1), } + nextJobs, err := r.Repo.QueryJobs(ctx, filter, nextPage, order) + if err != nil { + log.Warn("Error while querying next jobs") + return nil, err + } + + hasNextPage := false + if len(nextJobs) == 1 { + hasNextPage = true + } + + return &model.JobResultList{Items: jobs, Count: &count, HasNextPage: &hasNextPage}, nil } // JobsStatistics is the resolver for the jobsStatistics field. From e9a214c5b2272356ce89cbdac79056016134b735 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 19 Mar 2025 14:57:27 +0100 Subject: [PATCH 36/82] fix: add nullSafe condition to monitoringStatus display on metric queryError --- web/frontend/src/Job.root.svelte | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/web/frontend/src/Job.root.svelte b/web/frontend/src/Job.root.svelte index 6980230..22a69d8 100644 --- a/web/frontend/src/Job.root.svelte +++ b/web/frontend/src/Job.root.svelte @@ -231,7 +231,7 @@ {#if $initq.error} {$initq.error.message} - {:else if $initq.data} + {:else if $initq?.data} {#if $initq.data?.job?.metaData?.message} @@ -305,7 +305,7 @@ - {#if $initq.data} + {#if $initq?.data} From a6784b5549722466299541a8377c25800a83f318 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 8 Apr 2025 16:00:07 +0200 Subject: [PATCH 42/82] fix: reintroduce statstable id natural sort order - see Use natural sort order for IDs in statistics tables #369 --- web/frontend/src/job/statstab/StatsTableEntry.svelte | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/web/frontend/src/job/statstab/StatsTableEntry.svelte b/web/frontend/src/job/statstab/StatsTableEntry.svelte index b39eacb..c3161a5 100644 --- a/web/frontend/src/job/statstab/StatsTableEntry.svelte +++ b/web/frontend/src/job/statstab/StatsTableEntry.svelte @@ -41,7 +41,9 @@ if (a == null || b == null) return -1; if (field === "id") { - return s.dir != "up" ? a[field].localeCompare(b[field]) : b[field].localeCompare(a[field]) + return s.dir != "up" ? + a[field].localeCompare(b[field], undefined, {numeric: true, sensitivity: 'base'}) : + b[field].localeCompare(a[field], undefined, {numeric: true, sensitivity: 'base'}) } else { return s.dir != "up" ? a.data[field] - b.data[field] From a8d785beb348634ada38282689e05039cc081228 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 9 Apr 2025 07:27:59 +0200 Subject: [PATCH 43/82] Remove redundant check in auth package --- internal/api/rest.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/api/rest.go b/internal/api/rest.go index fd2f86d..8713976 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -105,6 +105,7 @@ func (api *RestApi) MountConfigApiRoutes(r *mux.Router) { r.StrictSlash(true) if api.Authentication != nil { + log.Debug("Mounting /configuration/ route") r.HandleFunc("/roles/", api.getRoles).Methods(http.MethodGet) r.HandleFunc("/users/", api.createUser).Methods(http.MethodPost, http.MethodPut) r.HandleFunc("/users/", api.getUsers).Methods(http.MethodGet) @@ -229,7 +230,7 @@ func securedCheck(r *http.Request) error { if user.AuthType == schema.AuthToken { // If nothing declared in config: deny all request to this endpoint - if config.Keys.ApiAllowedIPs == nil || len(config.Keys.ApiAllowedIPs) == 0 { + if len(config.Keys.ApiAllowedIPs) == 0 { return fmt.Errorf("missing configuration key ApiAllowedIPs") } From 28cdc1d9e5c6a455a0ca15e624d844b665af3270 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 9 Apr 2025 09:13:21 +0200 Subject: [PATCH 44/82] fix: Update endpoints in Swagger UI --- api/swagger.json | 27 +++++++++++++-------------- api/swagger.yaml | 27 +++++++++++++-------------- internal/api/docs.go | 28 ++++++++++++++-------------- internal/api/rest.go | 34 ++++++++++++++++------------------ 4 files changed, 56 insertions(+), 60 deletions(-) diff --git a/api/swagger.json b/api/swagger.json index 51b22c8..683b520 100644 --- a/api/swagger.json +++ b/api/swagger.json @@ -15,9 +15,8 @@ "version": "1.0.0" }, "host": "localhost:8080", - "basePath": "/api", "paths": { - "/clusters/": { + "/api/clusters/": { "get": { "security": [ { @@ -74,7 +73,7 @@ } } }, - "/jobs/": { + "/api/jobs/": { "get": { "security": [ { @@ -169,7 +168,7 @@ } } }, - "/jobs/delete_job/": { + "/api/jobs/delete_job/": { "delete": { "security": [ { @@ -244,7 +243,7 @@ } } }, - "/jobs/delete_job/{id}": { + "/api/jobs/delete_job/{id}": { "delete": { "security": [ { @@ -314,7 +313,7 @@ } } }, - "/jobs/delete_job_before/{ts}": { + "/api/jobs/delete_job_before/{ts}": { "delete": { "security": [ { @@ -384,7 +383,7 @@ } } }, - "/jobs/edit_meta/{id}": { + "/api/jobs/edit_meta/{id}": { "post": { "security": [ { @@ -454,7 +453,7 @@ } } }, - "/jobs/start_job/": { + "/api/jobs/start_job/": { "post": { "security": [ { @@ -523,7 +522,7 @@ } } }, - "/jobs/stop_job/": { + "/api/jobs/stop_job/": { "post": { "security": [ { @@ -595,7 +594,7 @@ } } }, - "/jobs/tag_job/{id}": { + "/api/jobs/tag_job/{id}": { "post": { "security": [ { @@ -668,7 +667,7 @@ } } }, - "/jobs/{id}": { + "/api/jobs/{id}": { "get": { "security": [ { @@ -827,7 +826,7 @@ } } }, - "/notice/": { + "/config/notice/": { "post": { "security": [ { @@ -893,7 +892,7 @@ } } }, - "/user/{id}": { + "/config/user/{id}": { "post": { "security": [ { @@ -998,7 +997,7 @@ } } }, - "/users/": { + "/config/users/": { "get": { "security": [ { diff --git a/api/swagger.yaml b/api/swagger.yaml index f5f0081..35ec6c4 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -1,4 +1,3 @@ -basePath: /api definitions: api.ApiReturnedUser: properties: @@ -671,7 +670,7 @@ info: title: ClusterCockpit REST API version: 1.0.0 paths: - /clusters/: + /api/clusters/: get: description: Get a list of all cluster configs. Specific cluster can be requested using query parameter. @@ -708,7 +707,7 @@ paths: summary: Lists all cluster configs tags: - Cluster query - /jobs/: + /api/jobs/: get: description: |- Get a list of all jobs. Filters can be applied using query parameters. @@ -773,7 +772,7 @@ paths: summary: Lists all jobs tags: - Job query - /jobs/{id}: + /api/jobs/{id}: get: description: |- Job to get is specified by database ID @@ -882,7 +881,7 @@ paths: summary: Get job meta and configurable metric data tags: - Job query - /jobs/delete_job/: + /api/jobs/delete_job/: delete: consumes: - application/json @@ -932,7 +931,7 @@ paths: summary: Remove a job from the sql database tags: - Job remove - /jobs/delete_job/{id}: + /api/jobs/delete_job/{id}: delete: description: Job to remove is specified by database ID. This will not remove the job from the job archive. @@ -979,7 +978,7 @@ paths: summary: Remove a job from the sql database tags: - Job remove - /jobs/delete_job_before/{ts}: + /api/jobs/delete_job_before/{ts}: delete: description: Remove all jobs with start time before timestamp. The jobs will not be removed from the job archive. @@ -1026,7 +1025,7 @@ paths: summary: Remove a job from the sql database tags: - Job remove - /jobs/edit_meta/{id}: + /api/jobs/edit_meta/{id}: post: consumes: - application/json @@ -1073,7 +1072,7 @@ paths: summary: Edit meta-data json tags: - Job add and modify - /jobs/start_job/: + /api/jobs/start_job/: post: consumes: - application/json @@ -1120,7 +1119,7 @@ paths: summary: Adds a new job as "running" tags: - Job add and modify - /jobs/stop_job/: + /api/jobs/stop_job/: post: description: |- Job to stop is specified by request body. All fields are required in this case. @@ -1168,7 +1167,7 @@ paths: summary: Marks job as completed and triggers archiving tags: - Job add and modify - /jobs/tag_job/{id}: + /api/jobs/tag_job/{id}: post: consumes: - application/json @@ -1218,7 +1217,7 @@ paths: summary: Adds one or more tags to a job tags: - Job add and modify - /notice/: + /config/notice/: post: consumes: - multipart/form-data @@ -1263,7 +1262,7 @@ paths: summary: Updates or empties the notice box content tags: - User - /user/{id}: + /config/user/{id}: post: consumes: - multipart/form-data @@ -1337,7 +1336,7 @@ paths: summary: Updates an existing user tags: - User - /users/: + /config/users/: delete: consumes: - multipart/form-data diff --git a/internal/api/docs.go b/internal/api/docs.go index 642003f..2408f85 100644 --- a/internal/api/docs.go +++ b/internal/api/docs.go @@ -23,7 +23,7 @@ const docTemplate = `{ "host": "{{.Host}}", "basePath": "{{.BasePath}}", "paths": { - "/clusters/": { + "/api/clusters/": { "get": { "security": [ { @@ -80,7 +80,7 @@ const docTemplate = `{ } } }, - "/jobs/": { + "/api/jobs/": { "get": { "security": [ { @@ -175,7 +175,7 @@ const docTemplate = `{ } } }, - "/jobs/delete_job/": { + "/api/jobs/delete_job/": { "delete": { "security": [ { @@ -250,7 +250,7 @@ const docTemplate = `{ } } }, - "/jobs/delete_job/{id}": { + "/api/jobs/delete_job/{id}": { "delete": { "security": [ { @@ -320,7 +320,7 @@ const docTemplate = `{ } } }, - "/jobs/delete_job_before/{ts}": { + "/api/jobs/delete_job_before/{ts}": { "delete": { "security": [ { @@ -390,7 +390,7 @@ const docTemplate = `{ } } }, - "/jobs/edit_meta/{id}": { + "/api/jobs/edit_meta/{id}": { "post": { "security": [ { @@ -460,7 +460,7 @@ const docTemplate = `{ } } }, - "/jobs/start_job/": { + "/api/jobs/start_job/": { "post": { "security": [ { @@ -529,7 +529,7 @@ const docTemplate = `{ } } }, - "/jobs/stop_job/": { + "/api/jobs/stop_job/": { "post": { "security": [ { @@ -601,7 +601,7 @@ const docTemplate = `{ } } }, - "/jobs/tag_job/{id}": { + "/api/jobs/tag_job/{id}": { "post": { "security": [ { @@ -674,7 +674,7 @@ const docTemplate = `{ } } }, - "/jobs/{id}": { + "/api/jobs/{id}": { "get": { "security": [ { @@ -833,7 +833,7 @@ const docTemplate = `{ } } }, - "/notice/": { + "/config/notice/": { "post": { "security": [ { @@ -899,7 +899,7 @@ const docTemplate = `{ } } }, - "/user/{id}": { + "/config/user/{id}": { "post": { "security": [ { @@ -1004,7 +1004,7 @@ const docTemplate = `{ } } }, - "/users/": { + "/config/users/": { "get": { "security": [ { @@ -2191,7 +2191,7 @@ const docTemplate = `{ var SwaggerInfo = &swag.Spec{ Version: "1.0.0", Host: "localhost:8080", - BasePath: "/api", + BasePath: "", Schemes: []string{}, Title: "ClusterCockpit REST API", Description: "API for batch job control.", diff --git a/internal/api/rest.go b/internal/api/rest.go index db9a860..85b0d13 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -46,7 +46,6 @@ import ( // @license.url https://opensource.org/licenses/MIT // @host localhost:8080 -// @basePath /api // @securityDefinitions.apikey ApiKeyAuth // @in header @@ -105,7 +104,6 @@ func (api *RestApi) MountConfigApiRoutes(r *mux.Router) { r.StrictSlash(true) if api.Authentication != nil { - log.Debug("Mounting /configuration/ route") r.HandleFunc("/roles/", api.getRoles).Methods(http.MethodGet) r.HandleFunc("/users/", api.createUser).Methods(http.MethodPost, http.MethodPut) r.HandleFunc("/users/", api.getUsers).Methods(http.MethodGet) @@ -272,7 +270,7 @@ func securedCheck(r *http.Request) error { // @failure 403 {object} api.ErrorResponse "Forbidden" // @failure 500 {object} api.ErrorResponse "Internal Server Error" // @security ApiKeyAuth -// @router /clusters/ [get] +// @router /api/clusters/ [get] func (api *RestApi) getClusters(rw http.ResponseWriter, r *http.Request) { if user := repository.GetUserFromContext(r.Context()); user != nil && !user.HasRole(schema.RoleApi) { @@ -327,7 +325,7 @@ func (api *RestApi) getClusters(rw http.ResponseWriter, r *http.Request) { // @failure 403 {object} api.ErrorResponse "Forbidden" // @failure 500 {object} api.ErrorResponse "Internal Server Error" // @security ApiKeyAuth -// @router /jobs/ [get] +// @router /api/jobs/ [get] func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { withMetadata := false filter := &model.JobFilter{} @@ -461,7 +459,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { // @failure 422 {object} api.ErrorResponse "Unprocessable Entity: finding job failed: sql: no rows in result set" // @failure 500 {object} api.ErrorResponse "Internal Server Error" // @security ApiKeyAuth -// @router /jobs/{id} [get] +// @router /api/jobs/{id} [get] func (api *RestApi) getCompleteJobById(rw http.ResponseWriter, r *http.Request) { // Fetch job from db id, ok := mux.Vars(r)["id"] @@ -554,7 +552,7 @@ func (api *RestApi) getCompleteJobById(rw http.ResponseWriter, r *http.Request) // @failure 422 {object} api.ErrorResponse "Unprocessable Entity: finding job failed: sql: no rows in result set" // @failure 500 {object} api.ErrorResponse "Internal Server Error" // @security ApiKeyAuth -// @router /jobs/{id} [post] +// @router /api/jobs/{id} [post] func (api *RestApi) getJobById(rw http.ResponseWriter, r *http.Request) { // Fetch job from db id, ok := mux.Vars(r)["id"] @@ -658,7 +656,7 @@ func (api *RestApi) getJobById(rw http.ResponseWriter, r *http.Request) { // @failure 404 {object} api.ErrorResponse "Job does not exist" // @failure 500 {object} api.ErrorResponse "Internal Server Error" // @security ApiKeyAuth -// @router /jobs/edit_meta/{id} [post] +// @router /api/jobs/edit_meta/{id} [post] func (api *RestApi) editMeta(rw http.ResponseWriter, r *http.Request) { id, err := strconv.ParseInt(mux.Vars(r)["id"], 10, 64) if err != nil { @@ -704,7 +702,7 @@ func (api *RestApi) editMeta(rw http.ResponseWriter, r *http.Request) { // @failure 404 {object} api.ErrorResponse "Job or tag does not exist" // @failure 500 {object} api.ErrorResponse "Internal Server Error" // @security ApiKeyAuth -// @router /jobs/tag_job/{id} [post] +// @router /api/jobs/tag_job/{id} [post] func (api *RestApi) tagJob(rw http.ResponseWriter, r *http.Request) { id, err := strconv.ParseInt(mux.Vars(r)["id"], 10, 64) if err != nil { @@ -765,7 +763,7 @@ func (api *RestApi) tagJob(rw http.ResponseWriter, r *http.Request) { // @failure 422 {object} api.ErrorResponse "Unprocessable Entity: The combination of jobId, clusterId and startTime does already exist" // @failure 500 {object} api.ErrorResponse "Internal Server Error" // @security ApiKeyAuth -// @router /jobs/start_job/ [post] +// @router /api/jobs/start_job/ [post] func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) { req := schema.JobMeta{BaseJob: schema.JobDefaults} if err := decode(r.Body, &req); err != nil { @@ -838,7 +836,7 @@ func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) { // @failure 422 {object} api.ErrorResponse "Unprocessable Entity: job has already been stopped" // @failure 500 {object} api.ErrorResponse "Internal Server Error" // @security ApiKeyAuth -// @router /jobs/stop_job/ [post] +// @router /api/jobs/stop_job/ [post] func (api *RestApi) stopJobByRequest(rw http.ResponseWriter, r *http.Request) { // Parse request body req := StopJobApiRequest{} @@ -879,7 +877,7 @@ func (api *RestApi) stopJobByRequest(rw http.ResponseWriter, r *http.Request) { // @failure 422 {object} api.ErrorResponse "Unprocessable Entity: finding job failed: sql: no rows in result set" // @failure 500 {object} api.ErrorResponse "Internal Server Error" // @security ApiKeyAuth -// @router /jobs/delete_job/{id} [delete] +// @router /api/jobs/delete_job/{id} [delete] func (api *RestApi) deleteJobById(rw http.ResponseWriter, r *http.Request) { // Fetch job (that will be stopped) from db id, ok := mux.Vars(r)["id"] @@ -922,7 +920,7 @@ func (api *RestApi) deleteJobById(rw http.ResponseWriter, r *http.Request) { // @failure 422 {object} api.ErrorResponse "Unprocessable Entity: finding job failed: sql: no rows in result set" // @failure 500 {object} api.ErrorResponse "Internal Server Error" // @security ApiKeyAuth -// @router /jobs/delete_job/ [delete] +// @router /api/jobs/delete_job/ [delete] func (api *RestApi) deleteJobByRequest(rw http.ResponseWriter, r *http.Request) { // Parse request body req := DeleteJobApiRequest{} @@ -972,7 +970,7 @@ func (api *RestApi) deleteJobByRequest(rw http.ResponseWriter, r *http.Request) // @failure 422 {object} api.ErrorResponse "Unprocessable Entity: finding job failed: sql: no rows in result set" // @failure 500 {object} api.ErrorResponse "Internal Server Error" // @security ApiKeyAuth -// @router /jobs/delete_job_before/{ts} [delete] +// @router /api/jobs/delete_job_before/{ts} [delete] func (api *RestApi) deleteJobBefore(rw http.ResponseWriter, r *http.Request) { var cnt int // Fetch job (that will be stopped) from db @@ -1110,7 +1108,7 @@ func (api *RestApi) getJobMetrics(rw http.ResponseWriter, r *http.Request) { // @failure 422 {string} string "Unprocessable Entity: creating user failed" // @failure 500 {string} string "Internal Server Error" // @security ApiKeyAuth -// @router /users/ [post] +// @router /config/users/ [post] func (api *RestApi) createUser(rw http.ResponseWriter, r *http.Request) { err := securedCheck(r) if err != nil { @@ -1174,7 +1172,7 @@ func (api *RestApi) createUser(rw http.ResponseWriter, r *http.Request) { // @failure 422 {string} string "Unprocessable Entity: deleting user failed" // @failure 500 {string} string "Internal Server Error" // @security ApiKeyAuth -// @router /users/ [delete] +// @router /config/users/ [delete] func (api *RestApi) deleteUser(rw http.ResponseWriter, r *http.Request) { err := securedCheck(r) if err != nil { @@ -1210,7 +1208,7 @@ func (api *RestApi) deleteUser(rw http.ResponseWriter, r *http.Request) { // @failure 403 {string} string "Forbidden" // @failure 500 {string} string "Internal Server Error" // @security ApiKeyAuth -// @router /users/ [get] +// @router /config/users/ [get] func (api *RestApi) getUsers(rw http.ResponseWriter, r *http.Request) { err := securedCheck(r) if err != nil { @@ -1252,7 +1250,7 @@ func (api *RestApi) getUsers(rw http.ResponseWriter, r *http.Request) { // @failure 422 {string} string "Unprocessable Entity: The user could not be updated" // @failure 500 {string} string "Internal Server Error" // @security ApiKeyAuth -// @router /user/{id} [post] +// @router /config/user/{id} [post] func (api *RestApi) updateUser(rw http.ResponseWriter, r *http.Request) { err := securedCheck(r) if err != nil { @@ -1317,7 +1315,7 @@ func (api *RestApi) updateUser(rw http.ResponseWriter, r *http.Request) { // @failure 422 {string} string "Unprocessable Entity: The user could not be updated" // @failure 500 {string} string "Internal Server Error" // @security ApiKeyAuth -// @router /notice/ [post] +// @router /config/notice/ [post] func (api *RestApi) editNotice(rw http.ResponseWriter, r *http.Request) { err := securedCheck(r) if err != nil { From 317f80a9846ddda13a5cca68d29bba5ca7619d8f Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 9 Apr 2025 09:40:52 +0200 Subject: [PATCH 45/82] fix: Replace deprecated gqlgen NewDefaultServer call --- cmd/cc-backend/server.go | 21 +++++++++++++++++---- internal/api/rest.go | 2 +- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/cmd/cc-backend/server.go b/cmd/cc-backend/server.go index 1408162..3c19730 100644 --- a/cmd/cc-backend/server.go +++ b/cmd/cc-backend/server.go @@ -18,6 +18,7 @@ import ( "time" "github.com/99designs/gqlgen/graphql/handler" + "github.com/99designs/gqlgen/graphql/handler/transport" "github.com/99designs/gqlgen/graphql/playground" "github.com/ClusterCockpit/cc-backend/internal/api" "github.com/ClusterCockpit/cc-backend/internal/archiver" @@ -31,6 +32,7 @@ import ( "github.com/ClusterCockpit/cc-backend/web" "github.com/gorilla/handlers" "github.com/gorilla/mux" + "github.com/gorilla/websocket" httpSwagger "github.com/swaggo/http-swagger" ) @@ -53,13 +55,24 @@ func serverInit() { // Setup the http.Handler/Router used by the server graph.Init() resolver := graph.GetResolverInstance() - graphQLEndpoint := handler.NewDefaultServer( + graphQLServer := handler.New( generated.NewExecutableSchema(generated.Config{Resolvers: resolver})) + graphQLServer.AddTransport(transport.SSE{}) + graphQLServer.AddTransport(transport.POST{}) + graphQLServer.AddTransport(transport.Websocket{ + KeepAlivePingInterval: 10 * time.Second, + Upgrader: websocket.Upgrader{ + CheckOrigin: func(r *http.Request) bool { + return true + }, + }, + }) + if os.Getenv("DEBUG") != "1" { // Having this handler means that a error message is returned via GraphQL instead of the connection simply beeing closed. // The problem with this is that then, no more stacktrace is printed to stderr. - graphQLEndpoint.SetRecoverFunc(func(ctx context.Context, err interface{}) error { + graphQLServer.SetRecoverFunc(func(ctx context.Context, err any) error { switch e := err.(type) { case string: return fmt.Errorf("MAIN > Panic: %s", e) @@ -78,7 +91,7 @@ func serverInit() { router = mux.NewRouter() buildInfo := web.Build{Version: version, Hash: commit, Buildtime: date} - info := map[string]interface{}{} + info := map[string]any{} info["hasOpenIDConnect"] = false if config.Keys.OpenIDConfig != nil { @@ -208,7 +221,7 @@ func serverInit() { router.PathPrefix("/swagger/").Handler(httpSwagger.Handler( httpSwagger.URL("http://" + config.Keys.Addr + "/swagger/doc.json"))).Methods(http.MethodGet) } - secured.Handle("/query", graphQLEndpoint) + secured.Handle("/query", graphQLServer) // Send a searchId and then reply with a redirect to a user, or directly send query to job table for jobid and project. secured.HandleFunc("/search", func(rw http.ResponseWriter, r *http.Request) { diff --git a/internal/api/rest.go b/internal/api/rest.go index 85b0d13..1ebe78e 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -214,7 +214,7 @@ func handleError(err error, statusCode int, rw http.ResponseWriter) { }) } -func decode(r io.Reader, val interface{}) error { +func decode(r io.Reader, val any) error { dec := json.NewDecoder(r) dec.DisallowUnknownFields() return dec.Decode(val) From fb6a4c3b874114883a49aacdd408d0c17dfc0a20 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 9 Apr 2025 16:00:27 +0200 Subject: [PATCH 46/82] review and move api endpoints secured check --- internal/api/rest.go | 95 +++++-------------------------------------- internal/auth/auth.go | 80 ++++++++++++++++++++++++++++++++++-- pkg/schema/config.go | 5 ++- 3 files changed, 91 insertions(+), 89 deletions(-) diff --git a/internal/api/rest.go b/internal/api/rest.go index 712a0b3..352dd94 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -69,11 +69,10 @@ func New() *RestApi { func (api *RestApi) MountApiRoutes(r *mux.Router) { r.StrictSlash(true) - + // REST API Uses TokenAuth r.HandleFunc("/jobs/start_job/", api.startJob).Methods(http.MethodPost, http.MethodPut) r.HandleFunc("/jobs/stop_job/", api.stopJobByRequest).Methods(http.MethodPost, http.MethodPut) // r.HandleFunc("/jobs/import/", api.importJob).Methods(http.MethodPost, http.MethodPut) - r.HandleFunc("/jobs/", api.getJobs).Methods(http.MethodGet) r.HandleFunc("/jobs/{id}", api.getJobById).Methods(http.MethodPost) r.HandleFunc("/jobs/{id}", api.getCompleteJobById).Methods(http.MethodGet) @@ -83,7 +82,6 @@ func (api *RestApi) MountApiRoutes(r *mux.Router) { r.HandleFunc("/jobs/delete_job/", api.deleteJobByRequest).Methods(http.MethodDelete) r.HandleFunc("/jobs/delete_job/{id}", api.deleteJobById).Methods(http.MethodDelete) r.HandleFunc("/jobs/delete_job_before/{ts}", api.deleteJobBefore).Methods(http.MethodDelete) - r.HandleFunc("/clusters/", api.getClusters).Methods(http.MethodGet) if api.MachineStateDir != "" { @@ -94,7 +92,7 @@ func (api *RestApi) MountApiRoutes(r *mux.Router) { func (api *RestApi) MountUserApiRoutes(r *mux.Router) { r.StrictSlash(true) - + // REST API Uses TokenAuth r.HandleFunc("/jobs/", api.getJobs).Methods(http.MethodGet) r.HandleFunc("/jobs/{id}", api.getJobById).Methods(http.MethodPost) r.HandleFunc("/jobs/{id}", api.getCompleteJobById).Methods(http.MethodGet) @@ -103,7 +101,7 @@ func (api *RestApi) MountUserApiRoutes(r *mux.Router) { func (api *RestApi) MountConfigApiRoutes(r *mux.Router) { r.StrictSlash(true) - + // Settings Frontend Uses SessionAuth if api.Authentication != nil { r.HandleFunc("/roles/", api.getRoles).Methods(http.MethodGet) r.HandleFunc("/users/", api.createUser).Methods(http.MethodPost, http.MethodPut) @@ -116,7 +114,7 @@ func (api *RestApi) MountConfigApiRoutes(r *mux.Router) { func (api *RestApi) MountFrontendApiRoutes(r *mux.Router) { r.StrictSlash(true) - + // Settings Frontrend Uses SessionAuth if api.Authentication != nil { r.HandleFunc("/jwt/", api.getJWT).Methods(http.MethodGet) r.HandleFunc("/configuration/", api.updateConfiguration).Methods(http.MethodPost) @@ -221,44 +219,6 @@ func decode(r io.Reader, val interface{}) error { return dec.Decode(val) } -func securedCheck(r *http.Request) error { - user := repository.GetUserFromContext(r.Context()) - if user == nil { - return fmt.Errorf("no user in context") - } - - if user.AuthType == schema.AuthToken { - // If nothing declared in config: deny all request to this endpoint - if config.Keys.ApiAllowedIPs == nil || len(config.Keys.ApiAllowedIPs) == 0 { - return fmt.Errorf("missing configuration key ApiAllowedIPs") - } - - if config.Keys.ApiAllowedIPs[0] == "*" { - return nil - } - - // extract IP address - IPAddress := r.Header.Get("X-Real-Ip") - if IPAddress == "" { - IPAddress = r.Header.Get("X-Forwarded-For") - } - if IPAddress == "" { - IPAddress = r.RemoteAddr - } - - if strings.Contains(IPAddress, ":") { - IPAddress = strings.Split(IPAddress, ":")[0] - } - - // check if IP is allowed - if !util.Contains(config.Keys.ApiAllowedIPs, IPAddress) { - return fmt.Errorf("unknown ip: %v", IPAddress) - } - } - - return nil -} - // getClusters godoc // @summary Lists all cluster configs // @tags Cluster query @@ -1093,7 +1053,6 @@ func (api *RestApi) getJobMetrics(rw http.ResponseWriter, r *http.Request) { // @summary Adds a new user // @tags User // @description User specified in form data will be saved to database. -// @description Only accessible from IPs registered with apiAllowedIPs configuration option. // @accept mpfd // @produce plain // @param username formData string true "Unique user ID" @@ -1111,11 +1070,7 @@ func (api *RestApi) getJobMetrics(rw http.ResponseWriter, r *http.Request) { // @security ApiKeyAuth // @router /users/ [post] func (api *RestApi) createUser(rw http.ResponseWriter, r *http.Request) { - err := securedCheck(r) - if err != nil { - http.Error(rw, err.Error(), http.StatusForbidden) - return - } + // SecuredCheck() only worked with TokenAuth: Removed rw.Header().Set("Content-Type", "text/plain") me := repository.GetUserFromContext(r.Context()) @@ -1162,7 +1117,6 @@ func (api *RestApi) createUser(rw http.ResponseWriter, r *http.Request) { // @summary Deletes a user // @tags User // @description User defined by username in form data will be deleted from database. -// @description Only accessible from IPs registered with apiAllowedIPs configuration option. // @accept mpfd // @produce plain // @param username formData string true "User ID to delete" @@ -1175,11 +1129,7 @@ func (api *RestApi) createUser(rw http.ResponseWriter, r *http.Request) { // @security ApiKeyAuth // @router /users/ [delete] func (api *RestApi) deleteUser(rw http.ResponseWriter, r *http.Request) { - err := securedCheck(r) - if err != nil { - http.Error(rw, err.Error(), http.StatusForbidden) - return - } + // SecuredCheck() only worked with TokenAuth: Removed if user := repository.GetUserFromContext(r.Context()); !user.HasRole(schema.RoleAdmin) { http.Error(rw, "Only admins are allowed to delete a user", http.StatusForbidden) @@ -1200,7 +1150,6 @@ func (api *RestApi) deleteUser(rw http.ResponseWriter, r *http.Request) { // @tags User // @description Returns a JSON-encoded list of users. // @description Required query-parameter defines if all users or only users with additional special roles are returned. -// @description Only accessible from IPs registered with apiAllowedIPs configuration option. // @produce json // @param not-just-user query bool true "If returned list should contain all users or only users with additional special roles" // @success 200 {array} api.ApiReturnedUser "List of users returned successfully" @@ -1211,11 +1160,7 @@ func (api *RestApi) deleteUser(rw http.ResponseWriter, r *http.Request) { // @security ApiKeyAuth // @router /users/ [get] func (api *RestApi) getUsers(rw http.ResponseWriter, r *http.Request) { - err := securedCheck(r) - if err != nil { - http.Error(rw, err.Error(), http.StatusForbidden) - return - } + // SecuredCheck() only worked with TokenAuth: Removed if user := repository.GetUserFromContext(r.Context()); !user.HasRole(schema.RoleAdmin) { http.Error(rw, "Only admins are allowed to fetch a list of users", http.StatusForbidden) @@ -1236,7 +1181,6 @@ func (api *RestApi) getUsers(rw http.ResponseWriter, r *http.Request) { // @tags User // @description Modifies user defined by username (id) in one of four possible ways. // @description If more than one formValue is set then only the highest priority field is used. -// @description Only accessible from IPs registered with apiAllowedIPs configuration option. // @accept mpfd // @produce plain // @param id path string true "Database ID of User" @@ -1253,11 +1197,7 @@ func (api *RestApi) getUsers(rw http.ResponseWriter, r *http.Request) { // @security ApiKeyAuth // @router /user/{id} [post] func (api *RestApi) updateUser(rw http.ResponseWriter, r *http.Request) { - err := securedCheck(r) - if err != nil { - http.Error(rw, err.Error(), http.StatusForbidden) - return - } + // SecuredCheck() only worked with TokenAuth: Removed if user := repository.GetUserFromContext(r.Context()); !user.HasRole(schema.RoleAdmin) { http.Error(rw, "Only admins are allowed to update a user", http.StatusForbidden) @@ -1305,7 +1245,6 @@ func (api *RestApi) updateUser(rw http.ResponseWriter, r *http.Request) { // @tags User // @description Modifies the content of notice.txt, shown as notice box on the homepage. // @description If more than one formValue is set then only the highest priority field is used. -// @description Only accessible from IPs registered with apiAllowedIPs configuration option. // @accept mpfd // @produce plain // @param new-content formData string false "Priority 1: New content to display" @@ -1318,11 +1257,7 @@ func (api *RestApi) updateUser(rw http.ResponseWriter, r *http.Request) { // @security ApiKeyAuth // @router /notice/ [post] func (api *RestApi) editNotice(rw http.ResponseWriter, r *http.Request) { - err := securedCheck(r) - if err != nil { - http.Error(rw, err.Error(), http.StatusForbidden) - return - } + // SecuredCheck() only worked with TokenAuth: Removed if user := repository.GetUserFromContext(r.Context()); !user.HasRole(schema.RoleAdmin) { http.Error(rw, "Only admins are allowed to update the notice.txt file", http.StatusForbidden) @@ -1364,12 +1299,6 @@ func (api *RestApi) editNotice(rw http.ResponseWriter, r *http.Request) { } func (api *RestApi) getJWT(rw http.ResponseWriter, r *http.Request) { - err := securedCheck(r) - if err != nil { - http.Error(rw, err.Error(), http.StatusForbidden) - return - } - rw.Header().Set("Content-Type", "text/plain") username := r.FormValue("username") me := repository.GetUserFromContext(r.Context()) @@ -1398,11 +1327,7 @@ func (api *RestApi) getJWT(rw http.ResponseWriter, r *http.Request) { } func (api *RestApi) getRoles(rw http.ResponseWriter, r *http.Request) { - err := securedCheck(r) - if err != nil { - http.Error(rw, err.Error(), http.StatusForbidden) - return - } + // SecuredCheck() only worked with TokenAuth: Removed user := repository.GetUserFromContext(r.Context()) if !user.HasRole(schema.RoleAdmin) { diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 262204c..d5e48ac 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -10,9 +10,11 @@ import ( "database/sql" "encoding/base64" "errors" + "fmt" "net" "net/http" "os" + "strings" "sync" "time" @@ -20,6 +22,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/repository" + "github.com/ClusterCockpit/cc-backend/internal/util" "github.com/ClusterCockpit/cc-backend/pkg/log" "github.com/ClusterCockpit/cc-backend/pkg/schema" "github.com/gorilla/sessions" @@ -233,9 +236,9 @@ func (auth *Authentication) Login( limiter := getIPUserLimiter(ip, username) if !limiter.Allow() { - log.Warnf("AUTH/RATE > Too many login attempts for combination IP: %s, Username: %s", ip, username) - onfailure(rw, r, errors.New("Too many login attempts, try again in a few minutes.")) - return + log.Warnf("AUTH/RATE > Too many login attempts for combination IP: %s, Username: %s", ip, username) + onfailure(rw, r, errors.New("Too many login attempts, try again in a few minutes.")) + return } var dbUser *schema.User @@ -325,6 +328,14 @@ func (auth *Authentication) AuthApi( onfailure(rw, r, err) return } + + ipErr := securedCheck(user, "api", r) + if ipErr != nil { + log.Infof("auth api -> secured check failed: %s", err.Error()) + onfailure(rw, r, ipErr) + return + } + if user != nil { switch { case len(user.Roles) == 1: @@ -360,6 +371,14 @@ func (auth *Authentication) AuthUserApi( onfailure(rw, r, err) return } + + ipErr := securedCheck(user, "userapi", r) + if ipErr != nil { + log.Infof("auth user api -> secured check failed: %s", err.Error()) + onfailure(rw, r, ipErr) + return + } + if user != nil { switch { case len(user.Roles) == 1: @@ -445,3 +464,58 @@ func (auth *Authentication) Logout(onsuccess http.Handler) http.Handler { onsuccess.ServeHTTP(rw, r) }) } + +// Helper Moved To MiddleWare Auth Handlers +func securedCheck(user *schema.User, checkEndpoint string, r *http.Request) error { + if user == nil { + return fmt.Errorf("no user for secured check") + } + + // extract IP address for checking + IPAddress := r.Header.Get("X-Real-Ip") + if IPAddress == "" { + IPAddress = r.Header.Get("X-Forwarded-For") + } + if IPAddress == "" { + IPAddress = r.RemoteAddr + } + + if strings.Contains(IPAddress, ":") { + IPAddress = strings.Split(IPAddress, ":")[0] + } + + // Used for checking TokenAuth'd Requests Only: Remove '== schema.AuthToken'-Condition + if checkEndpoint == "api" { + // If nothing declared in config: deny all request to this api endpoint + if config.Keys.ApiAllowedIPs == nil || len(config.Keys.ApiAllowedIPs) == 0 { + return fmt.Errorf("missing configuration key ApiAllowedIPs") + } + // If wildcard declared in config: Continue + if config.Keys.ApiAllowedIPs[0] == "*" { + return nil + } + // check if IP is allowed + if !util.Contains(config.Keys.ApiAllowedIPs, IPAddress) { + return fmt.Errorf("unknown ip: %v", IPAddress) + } + + } else if checkEndpoint == "userapi" { + // If nothing declared in config: deny all request to this api endpoint + if config.Keys.UserApiAllowedIPs == nil || len(config.Keys.UserApiAllowedIPs) == 0 { + return fmt.Errorf("missing configuration key UserApiAllowedIPs") + } + // If wildcard declared in config: Continue + if config.Keys.UserApiAllowedIPs[0] == "*" { + return nil + } + // check if IP is allowed + if !util.Contains(config.Keys.UserApiAllowedIPs, IPAddress) { + return fmt.Errorf("unknown user ip: %v", IPAddress) + } + + } else { + return fmt.Errorf("unknown checkEndpoint for secured check") + } + + return nil +} diff --git a/pkg/schema/config.go b/pkg/schema/config.go index f9116cf..16b4219 100644 --- a/pkg/schema/config.go +++ b/pkg/schema/config.go @@ -100,9 +100,12 @@ type ProgramConfig struct { // Address where the http (or https) server will listen on (for example: 'localhost:80'). Addr string `json:"addr"` - // Addresses from which secured API endpoints can be reached + // Addresses from which secured admin API endpoints can be reached, can be wildcard "*" ApiAllowedIPs []string `json:"apiAllowedIPs"` + // Addresses from which secured admin API endpoints can be reached, can be wildcard "*" + UserApiAllowedIPs []string `json:"userApiAllowedIPs"` + // Drop root permissions once .env was read and the port was taken. User string `json:"user"` Group string `json:"group"` From 25d3325049ad9060399da27d2e3e9807aaa0c630 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 14 Apr 2025 11:36:03 +0200 Subject: [PATCH 47/82] add getUsers to admin REST api --- internal/api/rest.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/api/rest.go b/internal/api/rest.go index 352dd94..0fa4611 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -70,6 +70,11 @@ func New() *RestApi { func (api *RestApi) MountApiRoutes(r *mux.Router) { r.StrictSlash(true) // REST API Uses TokenAuth + // User List + r.HandleFunc("/users/", api.getUsers).Methods(http.MethodGet) + // Cluster List + r.HandleFunc("/clusters/", api.getClusters).Methods(http.MethodGet) + // Job Handler r.HandleFunc("/jobs/start_job/", api.startJob).Methods(http.MethodPost, http.MethodPut) r.HandleFunc("/jobs/stop_job/", api.stopJobByRequest).Methods(http.MethodPost, http.MethodPut) // r.HandleFunc("/jobs/import/", api.importJob).Methods(http.MethodPost, http.MethodPut) @@ -82,7 +87,6 @@ func (api *RestApi) MountApiRoutes(r *mux.Router) { r.HandleFunc("/jobs/delete_job/", api.deleteJobByRequest).Methods(http.MethodDelete) r.HandleFunc("/jobs/delete_job/{id}", api.deleteJobById).Methods(http.MethodDelete) r.HandleFunc("/jobs/delete_job_before/{ts}", api.deleteJobBefore).Methods(http.MethodDelete) - r.HandleFunc("/clusters/", api.getClusters).Methods(http.MethodGet) if api.MachineStateDir != "" { r.HandleFunc("/machine_state/{cluster}/{host}", api.getMachineState).Methods(http.MethodGet) From 1755a4a7dfd851f211669fe155a75afe1373aa6f Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 14 Apr 2025 11:58:42 +0200 Subject: [PATCH 48/82] remove separate userapiallowedips config and check --- internal/auth/auth.go | 53 +++++++++++-------------------------------- pkg/schema/config.go | 3 --- 2 files changed, 13 insertions(+), 43 deletions(-) diff --git a/internal/auth/auth.go b/internal/auth/auth.go index d5e48ac..9201315 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -329,7 +329,7 @@ func (auth *Authentication) AuthApi( return } - ipErr := securedCheck(user, "api", r) + ipErr := securedCheck(user, r) if ipErr != nil { log.Infof("auth api -> secured check failed: %s", err.Error()) onfailure(rw, r, ipErr) @@ -372,13 +372,6 @@ func (auth *Authentication) AuthUserApi( return } - ipErr := securedCheck(user, "userapi", r) - if ipErr != nil { - log.Infof("auth user api -> secured check failed: %s", err.Error()) - onfailure(rw, r, ipErr) - return - } - if user != nil { switch { case len(user.Roles) == 1: @@ -466,7 +459,7 @@ func (auth *Authentication) Logout(onsuccess http.Handler) http.Handler { } // Helper Moved To MiddleWare Auth Handlers -func securedCheck(user *schema.User, checkEndpoint string, r *http.Request) error { +func securedCheck(user *schema.User, r *http.Request) error { if user == nil { return fmt.Errorf("no user for secured check") } @@ -484,37 +477,17 @@ func securedCheck(user *schema.User, checkEndpoint string, r *http.Request) erro IPAddress = strings.Split(IPAddress, ":")[0] } - // Used for checking TokenAuth'd Requests Only: Remove '== schema.AuthToken'-Condition - if checkEndpoint == "api" { - // If nothing declared in config: deny all request to this api endpoint - if config.Keys.ApiAllowedIPs == nil || len(config.Keys.ApiAllowedIPs) == 0 { - return fmt.Errorf("missing configuration key ApiAllowedIPs") - } - // If wildcard declared in config: Continue - if config.Keys.ApiAllowedIPs[0] == "*" { - return nil - } - // check if IP is allowed - if !util.Contains(config.Keys.ApiAllowedIPs, IPAddress) { - return fmt.Errorf("unknown ip: %v", IPAddress) - } - - } else if checkEndpoint == "userapi" { - // If nothing declared in config: deny all request to this api endpoint - if config.Keys.UserApiAllowedIPs == nil || len(config.Keys.UserApiAllowedIPs) == 0 { - return fmt.Errorf("missing configuration key UserApiAllowedIPs") - } - // If wildcard declared in config: Continue - if config.Keys.UserApiAllowedIPs[0] == "*" { - return nil - } - // check if IP is allowed - if !util.Contains(config.Keys.UserApiAllowedIPs, IPAddress) { - return fmt.Errorf("unknown user ip: %v", IPAddress) - } - - } else { - return fmt.Errorf("unknown checkEndpoint for secured check") + // If nothing declared in config: deny all request to this api endpoint + if len(config.Keys.ApiAllowedIPs) == 0 { + return fmt.Errorf("missing configuration key ApiAllowedIPs") + } + // If wildcard declared in config: Continue + if config.Keys.ApiAllowedIPs[0] == "*" { + return nil + } + // check if IP is allowed + if !util.Contains(config.Keys.ApiAllowedIPs, IPAddress) { + return fmt.Errorf("unknown ip: %v", IPAddress) } return nil diff --git a/pkg/schema/config.go b/pkg/schema/config.go index 16b4219..27d11be 100644 --- a/pkg/schema/config.go +++ b/pkg/schema/config.go @@ -103,9 +103,6 @@ type ProgramConfig struct { // Addresses from which secured admin API endpoints can be reached, can be wildcard "*" ApiAllowedIPs []string `json:"apiAllowedIPs"` - // Addresses from which secured admin API endpoints can be reached, can be wildcard "*" - UserApiAllowedIPs []string `json:"userApiAllowedIPs"` - // Drop root permissions once .env was read and the port was taken. User string `json:"user"` Group string `json:"group"` From 29ae2423f845a71f21b1a6f2453e8ad3de7fb3c0 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 16 Apr 2025 18:36:12 +0200 Subject: [PATCH 49/82] fix metricconfig pointer copy, add disabled metric card in jobView - skips disabled metrics in backend, see cc-backend tries to retrieve "removed" metrics #377 --- internal/metricdata/cc-metric-store.go | 28 ++++++++++++++++++++ pkg/archive/clusterConfig.go | 19 ++++++++++++-- web/frontend/src/Job.root.svelte | 36 ++++++++++++++++++-------- web/frontend/src/job/Metric.svelte | 2 -- 4 files changed, 70 insertions(+), 15 deletions(-) diff --git a/internal/metricdata/cc-metric-store.go b/internal/metricdata/cc-metric-store.go index 9516e2b..7c84d93 100644 --- a/internal/metricdata/cc-metric-store.go +++ b/internal/metricdata/cc-metric-store.go @@ -302,6 +302,20 @@ func (ccms *CCMetricStore) buildQueries( continue } + // Skip if metric is removed for subcluster + if len(mc.SubClusters) != 0 { + isRemoved := false + for _, scConfig := range mc.SubClusters { + if scConfig.Name == job.SubCluster && scConfig.Remove == true { + isRemoved = true + break + } + } + if isRemoved { + continue + } + } + // Avoid duplicates... handledScopes := make([]schema.MetricScope, 0, 3) @@ -985,6 +999,20 @@ func (ccms *CCMetricStore) buildNodeQueries( continue } + // Skip if metric is removed for subcluster + if mc.SubClusters != nil { + isRemoved := false + for _, scConfig := range mc.SubClusters { + if scConfig.Name == subCluster && scConfig.Remove == true { + isRemoved = true + break + } + } + if isRemoved { + continue + } + } + // Avoid duplicates... handledScopes := make([]schema.MetricScope, 0, 3) diff --git a/pkg/archive/clusterConfig.go b/pkg/archive/clusterConfig.go index 72718d0..d53941b 100644 --- a/pkg/archive/clusterConfig.go +++ b/pkg/archive/clusterConfig.go @@ -68,8 +68,23 @@ func initClusterConfig() error { } for _, sc := range cluster.SubClusters { - newMetric := mc - newMetric.SubClusters = nil + newMetric := &schema.MetricConfig{ + Unit: mc.Unit, + Energy: mc.Energy, + Name: mc.Name, + Scope: mc.Scope, + Aggregation: mc.Aggregation, + Peak: mc.Peak, + Caution: mc.Caution, + Alert: mc.Alert, + Timestep: mc.Timestep, + Normal: mc.Normal, + LowerIsBetter: mc.LowerIsBetter, + } + + if mc.Footprint != "" { + newMetric.Footprint = mc.Footprint + } if cfg, ok := scLookup[sc.Name]; ok { if !cfg.Remove { diff --git a/web/frontend/src/Job.root.svelte b/web/frontend/src/Job.root.svelte index 0a2aa26..92d8bb2 100644 --- a/web/frontend/src/Job.root.svelte +++ b/web/frontend/src/Job.root.svelte @@ -128,14 +128,13 @@ const pendingMetrics = ( ccconfig[`job_view_selectedMetrics:${job.cluster}:${job.subCluster}`] || ccconfig[`job_view_selectedMetrics:${job.cluster}`] - ) || - $initq.data.globalMetrics - .reduce((names, gm) => { - if (gm.availability.find((av) => av.cluster === job.cluster && av.subClusters.includes(job.subCluster))) { - names.push(gm.name); - } - return names; - }, []) + ) || + $initq.data.globalMetrics.reduce((names, gm) => { + if (gm.availability.find((av) => av.cluster === job.cluster && av.subClusters.includes(job.subCluster))) { + names.push(gm.name); + } + return names; + }, []) // Select default Scopes to load: Check before if any metric has accelerator scope by default const accScopeDefault = [...pendingMetrics].some(function (m) { @@ -338,10 +337,25 @@ scopes={item.data.map((x) => x.scope)} isShared={$initq.data.job.exclusive != 1} /> + {:else if item.disabled == true} + + + Disabled Metric + + +

Metric {item.metric} is disabled for subcluster {$initq.data.job.subCluster}.

+

To remove this card, open metric selection and press "Close and Apply".

+ + {:else} - No dataset returned for {item.metric} + + + Missing Metric + + +

No dataset returned for {item.metric}.

+ + {/if} {/if} diff --git a/web/frontend/src/job/Metric.svelte b/web/frontend/src/job/Metric.svelte index b68ef47..63a9b80 100644 --- a/web/frontend/src/job/Metric.svelte +++ b/web/frontend/src/job/Metric.svelte @@ -14,7 +14,6 @@ diff --git a/web/frontend/src/systems/nodelist/NodeListRow.svelte b/web/frontend/src/systems/nodelist/NodeListRow.svelte index 5202573..ee8ef49 100644 --- a/web/frontend/src/systems/nodelist/NodeListRow.svelte +++ b/web/frontend/src/systems/nodelist/NodeListRow.svelte @@ -14,7 +14,7 @@ getContextClient, } from "@urql/svelte"; import { Card, CardBody, Spinner } from "@sveltestrap/sveltestrap"; - import { maxScope, checkMetricDisabled } from "../../generic/utils.js"; + import { maxScope, checkMetricDisabled, scramble, scrambleNames } from "../../generic/utils.js"; import MetricPlot from "../../generic/plots/MetricPlot.svelte"; import NodeInfo from "./NodeInfo.svelte"; @@ -110,9 +110,12 @@ extendedLegendData = {} for (const accId of accSet) { const matchJob = $nodeJobsData.data.jobs.items.find((i) => i.resources.find((r) => r.accelerators.includes(accId))) + const matchUser = matchJob?.user ? matchJob.user : null extendedLegendData[accId] = { - user: matchJob?.user ? matchJob?.user : '-', - job: matchJob?.jobId ? matchJob?.jobId : '-', + user: (scrambleNames && matchUser) + ? scramble(matchUser) + : (matchUser ? matchUser : '-'), + job: matchJob?.jobId ? matchJob.jobId : '-', } } // Theoretically extendable for hwthreadIDs From 9bcf7adb67daa30c0c87a1e0dbaa10e87714716d Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 17 Apr 2025 17:31:59 +0200 Subject: [PATCH 51/82] add api calls for removing tags, initial branch commit --- internal/api/rest.go | 116 ++++++++++++++++++++++++++++++++++++ internal/repository/tags.go | 98 +++++++++++++++++++++++++++++- 2 files changed, 213 insertions(+), 1 deletion(-) diff --git a/internal/api/rest.go b/internal/api/rest.go index db9a860..89bdd5e 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -750,6 +750,122 @@ func (api *RestApi) tagJob(rw http.ResponseWriter, r *http.Request) { json.NewEncoder(rw).Encode(job) } +// removeTagJob godoc +// @summary Removes one or more tags from a job +// @tags Job add and modify +// @description Removes tag(s) from a job specified by DB ID. Name and Type of Tag(s) must match. +// @description Tag Scope is required for matching, options: "global", "admin". Private tags can not be deleted via API. +// @description If tagged job is already finished: Tag will be removed from respective archive files. +// @accept json +// @produce json +// @param id path int true "Job Database ID" +// @param request body api.TagJobApiRequest true "Array of tag-objects to remove" +// @success 200 {object} schema.Job "Updated job resource" +// @failure 400 {object} api.ErrorResponse "Bad Request" +// @failure 401 {object} api.ErrorResponse "Unauthorized" +// @failure 404 {object} api.ErrorResponse "Job or tag does not exist" +// @failure 500 {object} api.ErrorResponse "Internal Server Error" +// @security ApiKeyAuth +// @router /jobs/tag_job/{id} [delete] +func (api *RestApi) removeTagJob(rw http.ResponseWriter, r *http.Request) { + id, err := strconv.ParseInt(mux.Vars(r)["id"], 10, 64) + if err != nil { + http.Error(rw, err.Error(), http.StatusBadRequest) + return + } + + job, err := api.JobRepository.FindById(r.Context(), id) + if err != nil { + http.Error(rw, err.Error(), http.StatusNotFound) + return + } + + job.Tags, err = api.JobRepository.GetTags(repository.GetUserFromContext(r.Context()), &job.ID) + if err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) + return + } + + var req TagJobApiRequest + if err := decode(r.Body, &req); err != nil { + http.Error(rw, err.Error(), http.StatusBadRequest) + return + } + + for _, rtag := range req { + // Only Global and Admin Tags + if rtag.Scope != "global" && rtag.Scope != "admin" { + log.Warnf("Cannot delete private tag for job %d: Skip", job.JobID) + continue + } + + remainingTags, err := api.JobRepository.RemoveJobTagByRequest(repository.GetUserFromContext(r.Context()), job.ID, rtag.Type, rtag.Name, rtag.Scope) + if err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) + return + } + + // remainingTags := job.Tags[:0] + // for _, tag := range job.Tags { + // if tag.Type != rtag.Type && + // tag.Name != rtag.Name && + // tag.Scope != rtag.Scope { + // remainingTags = append(remainingTags, tag) + // } + // } + job.Tags = remainingTags + } + + rw.Header().Add("Content-Type", "application/json") + rw.WriteHeader(http.StatusOK) + json.NewEncoder(rw).Encode(job) +} + +// removeTags godoc +// @summary Removes all tags and job-relations for type:name tuple +// @tags Tag remove +// @description Removes tags by type and name. Name and Type of Tag(s) must match. +// @description Tag Scope is required for matching, options: "global", "admin". Private tags can not be deleted via API. +// @description Tag wills be removed from respective archive files. +// @accept json +// @produce plain +// @param request body api.TagJobApiRequest true "Array of tag-objects to remove" +// @success 200 {string} string "Success Response" +// @failure 400 {object} api.ErrorResponse "Bad Request" +// @failure 401 {object} api.ErrorResponse "Unauthorized" +// @failure 404 {object} api.ErrorResponse "Job or tag does not exist" +// @failure 500 {object} api.ErrorResponse "Internal Server Error" +// @security ApiKeyAuth +// @router /jobs/tag_job/ [delete] +func (api *RestApi) removeTags(rw http.ResponseWriter, r *http.Request) { + var req TagJobApiRequest + if err := decode(r.Body, &req); err != nil { + http.Error(rw, err.Error(), http.StatusBadRequest) + return + } + + targetCount := len(req) + currentCount := 0 + for _, rtag := range req { + // Only Global and Admin Tags + if rtag.Scope != "global" && rtag.Scope != "admin" { + log.Warn("Cannot delete private tag: Skip") + continue + } + + err := api.JobRepository.RemoveTagByRequest(rtag.Type, rtag.Name, rtag.Scope) + if err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) + return + } else { + currentCount++ + } + } + + rw.WriteHeader(http.StatusOK) + rw.Write([]byte(fmt.Sprintf("Deleted Tags from DB: %d of %d", currentCount, targetCount))) +} + // startJob godoc // @summary Adds a new job as "running" // @tags Job add and modify diff --git a/internal/repository/tags.go b/internal/repository/tags.go index 8120364..3a35b34 100644 --- a/internal/repository/tags.go +++ b/internal/repository/tags.go @@ -45,7 +45,7 @@ func (r *JobRepository) AddTag(user *schema.User, job int64, tag int64) ([]*sche return tags, archive.UpdateTags(j, archiveTags) } -// Removes a tag from a job +// Removes a tag from a job by its ID func (r *JobRepository) RemoveTag(user *schema.User, job, tag int64) ([]*schema.Tag, error) { j, err := r.FindByIdWithUser(user, job) if err != nil { @@ -76,6 +76,76 @@ func (r *JobRepository) RemoveTag(user *schema.User, job, tag int64) ([]*schema. return tags, archive.UpdateTags(j, archiveTags) } +// Removes a tag from a job by tag info +func (r *JobRepository) RemoveJobTagByRequest(user *schema.User, job int64, tagType string, tagName string, tagScope string) ([]*schema.Tag, error) { + // Get Tag ID to delete + tagID, err := r.loadTagIDByInfo(tagName, tagType, tagScope) + if err != nil { + log.Warn("Error while finding tagId with: %s, %s, %s", tagName, tagType, tagScope) + return nil, err + } + + // Get Job + j, err := r.FindByIdWithUser(user, job) + if err != nil { + log.Warn("Error while finding job by id") + return nil, err + } + + // Handle Delete + q := sq.Delete("jobtag").Where("jobtag.job_id = ?", job).Where("jobtag.tag_id = ?", tagID) + + if _, err := q.RunWith(r.stmtCache).Exec(); err != nil { + s, _, _ := q.ToSql() + log.Errorf("Error removing tag from table 'jobTag' with %s: %v", s, err) + return nil, err + } + + tags, err := r.GetTags(user, &job) + if err != nil { + log.Warn("Error while getting tags for job") + return nil, err + } + + archiveTags, err := r.getArchiveTags(&job) + if err != nil { + log.Warn("Error while getting tags for job") + return nil, err + } + + return tags, archive.UpdateTags(j, archiveTags) +} + +// Removes a tag from db by tag info +func (r *JobRepository) RemoveTagByRequest(tagType string, tagName string, tagScope string) error { + // Get Tag ID to delete + tagID, err := r.loadTagIDByInfo(tagName, tagType, tagScope) + if err != nil { + log.Warn("Error while finding tagId with: %s, %s, %s", tagName, tagType, tagScope) + return err + } + + // Handle Delete JobTagTable + qJobTag := sq.Delete("jobtag").Where("jobtag.tag_id = ?", tagID) + + if _, err := qJobTag.RunWith(r.stmtCache).Exec(); err != nil { + s, _, _ := qJobTag.ToSql() + log.Errorf("Error removing tag from table 'jobTag' with %s: %v", s, err) + return err + } + + // Handle Delete TagTable + qTag := sq.Delete("tag").Where("tag.id = ?", tagID) + + if _, err := qTag.RunWith(r.stmtCache).Exec(); err != nil { + s, _, _ := qTag.ToSql() + log.Errorf("Error removing tag from table 'tag' with %s: %v", s, err) + return err + } + + return nil +} + // CreateTag creates a new tag with the specified type and name and returns its database id. func (r *JobRepository) CreateTag(tagType string, tagName string, tagScope string) (tagId int64, err error) { // Default to "Global" scope if none defined @@ -325,3 +395,29 @@ func (r *JobRepository) checkScopeAuth(user *schema.User, operation string, scop return false, fmt.Errorf("error while checking tag operation auth: no user in context") } } + +func (r *JobRepository) loadTagIDByInfo(tagType string, tagName string, tagScope string) (tagID int64, err error) { + // Get Tag ID to delete + getq := sq.Select("id").From("tag"). + Where("tag_type = ?", tagType). + Where("tag_name = ?", tagName). + Where("tag_scope = ?", tagScope) + + rows, err := getq.RunWith(r.stmtCache).Query() + if err != nil { + s, _, _ := getq.ToSql() + log.Errorf("Error get tags for delete with %s: %v", s, err) + return 0, err + } + + dbTags := make([]*schema.Tag, 0) + for rows.Next() { + dbTag := &schema.Tag{} + if err := rows.Scan(&dbTag.ID); err != nil { + log.Warn("Error while scanning rows") + return 0, err + } + } + + return dbTags[0].ID, nil +} From 277f964b30e76a7726fc75bd3673d1f947068627 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 22 Apr 2025 13:47:25 +0200 Subject: [PATCH 52/82] move taglist a from go tmpl to svelte component --- web/frontend/rollup.config.mjs | 1 + web/frontend/src/Tags.root.svelte | 50 +++++++++++++++++++++++++++ web/frontend/src/tags.entrypoint.js | 12 +++++++ web/templates/monitoring/taglist.tmpl | 46 ++++++------------------ 4 files changed, 74 insertions(+), 35 deletions(-) create mode 100644 web/frontend/src/Tags.root.svelte create mode 100644 web/frontend/src/tags.entrypoint.js diff --git a/web/frontend/rollup.config.mjs b/web/frontend/rollup.config.mjs index 8336287..0e15105 100644 --- a/web/frontend/rollup.config.mjs +++ b/web/frontend/rollup.config.mjs @@ -62,6 +62,7 @@ export default [ entrypoint('jobs', 'src/jobs.entrypoint.js'), entrypoint('user', 'src/user.entrypoint.js'), entrypoint('list', 'src/list.entrypoint.js'), + entrypoint('taglist', 'src/tags.entrypoint.js'), entrypoint('job', 'src/job.entrypoint.js'), entrypoint('systems', 'src/systems.entrypoint.js'), entrypoint('node', 'src/node.entrypoint.js'), diff --git a/web/frontend/src/Tags.root.svelte b/web/frontend/src/Tags.root.svelte new file mode 100644 index 0000000..4f7a34e --- /dev/null +++ b/web/frontend/src/Tags.root.svelte @@ -0,0 +1,50 @@ + + + + +
+
+
+ {#each Object.entries(tagmap) as [tagType, tagList]} +
+ Tag Type: {tagType} + + {tagList.length} Tag{(tagList.length != 1)?'s':''} + +
+ {#each tagList as tag (tag.id)} + {#if tag.scope == "global"} + + {tag.name} + {tag.count} Job{(tag.count != 1)?'s':''} + Global + + {:else if tag.scope == "admin"} + + {tag.name} + {tag.count} Job{(tag.count != 1)?'s':''} + Admin + + {:else} + + {tag.name} + {tag.count} Job{(tag.count != 1)?'s':''} + Private + + {/if} + {/each} + {/each} +
+
+
diff --git a/web/frontend/src/tags.entrypoint.js b/web/frontend/src/tags.entrypoint.js new file mode 100644 index 0000000..14df2f9 --- /dev/null +++ b/web/frontend/src/tags.entrypoint.js @@ -0,0 +1,12 @@ +import {} from './header.entrypoint.js' +import Tags from './Tags.root.svelte' + +new Tags({ + target: document.getElementById('svelte-app'), + props: { + // authlevel: authlevel, + tagmap: tagmap, + } +}) + + diff --git a/web/templates/monitoring/taglist.tmpl b/web/templates/monitoring/taglist.tmpl index 7d762c3..4388e94 100644 --- a/web/templates/monitoring/taglist.tmpl +++ b/web/templates/monitoring/taglist.tmpl @@ -1,37 +1,13 @@ {{define "content"}} -
-
-
- {{ range $tagType, $tagList := .Infos.tagmap }} -
- Tag Type: {{ $tagType }} - - {{len $tagList}} Tag{{if ne (len $tagList) 1}}s{{end}} - -
- {{ range $tagList }} - {{if eq .scope "global"}} - - {{ .name }} - {{ .count }} Job{{if ne .count 1}}s{{end}} - Global - - {{else if eq .scope "admin"}} - - {{ .name }} - {{ .count }} Job{{if ne .count 1}}s{{end}} - Admin - - {{else}} - - {{ .name }} - {{ .count }} Job{{if ne .count 1}}s{{end}} - Private - - {{end}} - {{end}} - {{end}} -
-
-
+
+{{end}} +{{define "stylesheets"}} + +{{end}} +{{define "javascript"}} + + {{end}} From a3fb47154627d4b8dcaacf121a29053527388081 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 22 Apr 2025 17:33:17 +0200 Subject: [PATCH 53/82] adapt and improve svelte taglist component --- web/frontend/src/Tags.root.svelte | 163 ++++++++++++++++++++------ web/frontend/src/tags.entrypoint.js | 8 +- web/templates/monitoring/taglist.tmpl | 4 +- 3 files changed, 137 insertions(+), 38 deletions(-) diff --git a/web/frontend/src/Tags.root.svelte b/web/frontend/src/Tags.root.svelte index 4f7a34e..52288c9 100644 --- a/web/frontend/src/Tags.root.svelte +++ b/web/frontend/src/Tags.root.svelte @@ -2,49 +2,142 @@ @component Tag List Svelte Component. Displays All Tags, Allows deletion. Properties: - - `authlevel Int!`: Current Users Authority Level - - `tagmap Object!`: Map of Appwide Tags + - `username String!`: Users username. + - `isAdmin Bool!`: User has Admin Auth. + - `tagmap Object!`: Map of accessible, appwide tags. Prefiltered in backend. -->
-
-
- {#each Object.entries(tagmap) as [tagType, tagList]} -
- Tag Type: {tagType} - - {tagList.length} Tag{(tagList.length != 1)?'s':''} - -
- {#each tagList as tag (tag.id)} - {#if tag.scope == "global"} - - {tag.name} - {tag.count} Job{(tag.count != 1)?'s':''} - Global - - {:else if tag.scope == "admin"} - - {tag.name} - {tag.count} Job{(tag.count != 1)?'s':''} - Admin - - {:else} - - {tag.name} - {tag.count} Job{(tag.count != 1)?'s':''} - Private - - {/if} - {/each} - {/each} +
+
+ {#each Object.entries(tagmap) as [tagType, tagList]} +
+ Tag Type: {tagType} + {#if pendingChange === tagType} + + {/if} + + {tagList.length} Tag{(tagList.length != 1)?'s':''} +
+
+ {#each tagList as tag (tag.id)} + {#if tag.scope == "global"} + + + {#if isAdmin} + + {/if} + + {:else if tag.scope == "admin"} + + + {#if isAdmin} + + {/if} + + {:else} + + + {#if tag.scope == username} + + {/if} + + {/if} + {/each} +
+ {/each}
+
diff --git a/web/frontend/src/tags.entrypoint.js b/web/frontend/src/tags.entrypoint.js index 14df2f9..024a92d 100644 --- a/web/frontend/src/tags.entrypoint.js +++ b/web/frontend/src/tags.entrypoint.js @@ -4,9 +4,13 @@ import Tags from './Tags.root.svelte' new Tags({ target: document.getElementById('svelte-app'), props: { - // authlevel: authlevel, + username: username, + isAdmin: isAdmin, tagmap: tagmap, - } + }, + context: new Map([ + ['cc-config', clusterCockpitConfig] + ]) }) diff --git a/web/templates/monitoring/taglist.tmpl b/web/templates/monitoring/taglist.tmpl index 4388e94..66122fe 100644 --- a/web/templates/monitoring/taglist.tmpl +++ b/web/templates/monitoring/taglist.tmpl @@ -6,8 +6,10 @@ {{end}} {{define "javascript"}} {{end}} From 543ddf540ea540bf0c3dd5f550593cc096ca224a Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 23 Apr 2025 14:51:01 +0200 Subject: [PATCH 54/82] implement removeTagFromList mutation, add tag mutation access checks --- api/schema.graphqls | 1 + go.mod | 1 + internal/graph/generated/generated.go | 105 ++++++++++++++++++++++ internal/graph/schema.resolvers.go | 124 +++++++++++++++++++++++--- internal/repository/tags.go | 73 ++++++++------- pkg/schema/user.go | 8 +- web/frontend/src/Tags.root.svelte | 20 ++--- 7 files changed, 272 insertions(+), 60 deletions(-) diff --git a/api/schema.graphqls b/api/schema.graphqls index ed8843c..9092b4f 100644 --- a/api/schema.graphqls +++ b/api/schema.graphqls @@ -277,6 +277,7 @@ type Mutation { deleteTag(id: ID!): ID! addTagsToJob(job: ID!, tagIds: [ID!]!): [Tag!]! removeTagsFromJob(job: ID!, tagIds: [ID!]!): [Tag!]! + removeTagFromList(tagIds: [ID!]!): [Int!]! updateConfiguration(name: String!, value: String!): String } diff --git a/go.mod b/go.mod index 2e2aa36..47e3497 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,7 @@ module github.com/ClusterCockpit/cc-backend go 1.23.5 + toolchain go1.24.1 require ( diff --git a/internal/graph/generated/generated.go b/internal/graph/generated/generated.go index e5c9ca2..5dbdfd9 100644 --- a/internal/graph/generated/generated.go +++ b/internal/graph/generated/generated.go @@ -250,6 +250,7 @@ type ComplexityRoot struct { AddTagsToJob func(childComplexity int, job string, tagIds []string) int CreateTag func(childComplexity int, typeArg string, name string, scope string) int DeleteTag func(childComplexity int, id string) int + RemoveTagFromList func(childComplexity int, tagIds []string) int RemoveTagsFromJob func(childComplexity int, job string, tagIds []string) int UpdateConfiguration func(childComplexity int, name string, value string) int } @@ -399,6 +400,7 @@ type MutationResolver interface { DeleteTag(ctx context.Context, id string) (string, error) AddTagsToJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error) RemoveTagsFromJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error) + RemoveTagFromList(ctx context.Context, tagIds []string) ([]int, error) UpdateConfiguration(ctx context.Context, name string, value string) (*string, error) } type QueryResolver interface { @@ -1310,6 +1312,18 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Mutation.DeleteTag(childComplexity, args["id"].(string)), true + case "Mutation.removeTagFromList": + if e.complexity.Mutation.RemoveTagFromList == nil { + break + } + + args, err := ec.field_Mutation_removeTagFromList_args(context.TODO(), rawArgs) + if err != nil { + return 0, false + } + + return e.complexity.Mutation.RemoveTagFromList(childComplexity, args["tagIds"].([]string)), true + case "Mutation.removeTagsFromJob": if e.complexity.Mutation.RemoveTagsFromJob == nil { break @@ -2339,6 +2353,7 @@ type Mutation { deleteTag(id: ID!): ID! addTagsToJob(job: ID!, tagIds: [ID!]!): [Tag!]! removeTagsFromJob(job: ID!, tagIds: [ID!]!): [Tag!]! + removeTagFromList(tagIds: [ID!]!): [Int!]! updateConfiguration(name: String!, value: String!): String } @@ -2617,6 +2632,34 @@ func (ec *executionContext) field_Mutation_deleteTag_argsID( return zeroVal, nil } +func (ec *executionContext) field_Mutation_removeTagFromList_args(ctx context.Context, rawArgs map[string]any) (map[string]any, error) { + var err error + args := map[string]any{} + arg0, err := ec.field_Mutation_removeTagFromList_argsTagIds(ctx, rawArgs) + if err != nil { + return nil, err + } + args["tagIds"] = arg0 + return args, nil +} +func (ec *executionContext) field_Mutation_removeTagFromList_argsTagIds( + ctx context.Context, + rawArgs map[string]any, +) ([]string, error) { + if _, ok := rawArgs["tagIds"]; !ok { + var zeroVal []string + return zeroVal, nil + } + + ctx = graphql.WithPathContext(ctx, graphql.NewPathWithField("tagIds")) + if tmp, ok := rawArgs["tagIds"]; ok { + return ec.unmarshalNID2ᚕstringᚄ(ctx, tmp) + } + + var zeroVal []string + return zeroVal, nil +} + func (ec *executionContext) field_Mutation_removeTagsFromJob_args(ctx context.Context, rawArgs map[string]any) (map[string]any, error) { var err error args := map[string]any{} @@ -9690,6 +9733,61 @@ func (ec *executionContext) fieldContext_Mutation_removeTagsFromJob(ctx context. return fc, nil } +func (ec *executionContext) _Mutation_removeTagFromList(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_Mutation_removeTagFromList(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return ec.resolvers.Mutation().RemoveTagFromList(rctx, fc.Args["tagIds"].([]string)) + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.([]int) + fc.Result = res + return ec.marshalNInt2ᚕintᚄ(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_Mutation_removeTagFromList(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "Mutation", + Field: field, + IsMethod: true, + IsResolver: true, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type Int does not have child fields") + }, + } + defer func() { + if r := recover(); r != nil { + err = ec.Recover(ctx, r) + ec.Error(ctx, err) + } + }() + ctx = graphql.WithFieldContext(ctx, fc) + if fc.Args, err = ec.field_Mutation_removeTagFromList_args(ctx, field.ArgumentMap(ec.Variables)); err != nil { + ec.Error(ctx, err) + return fc, err + } + return fc, nil +} + func (ec *executionContext) _Mutation_updateConfiguration(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { fc, err := ec.fieldContext_Mutation_updateConfiguration(ctx, field) if err != nil { @@ -17765,6 +17863,13 @@ func (ec *executionContext) _Mutation(ctx context.Context, sel ast.SelectionSet) if out.Values[i] == graphql.Null { out.Invalids++ } + case "removeTagFromList": + out.Values[i] = ec.OperationContext.RootResolverMiddleware(innerCtx, func(ctx context.Context) (res graphql.Marshaler) { + return ec._Mutation_removeTagFromList(ctx, field) + }) + if out.Values[i] == graphql.Null { + out.Invalids++ + } case "updateConfiguration": out.Values[i] = ec.OperationContext.RootResolverMiddleware(innerCtx, func(ctx context.Context) (res graphql.Marshaler) { return ec._Mutation_updateConfiguration(ctx, field) diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index 029be87..46f485b 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -125,23 +125,41 @@ func (r *metricValueResolver) Name(ctx context.Context, obj *schema.MetricValue) // CreateTag is the resolver for the createTag field. func (r *mutationResolver) CreateTag(ctx context.Context, typeArg string, name string, scope string) (*schema.Tag, error) { - id, err := r.Repo.CreateTag(typeArg, name, scope) - if err != nil { - log.Warn("Error while creating tag") - return nil, err + user := repository.GetUserFromContext(ctx) + if user == nil { + return nil, fmt.Errorf("no user in context") } - return &schema.Tag{ID: id, Type: typeArg, Name: name, Scope: scope}, nil + // Test Access: Admins && Admin Tag OR Support/Admin and Global Tag OR Everyone && Private Tag + if user.HasRole(schema.RoleAdmin) && scope == "admin" || + user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) && scope == "global" || + user.Username == scope { + // Create in DB + id, err := r.Repo.CreateTag(typeArg, name, scope) + if err != nil { + log.Warn("Error while creating tag") + return nil, err + } + return &schema.Tag{ID: id, Type: typeArg, Name: name, Scope: scope}, nil + } else { + log.Warn("Not authorized to create tag with scope: %s", scope) + return nil, fmt.Errorf("Not authorized to create tag with scope: %s", scope) + } } // DeleteTag is the resolver for the deleteTag field. func (r *mutationResolver) DeleteTag(ctx context.Context, id string) (string, error) { + // This Uses ID string <-> ID string, removeTagFromList uses []string <-> []int panic(fmt.Errorf("not implemented: DeleteTag - deleteTag")) } // AddTagsToJob is the resolver for the addTagsToJob field. func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error) { - // Selectable Tags Pre-Filtered by Scope in Frontend: No backend check required + user := repository.GetUserFromContext(ctx) + if user == nil { + return nil, fmt.Errorf("no user in context") + } + jid, err := strconv.ParseInt(job, 10, 64) if err != nil { log.Warn("Error while adding tag to job") @@ -150,15 +168,32 @@ func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds tags := []*schema.Tag{} for _, tagId := range tagIds { + // Get ID tid, err := strconv.ParseInt(tagId, 10, 64) if err != nil { log.Warn("Error while parsing tag id") return nil, err } - if tags, err = r.Repo.AddTag(repository.GetUserFromContext(ctx), jid, tid); err != nil { - log.Warn("Error while adding tag") - return nil, err + // Test Exists + _, _, tscope, exists := r.Repo.TagInfo(tid) + if !exists { + log.Warn("Tag does not exist (ID): %d", tid) + return nil, fmt.Errorf("Tag does not exist (ID): %d", tid) + } + + // Test Access: Admins && Admin Tag OR Support/Admin and Global Tag OR Everyone && Private Tag + if user.HasRole(schema.RoleAdmin) && tscope == "admin" || + user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) && tscope == "global" || + user.Username == tscope { + // Add to Job + if tags, err = r.Repo.AddTag(user, jid, tid); err != nil { + log.Warn("Error while adding tag") + return nil, err + } + } else { + log.Warn("Not authorized to add tag: %d", tid) + return nil, fmt.Errorf("Not authorized to add tag: %d", tid) } } @@ -167,7 +202,11 @@ func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds // RemoveTagsFromJob is the resolver for the removeTagsFromJob field. func (r *mutationResolver) RemoveTagsFromJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error) { - // Removable Tags Pre-Filtered by Scope in Frontend: No backend check required + user := repository.GetUserFromContext(ctx) + if user == nil { + return nil, fmt.Errorf("no user in context") + } + jid, err := strconv.ParseInt(job, 10, 64) if err != nil { log.Warn("Error while parsing job id") @@ -176,21 +215,80 @@ func (r *mutationResolver) RemoveTagsFromJob(ctx context.Context, job string, ta tags := []*schema.Tag{} for _, tagId := range tagIds { + // Get ID tid, err := strconv.ParseInt(tagId, 10, 64) if err != nil { log.Warn("Error while parsing tag id") return nil, err } - if tags, err = r.Repo.RemoveTag(repository.GetUserFromContext(ctx), jid, tid); err != nil { - log.Warn("Error while removing tag") - return nil, err + // Test Exists + _, _, tscope, exists := r.Repo.TagInfo(tid) + if !exists { + log.Warn("Tag does not exist (ID): %d", tid) + return nil, fmt.Errorf("Tag does not exist (ID): %d", tid) } + + // Test Access: Admins && Admin Tag OR Support/Admin and Global Tag OR Everyone && Private Tag + if user.HasRole(schema.RoleAdmin) && tscope == "admin" || + user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) && tscope == "global" || + user.Username == tscope { + // Remove from Job + if tags, err = r.Repo.RemoveTag(user, jid, tid); err != nil { + log.Warn("Error while removing tag") + return nil, err + } + } else { + log.Warn("Not authorized to remove tag: %d", tid) + return nil, fmt.Errorf("Not authorized to remove tag: %d", tid) + } + } return tags, nil } +// RemoveTagFromList is the resolver for the removeTagFromList field. +func (r *mutationResolver) RemoveTagFromList(ctx context.Context, tagIds []string) ([]int, error) { + // Needs Contextuser + user := repository.GetUserFromContext(ctx) + if user == nil { + return nil, fmt.Errorf("no user in context") + } + + tags := []int{} + for _, tagId := range tagIds { + // Get ID + tid, err := strconv.ParseInt(tagId, 10, 64) + if err != nil { + log.Warn("Error while parsing tag id for removal") + return nil, err + } + + // Test Exists + _, _, tscope, exists := r.Repo.TagInfo(tid) + if !exists { + log.Warn("Tag does not exist (ID): %d", tid) + return nil, fmt.Errorf("Tag does not exist (ID): %d", tid) + } + + // Test Access: Admins && Admin Tag OR Everyone && Private Tag + if user.HasRole(schema.RoleAdmin) && (tscope == "global" || tscope == "admin") || user.Username == tscope { + // Remove from DB + if err = r.Repo.RemoveTagById(tid); err != nil { + log.Warn("Error while removing tag") + return nil, err + } else { + tags = append(tags, int(tid)) + } + } else { + log.Warn("Not authorized to remove tag: %d", tid) + return nil, fmt.Errorf("Not authorized to remove tag: %d", tid) + } + } + return tags, nil +} + // UpdateConfiguration is the resolver for the updateConfiguration field. func (r *mutationResolver) UpdateConfiguration(ctx context.Context, name string, value string) (*string, error) { if err := repository.GetUserCfgRepo().UpdateConfig(name, value, repository.GetUserFromContext(ctx)); err != nil { diff --git a/internal/repository/tags.go b/internal/repository/tags.go index 3a35b34..5712c94 100644 --- a/internal/repository/tags.go +++ b/internal/repository/tags.go @@ -79,10 +79,10 @@ func (r *JobRepository) RemoveTag(user *schema.User, job, tag int64) ([]*schema. // Removes a tag from a job by tag info func (r *JobRepository) RemoveJobTagByRequest(user *schema.User, job int64, tagType string, tagName string, tagScope string) ([]*schema.Tag, error) { // Get Tag ID to delete - tagID, err := r.loadTagIDByInfo(tagName, tagType, tagScope) - if err != nil { - log.Warn("Error while finding tagId with: %s, %s, %s", tagName, tagType, tagScope) - return nil, err + tagID, exists := r.TagId(tagType, tagName, tagScope) + if !exists { + log.Warn("Tag does not exist (name, type, scope): %s, %s, %s", tagName, tagType, tagScope) + return nil, fmt.Errorf("Tag does not exist (name, type, scope): %s, %s, %s", tagName, tagType, tagScope) } // Get Job @@ -119,12 +119,35 @@ func (r *JobRepository) RemoveJobTagByRequest(user *schema.User, job int64, tagT // Removes a tag from db by tag info func (r *JobRepository) RemoveTagByRequest(tagType string, tagName string, tagScope string) error { // Get Tag ID to delete - tagID, err := r.loadTagIDByInfo(tagName, tagType, tagScope) - if err != nil { - log.Warn("Error while finding tagId with: %s, %s, %s", tagName, tagType, tagScope) + tagID, exists := r.TagId(tagType, tagName, tagScope) + if !exists { + log.Warn("Tag does not exist (name, type, scope): %s, %s, %s", tagName, tagType, tagScope) + return fmt.Errorf("Tag does not exist (name, type, scope): %s, %s, %s", tagName, tagType, tagScope) + } + + // Handle Delete JobTagTable + qJobTag := sq.Delete("jobtag").Where("jobtag.tag_id = ?", tagID) + + if _, err := qJobTag.RunWith(r.stmtCache).Exec(); err != nil { + s, _, _ := qJobTag.ToSql() + log.Errorf("Error removing tag from table 'jobTag' with %s: %v", s, err) return err } + // Handle Delete TagTable + qTag := sq.Delete("tag").Where("tag.id = ?", tagID) + + if _, err := qTag.RunWith(r.stmtCache).Exec(); err != nil { + s, _, _ := qTag.ToSql() + log.Errorf("Error removing tag from table 'tag' with %s: %v", s, err) + return err + } + + return nil +} + +// Removes a tag from db by tag info +func (r *JobRepository) RemoveTagById(tagID int64) error { // Handle Delete JobTagTable qJobTag := sq.Delete("jobtag").Where("jobtag.tag_id = ?", tagID) @@ -279,6 +302,16 @@ func (r *JobRepository) TagId(tagType string, tagName string, tagScope string) ( return } +// TagInfo returns the database infos of the tag with the specified id. +func (r *JobRepository) TagInfo(tagId int64) (tagType string, tagName string, tagScope string, exists bool) { + exists = true + if err := sq.Select("tag.tag_type", "tag.tag_name", "tag.tag_scope").From("tag").Where("tag.id = ?", tagId). + RunWith(r.stmtCache).QueryRow().Scan(&tagType, &tagName, &tagScope); err != nil { + exists = false + } + return +} + // GetTags returns a list of all scoped tags if job is nil or of the tags that the job with that database ID has. func (r *JobRepository) GetTags(user *schema.User, job *int64) ([]*schema.Tag, error) { q := sq.Select("id", "tag_type", "tag_name", "tag_scope").From("tag") @@ -395,29 +428,3 @@ func (r *JobRepository) checkScopeAuth(user *schema.User, operation string, scop return false, fmt.Errorf("error while checking tag operation auth: no user in context") } } - -func (r *JobRepository) loadTagIDByInfo(tagType string, tagName string, tagScope string) (tagID int64, err error) { - // Get Tag ID to delete - getq := sq.Select("id").From("tag"). - Where("tag_type = ?", tagType). - Where("tag_name = ?", tagName). - Where("tag_scope = ?", tagScope) - - rows, err := getq.RunWith(r.stmtCache).Query() - if err != nil { - s, _, _ := getq.ToSql() - log.Errorf("Error get tags for delete with %s: %v", s, err) - return 0, err - } - - dbTags := make([]*schema.Tag, 0) - for rows.Next() { - dbTag := &schema.Tag{} - if err := rows.Scan(&dbTag.ID); err != nil { - log.Warn("Error while scanning rows") - return 0, err - } - } - - return dbTags[0].ID, nil -} diff --git a/pkg/schema/user.go b/pkg/schema/user.go index c004254..9b62cfa 100644 --- a/pkg/schema/user.go +++ b/pkg/schema/user.go @@ -85,6 +85,7 @@ func IsValidRole(role string) bool { return getRoleEnum(role) != RoleError } +// Check if User has SPECIFIED role AND role is VALID func (u *User) HasValidRole(role string) (hasRole bool, isValid bool) { if IsValidRole(role) { for _, r := range u.Roles { @@ -97,6 +98,7 @@ func (u *User) HasValidRole(role string) (hasRole bool, isValid bool) { return false, false } +// Check if User has SPECIFIED role func (u *User) HasRole(role Role) bool { for _, r := range u.Roles { if r == GetRoleString(role) { @@ -106,7 +108,7 @@ func (u *User) HasRole(role Role) bool { return false } -// Role-Arrays are short: performance not impacted by nested loop +// Check if User has ANY of the listed roles func (u *User) HasAnyRole(queryroles []Role) bool { for _, ur := range u.Roles { for _, qr := range queryroles { @@ -118,7 +120,7 @@ func (u *User) HasAnyRole(queryroles []Role) bool { return false } -// Role-Arrays are short: performance not impacted by nested loop +// Check if User has ALL of the listed roles func (u *User) HasAllRoles(queryroles []Role) bool { target := len(queryroles) matches := 0 @@ -138,7 +140,7 @@ func (u *User) HasAllRoles(queryroles []Role) bool { } } -// Role-Arrays are short: performance not impacted by nested loop +// Check if User has NONE of the listed roles func (u *User) HasNotRoles(queryroles []Role) bool { matches := 0 for _, ur := range u.Roles { diff --git a/web/frontend/src/Tags.root.svelte b/web/frontend/src/Tags.root.svelte index 52288c9..dc156e3 100644 --- a/web/frontend/src/Tags.root.svelte +++ b/web/frontend/src/Tags.root.svelte @@ -37,13 +37,8 @@ return mutationStore({ client: client, query: gql` - mutation ($job: ID!, $tagIds: [ID!]!) { - removeTag(tagIds: $tagIds) { - id - type - name - scope - } + mutation ($tagIds: [ID!]!) { + removeTagFromList(tagIds: $tagIds) } `, variables: { tagIds }, @@ -55,7 +50,13 @@ removeTagMutation({tagIds: [tag.id] }).subscribe( (res) => { if (res.fetching === false && !res.error) { - tagmap = res.data.removeTag; + // console.log('Removed:', res.data.removeTagFromList) + // console.log('Targets:', tagType, tagmap[tagType]) + // console.log('Filter:', tagmap[tagType].filter((t) => !res.data.removeTagFromList.includes(t.id))) + tagmap[tagType] = tagmap[tagType].filter((t) => !res.data.removeTagFromList.includes(t.id)); + if (tagmap[tagType].length === 0) { + delete tagmap[tagType] + } pendingChange = "none"; } else if (res.fetching === false && res.error) { throw res.error; @@ -63,9 +64,6 @@ }, ); } - - $: console.log(username, isAdmin) - $: console.log(pendingChange, tagmap)
From 1b3a12a4dcf1fcff6ea19680abb9f74d61062ce2 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 23 Apr 2025 15:01:12 +0200 Subject: [PATCH 55/82] feat: add remove functionality to tag view, add confirm alert --- web/frontend/src/Tags.root.svelte | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/web/frontend/src/Tags.root.svelte b/web/frontend/src/Tags.root.svelte index dc156e3..441134a 100644 --- a/web/frontend/src/Tags.root.svelte +++ b/web/frontend/src/Tags.root.svelte @@ -46,23 +46,22 @@ }; function removeTag(tag, tagType) { - pendingChange = tagType; - removeTagMutation({tagIds: [tag.id] }).subscribe( - (res) => { - if (res.fetching === false && !res.error) { - // console.log('Removed:', res.data.removeTagFromList) - // console.log('Targets:', tagType, tagmap[tagType]) - // console.log('Filter:', tagmap[tagType].filter((t) => !res.data.removeTagFromList.includes(t.id))) - tagmap[tagType] = tagmap[tagType].filter((t) => !res.data.removeTagFromList.includes(t.id)); - if (tagmap[tagType].length === 0) { - delete tagmap[tagType] + if (confirm("Are you sure you want to completely remove this tag?\n\n" + tagType + ':' + tag.name)) { + pendingChange = tagType; + removeTagMutation({tagIds: [tag.id] }).subscribe( + (res) => { + if (res.fetching === false && !res.error) { + tagmap[tagType] = tagmap[tagType].filter((t) => !res.data.removeTagFromList.includes(t.id)); + if (tagmap[tagType].length === 0) { + delete tagmap[tagType] + } + pendingChange = "none"; + } else if (res.fetching === false && res.error) { + throw res.error; } - pendingChange = "none"; - } else if (res.fetching === false && res.error) { - throw res.error; - } - }, - ); + }, + ); + } } From 48fa75386c4f1a6c145f34a7a197e7c2a470f88a Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 23 Apr 2025 16:12:56 +0200 Subject: [PATCH 56/82] feat: add tag removal api endpoints --- internal/api/rest.go | 14 ++++---------- internal/graph/schema.resolvers.go | 14 +++++++------- internal/repository/tags.go | 4 ++-- 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/internal/api/rest.go b/internal/api/rest.go index 89bdd5e..78cf276 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -78,12 +78,14 @@ func (api *RestApi) MountApiRoutes(r *mux.Router) { r.HandleFunc("/jobs/{id}", api.getJobById).Methods(http.MethodPost) r.HandleFunc("/jobs/{id}", api.getCompleteJobById).Methods(http.MethodGet) r.HandleFunc("/jobs/tag_job/{id}", api.tagJob).Methods(http.MethodPost, http.MethodPatch) + r.HandleFunc("/jobs/tag_job/{id}", api.removeTagJob).Methods(http.MethodDelete) r.HandleFunc("/jobs/edit_meta/{id}", api.editMeta).Methods(http.MethodPost, http.MethodPatch) r.HandleFunc("/jobs/metrics/{id}", api.getJobMetrics).Methods(http.MethodGet) r.HandleFunc("/jobs/delete_job/", api.deleteJobByRequest).Methods(http.MethodDelete) r.HandleFunc("/jobs/delete_job/{id}", api.deleteJobById).Methods(http.MethodDelete) r.HandleFunc("/jobs/delete_job_before/{ts}", api.deleteJobBefore).Methods(http.MethodDelete) + r.HandleFunc("/tags/", api.removeTags).Methods(http.MethodDelete) r.HandleFunc("/clusters/", api.getClusters).Methods(http.MethodGet) if api.MachineStateDir != "" { @@ -805,14 +807,6 @@ func (api *RestApi) removeTagJob(rw http.ResponseWriter, r *http.Request) { return } - // remainingTags := job.Tags[:0] - // for _, tag := range job.Tags { - // if tag.Type != rtag.Type && - // tag.Name != rtag.Name && - // tag.Scope != rtag.Scope { - // remainingTags = append(remainingTags, tag) - // } - // } job.Tags = remainingTags } @@ -836,7 +830,7 @@ func (api *RestApi) removeTagJob(rw http.ResponseWriter, r *http.Request) { // @failure 404 {object} api.ErrorResponse "Job or tag does not exist" // @failure 500 {object} api.ErrorResponse "Internal Server Error" // @security ApiKeyAuth -// @router /jobs/tag_job/ [delete] +// @router /tags/ [delete] func (api *RestApi) removeTags(rw http.ResponseWriter, r *http.Request) { var req TagJobApiRequest if err := decode(r.Body, &req); err != nil { @@ -863,7 +857,7 @@ func (api *RestApi) removeTags(rw http.ResponseWriter, r *http.Request) { } rw.WriteHeader(http.StatusOK) - rw.Write([]byte(fmt.Sprintf("Deleted Tags from DB: %d of %d", currentCount, targetCount))) + rw.Write([]byte(fmt.Sprintf("Deleted Tags from DB: %d successfull of %d requested\n", currentCount, targetCount))) } // startJob godoc diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index 46f485b..10e1b55 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -142,7 +142,7 @@ func (r *mutationResolver) CreateTag(ctx context.Context, typeArg string, name s } return &schema.Tag{ID: id, Type: typeArg, Name: name, Scope: scope}, nil } else { - log.Warn("Not authorized to create tag with scope: %s", scope) + log.Warnf("Not authorized to create tag with scope: %s", scope) return nil, fmt.Errorf("Not authorized to create tag with scope: %s", scope) } } @@ -178,7 +178,7 @@ func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds // Test Exists _, _, tscope, exists := r.Repo.TagInfo(tid) if !exists { - log.Warn("Tag does not exist (ID): %d", tid) + log.Warnf("Tag does not exist (ID): %d", tid) return nil, fmt.Errorf("Tag does not exist (ID): %d", tid) } @@ -192,7 +192,7 @@ func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds return nil, err } } else { - log.Warn("Not authorized to add tag: %d", tid) + log.Warnf("Not authorized to add tag: %d", tid) return nil, fmt.Errorf("Not authorized to add tag: %d", tid) } } @@ -225,7 +225,7 @@ func (r *mutationResolver) RemoveTagsFromJob(ctx context.Context, job string, ta // Test Exists _, _, tscope, exists := r.Repo.TagInfo(tid) if !exists { - log.Warn("Tag does not exist (ID): %d", tid) + log.Warnf("Tag does not exist (ID): %d", tid) return nil, fmt.Errorf("Tag does not exist (ID): %d", tid) } @@ -239,7 +239,7 @@ func (r *mutationResolver) RemoveTagsFromJob(ctx context.Context, job string, ta return nil, err } } else { - log.Warn("Not authorized to remove tag: %d", tid) + log.Warnf("Not authorized to remove tag: %d", tid) return nil, fmt.Errorf("Not authorized to remove tag: %d", tid) } @@ -268,7 +268,7 @@ func (r *mutationResolver) RemoveTagFromList(ctx context.Context, tagIds []strin // Test Exists _, _, tscope, exists := r.Repo.TagInfo(tid) if !exists { - log.Warn("Tag does not exist (ID): %d", tid) + log.Warnf("Tag does not exist (ID): %d", tid) return nil, fmt.Errorf("Tag does not exist (ID): %d", tid) } @@ -282,7 +282,7 @@ func (r *mutationResolver) RemoveTagFromList(ctx context.Context, tagIds []strin tags = append(tags, int(tid)) } } else { - log.Warn("Not authorized to remove tag: %d", tid) + log.Warnf("Not authorized to remove tag: %d", tid) return nil, fmt.Errorf("Not authorized to remove tag: %d", tid) } } diff --git a/internal/repository/tags.go b/internal/repository/tags.go index 5712c94..db44dbc 100644 --- a/internal/repository/tags.go +++ b/internal/repository/tags.go @@ -81,7 +81,7 @@ func (r *JobRepository) RemoveJobTagByRequest(user *schema.User, job int64, tagT // Get Tag ID to delete tagID, exists := r.TagId(tagType, tagName, tagScope) if !exists { - log.Warn("Tag does not exist (name, type, scope): %s, %s, %s", tagName, tagType, tagScope) + log.Warnf("Tag does not exist (name, type, scope): %s, %s, %s", tagName, tagType, tagScope) return nil, fmt.Errorf("Tag does not exist (name, type, scope): %s, %s, %s", tagName, tagType, tagScope) } @@ -121,7 +121,7 @@ func (r *JobRepository) RemoveTagByRequest(tagType string, tagName string, tagSc // Get Tag ID to delete tagID, exists := r.TagId(tagType, tagName, tagScope) if !exists { - log.Warn("Tag does not exist (name, type, scope): %s, %s, %s", tagName, tagType, tagScope) + log.Warnf("Tag does not exist (name, type, scope): %s, %s, %s", tagName, tagType, tagScope) return fmt.Errorf("Tag does not exist (name, type, scope): %s, %s, %s", tagName, tagType, tagScope) } From e3653daea3a792bf50a7d65381b1d511d83d8966 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 23 Apr 2025 17:59:26 +0200 Subject: [PATCH 57/82] reduce code in tag svelte view --- internal/repository/tags.go | 4 +- web/frontend/src/Tags.root.svelte | 66 +++++++++---------------------- 2 files changed, 20 insertions(+), 50 deletions(-) diff --git a/internal/repository/tags.go b/internal/repository/tags.go index db44dbc..544163e 100644 --- a/internal/repository/tags.go +++ b/internal/repository/tags.go @@ -45,7 +45,7 @@ func (r *JobRepository) AddTag(user *schema.User, job int64, tag int64) ([]*sche return tags, archive.UpdateTags(j, archiveTags) } -// Removes a tag from a job by its ID +// Removes a tag from a job by tag id func (r *JobRepository) RemoveTag(user *schema.User, job, tag int64) ([]*schema.Tag, error) { j, err := r.FindByIdWithUser(user, job) if err != nil { @@ -146,7 +146,7 @@ func (r *JobRepository) RemoveTagByRequest(tagType string, tagName string, tagSc return nil } -// Removes a tag from db by tag info +// Removes a tag from db by tag id func (r *JobRepository) RemoveTagById(tagID int64) error { // Handle Delete JobTagTable qJobTag := sq.Delete("jobtag").Where("jobtag.tag_id = ?", tagID) diff --git a/web/frontend/src/Tags.root.svelte b/web/frontend/src/Tags.root.svelte index 441134a..03311b4 100644 --- a/web/frontend/src/Tags.root.svelte +++ b/web/frontend/src/Tags.root.svelte @@ -80,58 +80,28 @@
{#each tagList as tag (tag.id)} - {#if tag.scope == "global"} - - - {#if isAdmin} - - {/if} - - {:else if tag.scope == "admin"} - - - {#if isAdmin} - - {/if} - - {:else} - - - {#if tag.scope == username} - {/if} - - {/if} + + {#if (isAdmin && (tag.scope == "admin" || tag.scope == "global")) || tag.scope == username } + + {/if} + {/each}
{/each} From 94a39fc61f44a021ebfe0e4a288caf5e2fb0ed36 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 24 Apr 2025 10:53:55 +0200 Subject: [PATCH 58/82] Readd tag endpoints --- internal/api/rest.go | 111 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/internal/api/rest.go b/internal/api/rest.go index 0fa4611..7029d9d 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -82,12 +82,15 @@ func (api *RestApi) MountApiRoutes(r *mux.Router) { r.HandleFunc("/jobs/{id}", api.getJobById).Methods(http.MethodPost) r.HandleFunc("/jobs/{id}", api.getCompleteJobById).Methods(http.MethodGet) r.HandleFunc("/jobs/tag_job/{id}", api.tagJob).Methods(http.MethodPost, http.MethodPatch) + r.HandleFunc("/jobs/tag_job/{id}", api.removeTagJob).Methods(http.MethodDelete) r.HandleFunc("/jobs/edit_meta/{id}", api.editMeta).Methods(http.MethodPost, http.MethodPatch) r.HandleFunc("/jobs/metrics/{id}", api.getJobMetrics).Methods(http.MethodGet) r.HandleFunc("/jobs/delete_job/", api.deleteJobByRequest).Methods(http.MethodDelete) r.HandleFunc("/jobs/delete_job/{id}", api.deleteJobById).Methods(http.MethodDelete) r.HandleFunc("/jobs/delete_job_before/{ts}", api.deleteJobBefore).Methods(http.MethodDelete) + r.HandleFunc("/tags/", api.removeTags).Methods(http.MethodDelete) + if api.MachineStateDir != "" { r.HandleFunc("/machine_state/{cluster}/{host}", api.getMachineState).Methods(http.MethodGet) r.HandleFunc("/machine_state/{cluster}/{host}", api.putMachineState).Methods(http.MethodPut, http.MethodPost) @@ -713,6 +716,114 @@ func (api *RestApi) tagJob(rw http.ResponseWriter, r *http.Request) { json.NewEncoder(rw).Encode(job) } +// removeTagJob godoc +// @summary Removes one or more tags from a job +// @tags Job add and modify +// @description Removes tag(s) from a job specified by DB ID. Name and Type of Tag(s) must match. +// @description Tag Scope is required for matching, options: "global", "admin". Private tags can not be deleted via API. +// @description If tagged job is already finished: Tag will be removed from respective archive files. +// @accept json +// @produce json +// @param id path int true "Job Database ID" +// @param request body api.TagJobApiRequest true "Array of tag-objects to remove" +// @success 200 {object} schema.Job "Updated job resource" +// @failure 400 {object} api.ErrorResponse "Bad Request" +// @failure 401 {object} api.ErrorResponse "Unauthorized" +// @failure 404 {object} api.ErrorResponse "Job or tag does not exist" +// @failure 500 {object} api.ErrorResponse "Internal Server Error" +// @security ApiKeyAuth +// @router /jobs/tag_job/{id} [delete] +func (api *RestApi) removeTagJob(rw http.ResponseWriter, r *http.Request) { + id, err := strconv.ParseInt(mux.Vars(r)["id"], 10, 64) + if err != nil { + http.Error(rw, err.Error(), http.StatusBadRequest) + return + } + + job, err := api.JobRepository.FindById(r.Context(), id) + if err != nil { + http.Error(rw, err.Error(), http.StatusNotFound) + return + } + + job.Tags, err = api.JobRepository.GetTags(repository.GetUserFromContext(r.Context()), &job.ID) + if err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) + return + } + + var req TagJobApiRequest + if err := decode(r.Body, &req); err != nil { + http.Error(rw, err.Error(), http.StatusBadRequest) + return + } + + for _, rtag := range req { + // Only Global and Admin Tags + if rtag.Scope != "global" && rtag.Scope != "admin" { + log.Warnf("Cannot delete private tag for job %d: Skip", job.JobID) + continue + } + + remainingTags, err := api.JobRepository.RemoveJobTagByRequest(repository.GetUserFromContext(r.Context()), job.ID, rtag.Type, rtag.Name, rtag.Scope) + if err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) + return + } + + job.Tags = remainingTags + } + + rw.Header().Add("Content-Type", "application/json") + rw.WriteHeader(http.StatusOK) + json.NewEncoder(rw).Encode(job) +} + +// removeTags godoc +// @summary Removes all tags and job-relations for type:name tuple +// @tags Tag remove +// @description Removes tags by type and name. Name and Type of Tag(s) must match. +// @description Tag Scope is required for matching, options: "global", "admin". Private tags can not be deleted via API. +// @description Tag wills be removed from respective archive files. +// @accept json +// @produce plain +// @param request body api.TagJobApiRequest true "Array of tag-objects to remove" +// @success 200 {string} string "Success Response" +// @failure 400 {object} api.ErrorResponse "Bad Request" +// @failure 401 {object} api.ErrorResponse "Unauthorized" +// @failure 404 {object} api.ErrorResponse "Job or tag does not exist" +// @failure 500 {object} api.ErrorResponse "Internal Server Error" +// @security ApiKeyAuth +// @router /tags/ [delete] +func (api *RestApi) removeTags(rw http.ResponseWriter, r *http.Request) { + var req TagJobApiRequest + if err := decode(r.Body, &req); err != nil { + http.Error(rw, err.Error(), http.StatusBadRequest) + return + } + + targetCount := len(req) + currentCount := 0 + for _, rtag := range req { + // Only Global and Admin Tags + if rtag.Scope != "global" && rtag.Scope != "admin" { + log.Warn("Cannot delete private tag: Skip") + continue + } + + err := api.JobRepository.RemoveTagByRequest(rtag.Type, rtag.Name, rtag.Scope) + if err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) + return + } else { + currentCount++ + } + } + + rw.WriteHeader(http.StatusOK) + rw.Write([]byte(fmt.Sprintf("Deleted Tags from DB: %d successfull of %d requested\n", currentCount, targetCount))) +} + // startJob godoc // @summary Adds a new job as "running" // @tags Job add and modify From 570eba37947636bcc383ee8aa9213e54c88f9585 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 24 Apr 2025 11:01:13 +0200 Subject: [PATCH 59/82] Cleanup Swagger docs --- internal/api/rest.go | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/internal/api/rest.go b/internal/api/rest.go index 7029d9d..8a21e68 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -1228,21 +1228,6 @@ func (api *RestApi) createUser(rw http.ResponseWriter, r *http.Request) { fmt.Fprintf(rw, "User %v successfully created!\n", username) } -// deleteUser godoc -// @summary Deletes a user -// @tags User -// @description User defined by username in form data will be deleted from database. -// @accept mpfd -// @produce plain -// @param username formData string true "User ID to delete" -// @success 200 "User deleted successfully" -// @failure 400 {string} string "Bad Request" -// @failure 401 {string} string "Unauthorized" -// @failure 403 {string} string "Forbidden" -// @failure 422 {string} string "Unprocessable Entity: deleting user failed" -// @failure 500 {string} string "Internal Server Error" -// @security ApiKeyAuth -// @router /users/ [delete] func (api *RestApi) deleteUser(rw http.ResponseWriter, r *http.Request) { // SecuredCheck() only worked with TokenAuth: Removed @@ -1291,26 +1276,6 @@ func (api *RestApi) getUsers(rw http.ResponseWriter, r *http.Request) { json.NewEncoder(rw).Encode(users) } -// updateUser godoc -// @summary Updates an existing user -// @tags User -// @description Modifies user defined by username (id) in one of four possible ways. -// @description If more than one formValue is set then only the highest priority field is used. -// @accept mpfd -// @produce plain -// @param id path string true "Database ID of User" -// @param add-role formData string false "Priority 1: Role to add" Enums(admin, support, manager, user, api) -// @param remove-role formData string false "Priority 2: Role to remove" Enums(admin, support, manager, user, api) -// @param add-project formData string false "Priority 3: Project to add" -// @param remove-project formData string false "Priority 4: Project to remove" -// @success 200 {string} string "Success Response Message" -// @failure 400 {string} string "Bad Request" -// @failure 401 {string} string "Unauthorized" -// @failure 403 {string} string "Forbidden" -// @failure 422 {string} string "Unprocessable Entity: The user could not be updated" -// @failure 500 {string} string "Internal Server Error" -// @security ApiKeyAuth -// @router /user/{id} [post] func (api *RestApi) updateUser(rw http.ResponseWriter, r *http.Request) { // SecuredCheck() only worked with TokenAuth: Removed From 65df27154c3f26d1748c2ca160af43f6abbc283d Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 24 Apr 2025 11:14:51 +0200 Subject: [PATCH 60/82] Cleanup and regenerate Swagger docs --- api/swagger.json | 396 +++++++++++-------------------------------- api/swagger.yaml | 279 ++++++++---------------------- internal/api/docs.go | 396 +++++++++++-------------------------------- internal/api/rest.go | 38 +---- 4 files changed, 269 insertions(+), 840 deletions(-) diff --git a/api/swagger.json b/api/swagger.json index 683b520..c05ec77 100644 --- a/api/swagger.json +++ b/api/swagger.json @@ -826,185 +826,14 @@ } } }, - "/config/notice/": { - "post": { - "security": [ - { - "ApiKeyAuth": [] - } - ], - "description": "Modifies the content of notice.txt, shown as notice box on the homepage.\nIf more than one formValue is set then only the highest priority field is used.\nOnly accessible from IPs registered with apiAllowedIPs configuration option.", - "consumes": [ - "multipart/form-data" - ], - "produces": [ - "text/plain" - ], - "tags": [ - "User" - ], - "summary": "Updates or empties the notice box content", - "parameters": [ - { - "type": "string", - "description": "Priority 1: New content to display", - "name": "new-content", - "in": "formData" - } - ], - "responses": { - "200": { - "description": "Success Response Message", - "schema": { - "type": "string" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "type": "string" - } - }, - "401": { - "description": "Unauthorized", - "schema": { - "type": "string" - } - }, - "403": { - "description": "Forbidden", - "schema": { - "type": "string" - } - }, - "422": { - "description": "Unprocessable Entity: The user could not be updated", - "schema": { - "type": "string" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "type": "string" - } - } - } - } - }, - "/config/user/{id}": { - "post": { - "security": [ - { - "ApiKeyAuth": [] - } - ], - "description": "Modifies user defined by username (id) in one of four possible ways.\nIf more than one formValue is set then only the highest priority field is used.\nOnly accessible from IPs registered with apiAllowedIPs configuration option.", - "consumes": [ - "multipart/form-data" - ], - "produces": [ - "text/plain" - ], - "tags": [ - "User" - ], - "summary": "Updates an existing user", - "parameters": [ - { - "type": "string", - "description": "Database ID of User", - "name": "id", - "in": "path", - "required": true - }, - { - "enum": [ - "admin", - "support", - "manager", - "user", - "api" - ], - "type": "string", - "description": "Priority 1: Role to add", - "name": "add-role", - "in": "formData" - }, - { - "enum": [ - "admin", - "support", - "manager", - "user", - "api" - ], - "type": "string", - "description": "Priority 2: Role to remove", - "name": "remove-role", - "in": "formData" - }, - { - "type": "string", - "description": "Priority 3: Project to add", - "name": "add-project", - "in": "formData" - }, - { - "type": "string", - "description": "Priority 4: Project to remove", - "name": "remove-project", - "in": "formData" - } - ], - "responses": { - "200": { - "description": "Success Response Message", - "schema": { - "type": "string" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "type": "string" - } - }, - "401": { - "description": "Unauthorized", - "schema": { - "type": "string" - } - }, - "403": { - "description": "Forbidden", - "schema": { - "type": "string" - } - }, - "422": { - "description": "Unprocessable Entity: The user could not be updated", - "schema": { - "type": "string" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "type": "string" - } - } - } - } - }, - "/config/users/": { + "/api/users/": { "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Returns a JSON-encoded list of users.\nRequired query-parameter defines if all users or only users with additional special roles are returned.\nOnly accessible from IPs registered with apiAllowedIPs configuration option.", + "description": "Returns a JSON-encoded list of users.\nRequired query-parameter defines if all users or only users with additional special roles are returned.", "produces": [ "application/json" ], @@ -1056,70 +885,111 @@ } } } - }, - "post": { + } + }, + "/jobs/tag_job/{id}": { + "delete": { "security": [ { "ApiKeyAuth": [] } ], - "description": "User specified in form data will be saved to database.\nOnly accessible from IPs registered with apiAllowedIPs configuration option.", + "description": "Removes tag(s) from a job specified by DB ID. Name and Type of Tag(s) must match.\nTag Scope is required for matching, options: \"global\", \"admin\". Private tags can not be deleted via API.\nIf tagged job is already finished: Tag will be removed from respective archive files.", "consumes": [ - "multipart/form-data" + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Job add and modify" + ], + "summary": "Removes one or more tags from a job", + "parameters": [ + { + "type": "integer", + "description": "Job Database ID", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "Array of tag-objects to remove", + "name": "request", + "in": "body", + "required": true, + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/api.ApiTag" + } + } + } + ], + "responses": { + "200": { + "description": "Updated job resource", + "schema": { + "$ref": "#/definitions/schema.Job" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "404": { + "description": "Job or tag does not exist", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + } + } + } + }, + "/tags/": { + "delete": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Removes tags by type and name. Name and Type of Tag(s) must match.\nTag Scope is required for matching, options: \"global\", \"admin\". Private tags can not be deleted via API.\nTag wills be removed from respective archive files.", + "consumes": [ + "application/json" ], "produces": [ "text/plain" ], "tags": [ - "User" + "Tag remove" ], - "summary": "Adds a new user", + "summary": "Removes all tags and job-relations for type:name tuple", "parameters": [ { - "type": "string", - "description": "Unique user ID", - "name": "username", - "in": "formData", - "required": true - }, - { - "type": "string", - "description": "User password", - "name": "password", - "in": "formData", - "required": true - }, - { - "enum": [ - "admin", - "support", - "manager", - "user", - "api" - ], - "type": "string", - "description": "User role", - "name": "role", - "in": "formData", - "required": true - }, - { - "type": "string", - "description": "Managed project, required for new manager role user", - "name": "project", - "in": "formData" - }, - { - "type": "string", - "description": "Users name", - "name": "name", - "in": "formData" - }, - { - "type": "string", - "description": "Users email", - "name": "email", - "in": "formData" + "description": "Array of tag-objects to remove", + "name": "request", + "in": "body", + "required": true, + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/api.ApiTag" + } + } } ], "responses": { @@ -1132,93 +1002,25 @@ "400": { "description": "Bad Request", "schema": { - "type": "string" + "$ref": "#/definitions/api.ErrorResponse" } }, "401": { "description": "Unauthorized", "schema": { - "type": "string" + "$ref": "#/definitions/api.ErrorResponse" } }, - "403": { - "description": "Forbidden", + "404": { + "description": "Job or tag does not exist", "schema": { - "type": "string" - } - }, - "422": { - "description": "Unprocessable Entity: creating user failed", - "schema": { - "type": "string" + "$ref": "#/definitions/api.ErrorResponse" } }, "500": { "description": "Internal Server Error", "schema": { - "type": "string" - } - } - } - }, - "delete": { - "security": [ - { - "ApiKeyAuth": [] - } - ], - "description": "User defined by username in form data will be deleted from database.\nOnly accessible from IPs registered with apiAllowedIPs configuration option.", - "consumes": [ - "multipart/form-data" - ], - "produces": [ - "text/plain" - ], - "tags": [ - "User" - ], - "summary": "Deletes a user", - "parameters": [ - { - "type": "string", - "description": "User ID to delete", - "name": "username", - "in": "formData", - "required": true - } - ], - "responses": { - "200": { - "description": "User deleted successfully" - }, - "400": { - "description": "Bad Request", - "schema": { - "type": "string" - } - }, - "401": { - "description": "Unauthorized", - "schema": { - "type": "string" - } - }, - "403": { - "description": "Forbidden", - "schema": { - "type": "string" - } - }, - "422": { - "description": "Unprocessable Entity: deleting user failed", - "schema": { - "type": "string" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "type": "string" + "$ref": "#/definitions/api.ErrorResponse" } } } diff --git a/api/swagger.yaml b/api/swagger.yaml index 35ec6c4..26210be 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -1217,173 +1217,11 @@ paths: summary: Adds one or more tags to a job tags: - Job add and modify - /config/notice/: - post: - consumes: - - multipart/form-data - description: |- - Modifies the content of notice.txt, shown as notice box on the homepage. - If more than one formValue is set then only the highest priority field is used. - Only accessible from IPs registered with apiAllowedIPs configuration option. - parameters: - - description: 'Priority 1: New content to display' - in: formData - name: new-content - type: string - produces: - - text/plain - responses: - "200": - description: Success Response Message - schema: - type: string - "400": - description: Bad Request - schema: - type: string - "401": - description: Unauthorized - schema: - type: string - "403": - description: Forbidden - schema: - type: string - "422": - description: 'Unprocessable Entity: The user could not be updated' - schema: - type: string - "500": - description: Internal Server Error - schema: - type: string - security: - - ApiKeyAuth: [] - summary: Updates or empties the notice box content - tags: - - User - /config/user/{id}: - post: - consumes: - - multipart/form-data - description: |- - Modifies user defined by username (id) in one of four possible ways. - If more than one formValue is set then only the highest priority field is used. - Only accessible from IPs registered with apiAllowedIPs configuration option. - parameters: - - description: Database ID of User - in: path - name: id - required: true - type: string - - description: 'Priority 1: Role to add' - enum: - - admin - - support - - manager - - user - - api - in: formData - name: add-role - type: string - - description: 'Priority 2: Role to remove' - enum: - - admin - - support - - manager - - user - - api - in: formData - name: remove-role - type: string - - description: 'Priority 3: Project to add' - in: formData - name: add-project - type: string - - description: 'Priority 4: Project to remove' - in: formData - name: remove-project - type: string - produces: - - text/plain - responses: - "200": - description: Success Response Message - schema: - type: string - "400": - description: Bad Request - schema: - type: string - "401": - description: Unauthorized - schema: - type: string - "403": - description: Forbidden - schema: - type: string - "422": - description: 'Unprocessable Entity: The user could not be updated' - schema: - type: string - "500": - description: Internal Server Error - schema: - type: string - security: - - ApiKeyAuth: [] - summary: Updates an existing user - tags: - - User - /config/users/: - delete: - consumes: - - multipart/form-data - description: |- - User defined by username in form data will be deleted from database. - Only accessible from IPs registered with apiAllowedIPs configuration option. - parameters: - - description: User ID to delete - in: formData - name: username - required: true - type: string - produces: - - text/plain - responses: - "200": - description: User deleted successfully - "400": - description: Bad Request - schema: - type: string - "401": - description: Unauthorized - schema: - type: string - "403": - description: Forbidden - schema: - type: string - "422": - description: 'Unprocessable Entity: deleting user failed' - schema: - type: string - "500": - description: Internal Server Error - schema: - type: string - security: - - ApiKeyAuth: [] - summary: Deletes a user - tags: - - User + /api/users/: get: description: |- Returns a JSON-encoded list of users. Required query-parameter defines if all users or only users with additional special roles are returned. - Only accessible from IPs registered with apiAllowedIPs configuration option. parameters: - description: If returned list should contain all users or only users with additional special roles @@ -1421,46 +1259,73 @@ paths: summary: Returns a list of users tags: - User - post: + /jobs/tag_job/{id}: + delete: consumes: - - multipart/form-data + - application/json description: |- - User specified in form data will be saved to database. - Only accessible from IPs registered with apiAllowedIPs configuration option. + Removes tag(s) from a job specified by DB ID. Name and Type of Tag(s) must match. + Tag Scope is required for matching, options: "global", "admin". Private tags can not be deleted via API. + If tagged job is already finished: Tag will be removed from respective archive files. parameters: - - description: Unique user ID - in: formData - name: username + - description: Job Database ID + in: path + name: id required: true - type: string - - description: User password - in: formData - name: password + type: integer + - description: Array of tag-objects to remove + in: body + name: request required: true - type: string - - description: User role - enum: - - admin - - support - - manager - - user - - api - in: formData - name: role + schema: + items: + $ref: '#/definitions/api.ApiTag' + type: array + produces: + - application/json + responses: + "200": + description: Updated job resource + schema: + $ref: '#/definitions/schema.Job' + "400": + description: Bad Request + schema: + $ref: '#/definitions/api.ErrorResponse' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/api.ErrorResponse' + "404": + description: Job or tag does not exist + schema: + $ref: '#/definitions/api.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/api.ErrorResponse' + security: + - ApiKeyAuth: [] + summary: Removes one or more tags from a job + tags: + - Job add and modify + /tags/: + delete: + consumes: + - application/json + description: |- + Removes tags by type and name. Name and Type of Tag(s) must match. + Tag Scope is required for matching, options: "global", "admin". Private tags can not be deleted via API. + Tag wills be removed from respective archive files. + parameters: + - description: Array of tag-objects to remove + in: body + name: request required: true - type: string - - description: Managed project, required for new manager role user - in: formData - name: project - type: string - - description: Users name - in: formData - name: name - type: string - - description: Users email - in: formData - name: email - type: string + schema: + items: + $ref: '#/definitions/api.ApiTag' + type: array produces: - text/plain responses: @@ -1471,28 +1336,24 @@ paths: "400": description: Bad Request schema: - type: string + $ref: '#/definitions/api.ErrorResponse' "401": description: Unauthorized schema: - type: string - "403": - description: Forbidden + $ref: '#/definitions/api.ErrorResponse' + "404": + description: Job or tag does not exist schema: - type: string - "422": - description: 'Unprocessable Entity: creating user failed' - schema: - type: string + $ref: '#/definitions/api.ErrorResponse' "500": description: Internal Server Error schema: - type: string + $ref: '#/definitions/api.ErrorResponse' security: - ApiKeyAuth: [] - summary: Adds a new user + summary: Removes all tags and job-relations for type:name tuple tags: - - User + - Tag remove securityDefinitions: ApiKeyAuth: in: header diff --git a/internal/api/docs.go b/internal/api/docs.go index 2408f85..c1cd391 100644 --- a/internal/api/docs.go +++ b/internal/api/docs.go @@ -833,185 +833,14 @@ const docTemplate = `{ } } }, - "/config/notice/": { - "post": { - "security": [ - { - "ApiKeyAuth": [] - } - ], - "description": "Modifies the content of notice.txt, shown as notice box on the homepage.\nIf more than one formValue is set then only the highest priority field is used.\nOnly accessible from IPs registered with apiAllowedIPs configuration option.", - "consumes": [ - "multipart/form-data" - ], - "produces": [ - "text/plain" - ], - "tags": [ - "User" - ], - "summary": "Updates or empties the notice box content", - "parameters": [ - { - "type": "string", - "description": "Priority 1: New content to display", - "name": "new-content", - "in": "formData" - } - ], - "responses": { - "200": { - "description": "Success Response Message", - "schema": { - "type": "string" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "type": "string" - } - }, - "401": { - "description": "Unauthorized", - "schema": { - "type": "string" - } - }, - "403": { - "description": "Forbidden", - "schema": { - "type": "string" - } - }, - "422": { - "description": "Unprocessable Entity: The user could not be updated", - "schema": { - "type": "string" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "type": "string" - } - } - } - } - }, - "/config/user/{id}": { - "post": { - "security": [ - { - "ApiKeyAuth": [] - } - ], - "description": "Modifies user defined by username (id) in one of four possible ways.\nIf more than one formValue is set then only the highest priority field is used.\nOnly accessible from IPs registered with apiAllowedIPs configuration option.", - "consumes": [ - "multipart/form-data" - ], - "produces": [ - "text/plain" - ], - "tags": [ - "User" - ], - "summary": "Updates an existing user", - "parameters": [ - { - "type": "string", - "description": "Database ID of User", - "name": "id", - "in": "path", - "required": true - }, - { - "enum": [ - "admin", - "support", - "manager", - "user", - "api" - ], - "type": "string", - "description": "Priority 1: Role to add", - "name": "add-role", - "in": "formData" - }, - { - "enum": [ - "admin", - "support", - "manager", - "user", - "api" - ], - "type": "string", - "description": "Priority 2: Role to remove", - "name": "remove-role", - "in": "formData" - }, - { - "type": "string", - "description": "Priority 3: Project to add", - "name": "add-project", - "in": "formData" - }, - { - "type": "string", - "description": "Priority 4: Project to remove", - "name": "remove-project", - "in": "formData" - } - ], - "responses": { - "200": { - "description": "Success Response Message", - "schema": { - "type": "string" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "type": "string" - } - }, - "401": { - "description": "Unauthorized", - "schema": { - "type": "string" - } - }, - "403": { - "description": "Forbidden", - "schema": { - "type": "string" - } - }, - "422": { - "description": "Unprocessable Entity: The user could not be updated", - "schema": { - "type": "string" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "type": "string" - } - } - } - } - }, - "/config/users/": { + "/api/users/": { "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Returns a JSON-encoded list of users.\nRequired query-parameter defines if all users or only users with additional special roles are returned.\nOnly accessible from IPs registered with apiAllowedIPs configuration option.", + "description": "Returns a JSON-encoded list of users.\nRequired query-parameter defines if all users or only users with additional special roles are returned.", "produces": [ "application/json" ], @@ -1063,70 +892,111 @@ const docTemplate = `{ } } } - }, - "post": { + } + }, + "/jobs/tag_job/{id}": { + "delete": { "security": [ { "ApiKeyAuth": [] } ], - "description": "User specified in form data will be saved to database.\nOnly accessible from IPs registered with apiAllowedIPs configuration option.", + "description": "Removes tag(s) from a job specified by DB ID. Name and Type of Tag(s) must match.\nTag Scope is required for matching, options: \"global\", \"admin\". Private tags can not be deleted via API.\nIf tagged job is already finished: Tag will be removed from respective archive files.", "consumes": [ - "multipart/form-data" + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Job add and modify" + ], + "summary": "Removes one or more tags from a job", + "parameters": [ + { + "type": "integer", + "description": "Job Database ID", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "Array of tag-objects to remove", + "name": "request", + "in": "body", + "required": true, + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/api.ApiTag" + } + } + } + ], + "responses": { + "200": { + "description": "Updated job resource", + "schema": { + "$ref": "#/definitions/schema.Job" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "404": { + "description": "Job or tag does not exist", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + } + } + } + }, + "/tags/": { + "delete": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Removes tags by type and name. Name and Type of Tag(s) must match.\nTag Scope is required for matching, options: \"global\", \"admin\". Private tags can not be deleted via API.\nTag wills be removed from respective archive files.", + "consumes": [ + "application/json" ], "produces": [ "text/plain" ], "tags": [ - "User" + "Tag remove" ], - "summary": "Adds a new user", + "summary": "Removes all tags and job-relations for type:name tuple", "parameters": [ { - "type": "string", - "description": "Unique user ID", - "name": "username", - "in": "formData", - "required": true - }, - { - "type": "string", - "description": "User password", - "name": "password", - "in": "formData", - "required": true - }, - { - "enum": [ - "admin", - "support", - "manager", - "user", - "api" - ], - "type": "string", - "description": "User role", - "name": "role", - "in": "formData", - "required": true - }, - { - "type": "string", - "description": "Managed project, required for new manager role user", - "name": "project", - "in": "formData" - }, - { - "type": "string", - "description": "Users name", - "name": "name", - "in": "formData" - }, - { - "type": "string", - "description": "Users email", - "name": "email", - "in": "formData" + "description": "Array of tag-objects to remove", + "name": "request", + "in": "body", + "required": true, + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/api.ApiTag" + } + } } ], "responses": { @@ -1139,93 +1009,25 @@ const docTemplate = `{ "400": { "description": "Bad Request", "schema": { - "type": "string" + "$ref": "#/definitions/api.ErrorResponse" } }, "401": { "description": "Unauthorized", "schema": { - "type": "string" + "$ref": "#/definitions/api.ErrorResponse" } }, - "403": { - "description": "Forbidden", + "404": { + "description": "Job or tag does not exist", "schema": { - "type": "string" - } - }, - "422": { - "description": "Unprocessable Entity: creating user failed", - "schema": { - "type": "string" + "$ref": "#/definitions/api.ErrorResponse" } }, "500": { "description": "Internal Server Error", "schema": { - "type": "string" - } - } - } - }, - "delete": { - "security": [ - { - "ApiKeyAuth": [] - } - ], - "description": "User defined by username in form data will be deleted from database.\nOnly accessible from IPs registered with apiAllowedIPs configuration option.", - "consumes": [ - "multipart/form-data" - ], - "produces": [ - "text/plain" - ], - "tags": [ - "User" - ], - "summary": "Deletes a user", - "parameters": [ - { - "type": "string", - "description": "User ID to delete", - "name": "username", - "in": "formData", - "required": true - } - ], - "responses": { - "200": { - "description": "User deleted successfully" - }, - "400": { - "description": "Bad Request", - "schema": { - "type": "string" - } - }, - "401": { - "description": "Unauthorized", - "schema": { - "type": "string" - } - }, - "403": { - "description": "Forbidden", - "schema": { - "type": "string" - } - }, - "422": { - "description": "Unprocessable Entity: deleting user failed", - "schema": { - "type": "string" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "type": "string" + "$ref": "#/definitions/api.ErrorResponse" } } } diff --git a/internal/api/rest.go b/internal/api/rest.go index 2b2a3bd..669768e 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -1163,26 +1163,6 @@ func (api *RestApi) getJobMetrics(rw http.ResponseWriter, r *http.Request) { }) } -// createUser godoc -// @summary Adds a new user -// @tags User -// @description User specified in form data will be saved to database. -// @accept mpfd -// @produce plain -// @param username formData string true "Unique user ID" -// @param password formData string true "User password" -// @param role formData string true "User role" Enums(admin, support, manager, user, api) -// @param project formData string false "Managed project, required for new manager role user" -// @param name formData string false "Users name" -// @param email formData string false "Users email" -// @success 200 {string} string "Success Response" -// @failure 400 {string} string "Bad Request" -// @failure 401 {string} string "Unauthorized" -// @failure 403 {string} string "Forbidden" -// @failure 422 {string} string "Unprocessable Entity: creating user failed" -// @failure 500 {string} string "Internal Server Error" -// @security ApiKeyAuth -// @router /config/users/ [post] func (api *RestApi) createUser(rw http.ResponseWriter, r *http.Request) { // SecuredCheck() only worked with TokenAuth: Removed @@ -1257,7 +1237,7 @@ func (api *RestApi) deleteUser(rw http.ResponseWriter, r *http.Request) { // @failure 403 {string} string "Forbidden" // @failure 500 {string} string "Internal Server Error" // @security ApiKeyAuth -// @router /config/users/ [get] +// @router /api/users/ [get] func (api *RestApi) getUsers(rw http.ResponseWriter, r *http.Request) { // SecuredCheck() only worked with TokenAuth: Removed @@ -1319,22 +1299,6 @@ func (api *RestApi) updateUser(rw http.ResponseWriter, r *http.Request) { } } -// editNotice godoc -// @summary Updates or empties the notice box content -// @tags User -// @description Modifies the content of notice.txt, shown as notice box on the homepage. -// @description If more than one formValue is set then only the highest priority field is used. -// @accept mpfd -// @produce plain -// @param new-content formData string false "Priority 1: New content to display" -// @success 200 {string} string "Success Response Message" -// @failure 400 {string} string "Bad Request" -// @failure 401 {string} string "Unauthorized" -// @failure 403 {string} string "Forbidden" -// @failure 422 {string} string "Unprocessable Entity: The user could not be updated" -// @failure 500 {string} string "Internal Server Error" -// @security ApiKeyAuth -// @router /notice/ [post] func (api *RestApi) editNotice(rw http.ResponseWriter, r *http.Request) { // SecuredCheck() only worked with TokenAuth: Removed From acaad69917acba109a6f544656271925f846c790 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 24 Apr 2025 11:42:34 +0200 Subject: [PATCH 61/82] Prepare Bugfix Release 1.4.4 --- Makefile | 2 +- ReleaseNotes.md | 18 ++---------------- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 0721fc4..5702ba1 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ TARGET = ./cc-backend VAR = ./var CFG = config.json .env FRONTEND = ./web/frontend -VERSION = 1.4.3 +VERSION = 1.4.4 GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development') CURRENT_TIME = $(shell date +"%Y-%m-%d:T%H:%M:%S") LD_FLAGS = '-s -X main.date=${CURRENT_TIME} -X main.version=${VERSION} -X main.commit=${GIT_HASH}' diff --git a/ReleaseNotes.md b/ReleaseNotes.md index beb8ee1..3e3939d 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -1,4 +1,4 @@ -# `cc-backend` version 1.4.3 +# `cc-backend` version 1.4.4 Supports job archive version 2 and database version 8. @@ -22,21 +22,7 @@ For release specific notes visit the [ClusterCockpit Documentation](https://clus ## New features -- Detailed Node List - - Adds new routes `/systems/list/$cluster` and `/systems/list/$cluster/$subcluster` - - Displays live, scoped metric data requested from the nodes indepenent of jobs -- Color Blind Mode - - Set on a per-user basis in options - - Applies to plot data, plot background color, statsseries colors, roofline timescale -- Job-View metric selection is now persisted based on the jobs subcluster. -Helpful for heterogeneous subcluster configurations. -- Histogram Bin Select in User-View - - Metric-Histograms: `10 Bins` now default, selectable options `20, 50, 100` - - Job-Duration-Histogram: `48h in 1h Bins` now default, selectable options: - - `60 minutes in 1 minute Bins` - - `12 hours in 10 minute Bins` - - `3 days in 6 hour Bins` - - `7 days in 12 hour Bins` +- Enable to delete tags from the web interface ## Known issues From aba75b3a1901be6b4117968cb22049ce0452ae6d Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 24 Apr 2025 12:57:37 +0200 Subject: [PATCH 62/82] Remove websocket sse GraphQL support --- cmd/cc-backend/server.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/cmd/cc-backend/server.go b/cmd/cc-backend/server.go index 3c19730..cbd85b7 100644 --- a/cmd/cc-backend/server.go +++ b/cmd/cc-backend/server.go @@ -32,7 +32,6 @@ import ( "github.com/ClusterCockpit/cc-backend/web" "github.com/gorilla/handlers" "github.com/gorilla/mux" - "github.com/gorilla/websocket" httpSwagger "github.com/swaggo/http-swagger" ) @@ -58,16 +57,16 @@ func serverInit() { graphQLServer := handler.New( generated.NewExecutableSchema(generated.Config{Resolvers: resolver})) - graphQLServer.AddTransport(transport.SSE{}) + // graphQLServer.AddTransport(transport.SSE{}) graphQLServer.AddTransport(transport.POST{}) - graphQLServer.AddTransport(transport.Websocket{ - KeepAlivePingInterval: 10 * time.Second, - Upgrader: websocket.Upgrader{ - CheckOrigin: func(r *http.Request) bool { - return true - }, - }, - }) + // graphQLServer.AddTransport(transport.Websocket{ + // KeepAlivePingInterval: 10 * time.Second, + // Upgrader: websocket.Upgrader{ + // CheckOrigin: func(r *http.Request) bool { + // return true + // }, + // }, + // }) if os.Getenv("DEBUG") != "1" { // Having this handler means that a error message is returned via GraphQL instead of the connection simply beeing closed. From 6ca14c55f2e05d39f538feb9c9484300cfb2702a Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 25 Apr 2025 18:09:21 +0200 Subject: [PATCH 63/82] fix: fix error in jobsMetricStatisticsHistogram calculation - also reduces overhead, simplifies query --- internal/repository/stats.go | 95 ++++++++++++++---------------------- 1 file changed, 36 insertions(+), 59 deletions(-) diff --git a/internal/repository/stats.go b/internal/repository/stats.go index ad518bd..410ba6c 100644 --- a/internal/repository/stats.go +++ b/internal/repository/stats.go @@ -674,57 +674,32 @@ func (r *JobRepository) jobsMetricStatisticsHistogram( } } - // log.Debugf("Metric %s, Peak %f, Unit %s, Aggregation %s", metric, peak, unit, aggreg) - // Make bins, see https://jereze.com/code/sql-histogram/ - + // log.Debugf("Metric %s, Peak %f, Unit %s", metric, peak, unit) + // Make bins, see https://jereze.com/code/sql-histogram/ (Modified here) start := time.Now() - jm := fmt.Sprintf(`json_extract(footprint, "$.%s")`, (metric + "_" + footprintStat)) - crossJoinQuery := sq.Select( - fmt.Sprintf(`max(%s) as max`, jm), - fmt.Sprintf(`min(%s) as min`, jm), - ).From("job").Where( - "JSON_VALID(footprint)", - ).Where( - fmt.Sprintf(`%s is not null`, jm), - ).Where( - fmt.Sprintf(`%s <= %f`, jm, peak), - ) - - crossJoinQuery, cjqerr := SecurityCheck(ctx, crossJoinQuery) - - if cjqerr != nil { - return nil, cjqerr - } - - for _, f := range filters { - crossJoinQuery = BuildWhereClause(f, crossJoinQuery) - } - - crossJoinQuerySql, crossJoinQueryArgs, sqlerr := crossJoinQuery.ToSql() - if sqlerr != nil { - return nil, sqlerr - } - - binQuery := fmt.Sprintf(`CAST( (case when %s = value.max - then value.max*0.999999999 else %s end - value.min) / (value.max - - value.min) * %v as INTEGER )`, jm, jm, *bins) + // Find Jobs' Value Bin Number: Divide Value by Peak, Multiply by RequestedBins, then CAST to INT: Gets Bin-Number of Job + binQuery := fmt.Sprintf(`CAST( + ((case when json_extract(footprint, "$.%s") = %f then %f*0.999999999 else json_extract(footprint, "$.%s") end) / %f) + * %v as INTEGER )`, + (metric + "_" + footprintStat), peak, peak, (metric + "_" + footprintStat), peak, *bins) mainQuery := sq.Select( fmt.Sprintf(`%s + 1 as bin`, binQuery), - fmt.Sprintf(`count(%s) as count`, jm), - fmt.Sprintf(`CAST(((value.max / %d) * (%v )) as INTEGER ) as min`, *bins, binQuery), - fmt.Sprintf(`CAST(((value.max / %d) * (%v + 1 )) as INTEGER ) as max`, *bins, binQuery), - ).From("job").CrossJoin( - fmt.Sprintf(`(%s) as value`, crossJoinQuerySql), crossJoinQueryArgs..., - ).Where(fmt.Sprintf(`%s is not null and %s <= %f`, jm, jm, peak)) + fmt.Sprintf(`count(*) as count`), + // For Debug: // fmt.Sprintf(`CAST((%f / %d) as INTEGER ) * %s as min`, peak, *bins, binQuery), + // For Debug: // fmt.Sprintf(`CAST((%f / %d) as INTEGER ) * (%s + 1) as max`, peak, *bins, binQuery), + ).From("job").Where( + "JSON_VALID(footprint)", + ).Where(fmt.Sprintf(`json_extract(footprint, "$.%s") is not null and json_extract(footprint, "$.%s") <= %f`, (metric + "_" + footprintStat), (metric + "_" + footprintStat), peak)) + // Only accessible Jobs... mainQuery, qerr := SecurityCheck(ctx, mainQuery) - if qerr != nil { return nil, qerr } + // Filters... for _, f := range filters { mainQuery = BuildWhereClause(f, mainQuery) } @@ -738,32 +713,34 @@ func (r *JobRepository) jobsMetricStatisticsHistogram( return nil, err } - // Setup Array + // Setup Return Array With Bin-Numbers for Match and Min/Max based on Peak points := make([]*model.MetricHistoPoint, 0) + binStep := int(peak) / *bins for i := 1; i <= *bins; i++ { - binMax := ((int(peak) / *bins) * i) - binMin := ((int(peak) / *bins) * (i - 1)) - point := model.MetricHistoPoint{Bin: &i, Count: 0, Min: &binMin, Max: &binMax} - points = append(points, &point) + binMin := (binStep * (i - 1)) + binMax := (binStep * i) + epoint := model.MetricHistoPoint{Bin: &i, Count: 0, Min: &binMin, Max: &binMax} + points = append(points, &epoint) } - for rows.Next() { - point := model.MetricHistoPoint{} - if err := rows.Scan(&point.Bin, &point.Count, &point.Min, &point.Max); err != nil { - log.Warnf("Error while scanning rows for %s", jm) - return nil, err // Totally bricks cc-backend if returned and if all metrics requested? + for rows.Next() { // Fill Count if Bin-No. Matches (Not every Bin exists in DB!) + rpoint := model.MetricHistoPoint{} + if err := rows.Scan(&rpoint.Bin, &rpoint.Count); err != nil { // Required for Debug: &rpoint.Min, &rpoint.Max + log.Warnf("Error while scanning rows for %s", metric) + return nil, err // FIXME: Totally bricks cc-backend if returned and if all metrics requested? } for _, e := range points { - if e.Bin != nil && point.Bin != nil { - if *e.Bin == *point.Bin { - e.Count = point.Count - if point.Min != nil { - e.Min = point.Min - } - if point.Max != nil { - e.Max = point.Max - } + if e.Bin != nil && rpoint.Bin != nil { + if *e.Bin == *rpoint.Bin { + e.Count = rpoint.Count + // Only Required For Debug: Check DB returned Min/Max against Backend Init above + // if rpoint.Min != nil { + // log.Warnf(">>>> Bin %d Min Set For %s to %d (Init'd with: %d)", *e.Bin, metric, *rpoint.Min, *e.Min) + // } + // if rpoint.Max != nil { + // log.Warnf(">>>> Bin %d Max Set For %s to %d (Init'd with: %d)", *e.Bin, metric, *rpoint.Max, *e.Max) + // } break } } From 61f0521072bd4d7151e4ef56695983bd2462608b Mon Sep 17 00:00:00 2001 From: brinkcoder Date: Fri, 25 Apr 2025 22:37:16 +0200 Subject: [PATCH 64/82] fix: correct logging variable from err to ipErr in AuthApi --- internal/auth/auth.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 9201315..5f88bbb 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -331,7 +331,7 @@ func (auth *Authentication) AuthApi( ipErr := securedCheck(user, r) if ipErr != nil { - log.Infof("auth api -> secured check failed: %s", err.Error()) + log.Infof("auth api -> secured check failed: %s", ipErr.Error()) onfailure(rw, r, ipErr) return } From 161f0744aa7ffd4b5f93836c614cdfbfb2a2a441 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Mon, 28 Apr 2025 09:54:22 +0200 Subject: [PATCH 65/82] fix: enforce apiAllowedIPs config option Fixes #385 --- ReleaseNotes.md | 14 ++++ cmd/cc-backend/init.go | 12 +++ configs/config-demo.json | 3 + configs/config.json | 106 ++++++++++++++----------- internal/api/api_test.go | 3 + internal/importer/importer_test.go | 3 + internal/repository/userConfig_test.go | 3 + pkg/schema/schemas/config.schema.json | 3 +- pkg/schema/validate_test.go | 26 +++--- 9 files changed, 113 insertions(+), 60 deletions(-) diff --git a/ReleaseNotes.md b/ReleaseNotes.md index 3e3939d..860f62a 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -6,6 +6,20 @@ This is a bug fix release of `cc-backend`, the API backend and frontend implementation of ClusterCockpit. For release specific notes visit the [ClusterCockpit Documentation](https://clusterockpit.org/docs/release/). +## Breaking changes + +The option `apiAllowedIPs` is now a required configuration attribute in +`config.json`. This option restricts access to the admin API. + +To retain the previous behavior that the API is per default accessible from +everywhere set: + +```json + "apiAllowedIPs": [ + "*" + ] +``` + ## Breaking changes for minor release 1.4.x - You need to perform a database migration. Depending on your database size the diff --git a/cmd/cc-backend/init.go b/cmd/cc-backend/init.go index f899ec1..0a5b836 100644 --- a/cmd/cc-backend/init.go +++ b/cmd/cc-backend/init.go @@ -32,6 +32,18 @@ const configString = ` "jwts": { "max-age": "2000h" }, + "apiAllowedIPs": [ + "*" + ], + "enable-resampling": { + "trigger": 30, + "resolutions": [ + 600, + 300, + 120, + 60 + ] + }, "clusters": [ { "name": "name", diff --git a/configs/config-demo.json b/configs/config-demo.json index e8d4570..9425bd2 100644 --- a/configs/config-demo.json +++ b/configs/config-demo.json @@ -17,6 +17,9 @@ 60 ] }, + "apiAllowedIPs": [ + "*" + ], "emission-constant": 317, "clusters": [ { diff --git a/configs/config.json b/configs/config.json index d5b8ada..f946b20 100644 --- a/configs/config.json +++ b/configs/config.json @@ -1,50 +1,62 @@ { - "addr": "0.0.0.0:443", - "ldap": { - "url": "ldaps://test", - "user_base": "ou=people,ou=hpc,dc=test,dc=de", - "search_dn": "cn=hpcmonitoring,ou=roadm,ou=profile,ou=hpc,dc=test,dc=de", - "user_bind": "uid={username},ou=people,ou=hpc,dc=test,dc=de", - "user_filter": "(&(objectclass=posixAccount))" - }, - "https-cert-file": "/etc/letsencrypt/live/url/fullchain.pem", - "https-key-file": "/etc/letsencrypt/live/url/privkey.pem", - "user": "clustercockpit", - "group": "clustercockpit", - "archive": { - "kind": "file", - "path": "./var/job-archive" - }, - "validate": true, - "clusters": [ - { - "name": "test", - "metricDataRepository": { - "kind": "cc-metric-store", - "url": "http://localhost:8082", - "token": "eyJhbGciOiJF-E-pQBQ" - }, - "filterRanges": { - "numNodes": { - "from": 1, - "to": 64 - }, - "duration": { - "from": 0, - "to": 86400 - }, - "startTime": { - "from": "2022-01-01T00:00:00Z", - "to": null - } - } + "addr": "0.0.0.0:443", + "ldap": { + "url": "ldaps://test", + "user_base": "ou=people,ou=hpc,dc=test,dc=de", + "search_dn": "cn=hpcmonitoring,ou=roadm,ou=profile,ou=hpc,dc=test,dc=de", + "user_bind": "uid={username},ou=people,ou=hpc,dc=test,dc=de", + "user_filter": "(&(objectclass=posixAccount))" + }, + "https-cert-file": "/etc/letsencrypt/live/url/fullchain.pem", + "https-key-file": "/etc/letsencrypt/live/url/privkey.pem", + "user": "clustercockpit", + "group": "clustercockpit", + "archive": { + "kind": "file", + "path": "./var/job-archive" + }, + "validate": false, + "apiAllowedIPs": [ + "*" + ], + "clusters": [ + { + "name": "test", + "metricDataRepository": { + "kind": "cc-metric-store", + "url": "http://localhost:8082", + "token": "eyJhbGciOiJF-E-pQBQ" + }, + "filterRanges": { + "numNodes": { + "from": 1, + "to": 64 + }, + "duration": { + "from": 0, + "to": 86400 + }, + "startTime": { + "from": "2022-01-01T00:00:00Z", + "to": null } - ], - "jwts": { - "cookieName": "", - "validateUser": false, - "max-age": "2000h", - "trustedIssuer": "" - }, - "short-running-jobs-duration": 300 + } + } + ], + "jwts": { + "cookieName": "", + "validateUser": false, + "max-age": "2000h", + "trustedIssuer": "" + }, + "enable-resampling": { + "trigger": 30, + "resolutions": [ + 600, + 300, + 120, + 60 + ] + }, + "short-running-jobs-duration": 300 } diff --git a/internal/api/api_test.go b/internal/api/api_test.go index c47bd4d..e67813c 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -45,6 +45,9 @@ func setup(t *testing.T) *api.RestApi { "jwts": { "max-age": "2m" }, + "apiAllowedIPs": [ + "*" + ], "clusters": [ { "name": "testcluster", diff --git a/internal/importer/importer_test.go b/internal/importer/importer_test.go index 4e839cf..209b6be 100644 --- a/internal/importer/importer_test.go +++ b/internal/importer/importer_test.go @@ -45,6 +45,9 @@ func setup(t *testing.T) *repository.JobRepository { "jwts": { "max-age": "2m" }, + "apiAllowedIPs": [ + "*" + ], "clusters": [ { "name": "testcluster", diff --git a/internal/repository/userConfig_test.go b/internal/repository/userConfig_test.go index c01bb5c..cd15c9d 100644 --- a/internal/repository/userConfig_test.go +++ b/internal/repository/userConfig_test.go @@ -25,6 +25,9 @@ func setupUserTest(t *testing.T) *UserCfgRepo { "jwts": { "max-age": "2m" }, + "apiAllowedIPs": [ + "*" + ], "clusters": [ { "name": "testcluster", diff --git a/pkg/schema/schemas/config.schema.json b/pkg/schema/schemas/config.schema.json index f372fc1..c844174 100644 --- a/pkg/schema/schemas/config.schema.json +++ b/pkg/schema/schemas/config.schema.json @@ -492,6 +492,7 @@ }, "required": [ "jwts", - "clusters" + "clusters", + "apiAllowedIPs" ] } diff --git a/pkg/schema/validate_test.go b/pkg/schema/validate_test.go index 2dc97c1..f4943f0 100644 --- a/pkg/schema/validate_test.go +++ b/pkg/schema/validate_test.go @@ -14,17 +14,20 @@ func TestValidateConfig(t *testing.T) { "jwts": { "max-age": "2m" }, - "clusters": [ - { - "name": "testcluster", - "metricDataRepository": { - "kind": "cc-metric-store", - "url": "localhost:8082"}, - "filterRanges": { - "numNodes": { "from": 1, "to": 64 }, - "duration": { "from": 0, "to": 86400 }, - "startTime": { "from": "2022-01-01T00:00:00Z", "to": null } - }}] + "apiAllowedIPs": [ + "*" + ], + "clusters": [ + { + "name": "testcluster", + "metricDataRepository": { + "kind": "cc-metric-store", + "url": "localhost:8082"}, + "filterRanges": { + "numNodes": { "from": 1, "to": 64 }, + "duration": { "from": 0, "to": 86400 }, + "startTime": { "from": "2022-01-01T00:00:00Z", "to": null } + }}] }`) if err := Validate(Config, bytes.NewReader(json)); err != nil { @@ -33,7 +36,6 @@ func TestValidateConfig(t *testing.T) { } func TestValidateJobMeta(t *testing.T) { - } func TestValidateCluster(t *testing.T) { From df497d5952a66771fd7cc20160186564d996a6ed Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 29 Apr 2025 15:10:06 +0200 Subject: [PATCH 66/82] initial branch commit, add job compare switch, add gql resolver --- api/schema.graphqls | 19 +- internal/graph/generated/generated.go | 1153 +++++++++++------ internal/graph/model/models_gen.go | 22 +- internal/graph/schema.resolvers.go | 68 +- web/frontend/src/Jobs.root.svelte | 62 +- web/frontend/src/generic/JobCompare.svelte | 156 +++ web/frontend/src/generic/JobList.svelte | 6 +- web/frontend/src/generic/plots/Polar.svelte | 2 +- .../job/jobsummary/JobFootprintPolar.svelte | 2 +- 9 files changed, 1050 insertions(+), 440 deletions(-) create mode 100644 web/frontend/src/generic/JobCompare.svelte diff --git a/api/schema.graphqls b/api/schema.graphqls index 9092b4f..1942454 100644 --- a/api/schema.graphqls +++ b/api/schema.graphqls @@ -158,7 +158,7 @@ type StatsSeries { max: [NullableFloat!]! } -type JobStatsWithScope { +type NamedStatsWithScope { name: String! scope: MetricScope! stats: [ScopedStats!]! @@ -171,8 +171,13 @@ type ScopedStats { } type JobStats { - name: String! - stats: MetricStatistics! + jobId: Int! + stats: [NamedStats!]! +} + +type NamedStats { + name: String! + data: MetricStatistics! } type Unit { @@ -259,12 +264,13 @@ type Query { job(id: ID!): Job jobMetrics(id: ID!, metrics: [String!], scopes: [MetricScope!], resolution: Int): [JobMetricWithName!]! - jobStats(id: ID!, metrics: [String!]): [JobStats!]! - scopedJobStats(id: ID!, metrics: [String!], scopes: [MetricScope!]): [JobStatsWithScope!]! - jobsFootprints(filter: [JobFilter!], metrics: [String!]!): Footprints + jobStats(id: ID!, metrics: [String!]): [NamedStats!]! + scopedJobStats(id: ID!, metrics: [String!], scopes: [MetricScope!]): [NamedStatsWithScope!]! jobs(filter: [JobFilter!], page: PageRequest, order: OrderByInput): JobResultList! jobsStatistics(filter: [JobFilter!], metrics: [String!], page: PageRequest, sortBy: SortByAggregate, groupBy: Aggregate, numDurationBins: String, numMetricBins: Int): [JobsStatistics!]! + jobsMetricStats(filter: [JobFilter!], metrics: [String!]): [JobStats!]! + jobsFootprints(filter: [JobFilter!], metrics: [String!]!): Footprints rooflineHeatmap(filter: [JobFilter!]!, rows: Int!, cols: Int!, minX: Float!, minY: Float!, maxX: Float!, maxY: Float!): [[Float!]!]! @@ -288,6 +294,7 @@ type TimeRangeOutput { range: String, from: Time!, to: Time! } input JobFilter { tags: [ID!] jobId: StringInput + jobIds: [ID!] arrayJobId: Int user: StringInput project: StringInput diff --git a/internal/graph/generated/generated.go b/internal/graph/generated/generated.go index 5dbdfd9..14d1b57 100644 --- a/internal/graph/generated/generated.go +++ b/internal/graph/generated/generated.go @@ -171,13 +171,7 @@ type ComplexityRoot struct { } JobStats struct { - Name func(childComplexity int) int - Stats func(childComplexity int) int - } - - JobStatsWithScope struct { - Name func(childComplexity int) int - Scope func(childComplexity int) int + JobID func(childComplexity int) int Stats func(childComplexity int) int } @@ -255,6 +249,17 @@ type ComplexityRoot struct { UpdateConfiguration func(childComplexity int, name string, value string) int } + NamedStats struct { + Data func(childComplexity int) int + Name func(childComplexity int) int + } + + NamedStatsWithScope struct { + Name func(childComplexity int) int + Scope func(childComplexity int) int + Stats func(childComplexity int) int + } + NodeMetrics struct { Host func(childComplexity int) int Metrics func(childComplexity int) int @@ -279,6 +284,7 @@ type ComplexityRoot struct { JobStats func(childComplexity int, id string, metrics []string) int Jobs func(childComplexity int, filter []*model.JobFilter, page *model.PageRequest, order *model.OrderByInput) int JobsFootprints func(childComplexity int, filter []*model.JobFilter, metrics []string) int + JobsMetricStats func(childComplexity int, filter []*model.JobFilter, metrics []string) int JobsStatistics func(childComplexity int, filter []*model.JobFilter, metrics []string, page *model.PageRequest, sortBy *model.SortByAggregate, groupBy *model.Aggregate, numDurationBins *string, numMetricBins *int) int NodeMetrics func(childComplexity int, cluster string, nodes []string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time) int NodeMetricsList func(childComplexity int, cluster string, subCluster string, nodeFilter string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time, page *model.PageRequest, resolution *int) int @@ -411,11 +417,12 @@ type QueryResolver interface { AllocatedNodes(ctx context.Context, cluster string) ([]*model.Count, error) Job(ctx context.Context, id string) (*schema.Job, error) JobMetrics(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope, resolution *int) ([]*model.JobMetricWithName, error) - JobStats(ctx context.Context, id string, metrics []string) ([]*model.JobStats, error) - ScopedJobStats(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope) ([]*model.JobStatsWithScope, error) - JobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) + JobStats(ctx context.Context, id string, metrics []string) ([]*model.NamedStats, error) + ScopedJobStats(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope) ([]*model.NamedStatsWithScope, error) Jobs(ctx context.Context, filter []*model.JobFilter, page *model.PageRequest, order *model.OrderByInput) (*model.JobResultList, error) JobsStatistics(ctx context.Context, filter []*model.JobFilter, metrics []string, page *model.PageRequest, sortBy *model.SortByAggregate, groupBy *model.Aggregate, numDurationBins *string, numMetricBins *int) ([]*model.JobsStatistics, error) + JobsMetricStats(ctx context.Context, filter []*model.JobFilter, metrics []string) ([]*model.JobStats, error) + JobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) RooflineHeatmap(ctx context.Context, filter []*model.JobFilter, rows int, cols int, minX float64, minY float64, maxX float64, maxY float64) ([][]float64, error) NodeMetrics(ctx context.Context, cluster string, nodes []string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time) ([]*model.NodeMetrics, error) NodeMetricsList(ctx context.Context, cluster string, subCluster string, nodeFilter string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time, page *model.PageRequest, resolution *int) (*model.NodesResultList, error) @@ -933,12 +940,12 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.JobResultList.Offset(childComplexity), true - case "JobStats.name": - if e.complexity.JobStats.Name == nil { + case "JobStats.jobId": + if e.complexity.JobStats.JobID == nil { break } - return e.complexity.JobStats.Name(childComplexity), true + return e.complexity.JobStats.JobID(childComplexity), true case "JobStats.stats": if e.complexity.JobStats.Stats == nil { @@ -947,27 +954,6 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.JobStats.Stats(childComplexity), true - case "JobStatsWithScope.name": - if e.complexity.JobStatsWithScope.Name == nil { - break - } - - return e.complexity.JobStatsWithScope.Name(childComplexity), true - - case "JobStatsWithScope.scope": - if e.complexity.JobStatsWithScope.Scope == nil { - break - } - - return e.complexity.JobStatsWithScope.Scope(childComplexity), true - - case "JobStatsWithScope.stats": - if e.complexity.JobStatsWithScope.Stats == nil { - break - } - - return e.complexity.JobStatsWithScope.Stats(childComplexity), true - case "JobsStatistics.histDuration": if e.complexity.JobsStatistics.HistDuration == nil { break @@ -1348,6 +1334,41 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Mutation.UpdateConfiguration(childComplexity, args["name"].(string), args["value"].(string)), true + case "NamedStats.data": + if e.complexity.NamedStats.Data == nil { + break + } + + return e.complexity.NamedStats.Data(childComplexity), true + + case "NamedStats.name": + if e.complexity.NamedStats.Name == nil { + break + } + + return e.complexity.NamedStats.Name(childComplexity), true + + case "NamedStatsWithScope.name": + if e.complexity.NamedStatsWithScope.Name == nil { + break + } + + return e.complexity.NamedStatsWithScope.Name(childComplexity), true + + case "NamedStatsWithScope.scope": + if e.complexity.NamedStatsWithScope.Scope == nil { + break + } + + return e.complexity.NamedStatsWithScope.Scope(childComplexity), true + + case "NamedStatsWithScope.stats": + if e.complexity.NamedStatsWithScope.Stats == nil { + break + } + + return e.complexity.NamedStatsWithScope.Stats(childComplexity), true + case "NodeMetrics.host": if e.complexity.NodeMetrics.Host == nil { break @@ -1497,6 +1518,18 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Query.JobsFootprints(childComplexity, args["filter"].([]*model.JobFilter), args["metrics"].([]string)), true + case "Query.jobsMetricStats": + if e.complexity.Query.JobsMetricStats == nil { + break + } + + args, err := ec.field_Query_jobsMetricStats_args(context.TODO(), rawArgs) + if err != nil { + return 0, false + } + + return e.complexity.Query.JobsMetricStats(childComplexity, args["filter"].([]*model.JobFilter), args["metrics"].([]string)), true + case "Query.jobsStatistics": if e.complexity.Query.JobsStatistics == nil { break @@ -2234,7 +2267,7 @@ type StatsSeries { max: [NullableFloat!]! } -type JobStatsWithScope { +type NamedStatsWithScope { name: String! scope: MetricScope! stats: [ScopedStats!]! @@ -2247,8 +2280,13 @@ type ScopedStats { } type JobStats { - name: String! - stats: MetricStatistics! + jobId: Int! + stats: [NamedStats!]! +} + +type NamedStats { + name: String! + data: MetricStatistics! } type Unit { @@ -2335,12 +2373,13 @@ type Query { job(id: ID!): Job jobMetrics(id: ID!, metrics: [String!], scopes: [MetricScope!], resolution: Int): [JobMetricWithName!]! - jobStats(id: ID!, metrics: [String!]): [JobStats!]! - scopedJobStats(id: ID!, metrics: [String!], scopes: [MetricScope!]): [JobStatsWithScope!]! - jobsFootprints(filter: [JobFilter!], metrics: [String!]!): Footprints + jobStats(id: ID!, metrics: [String!]): [NamedStats!]! + scopedJobStats(id: ID!, metrics: [String!], scopes: [MetricScope!]): [NamedStatsWithScope!]! jobs(filter: [JobFilter!], page: PageRequest, order: OrderByInput): JobResultList! jobsStatistics(filter: [JobFilter!], metrics: [String!], page: PageRequest, sortBy: SortByAggregate, groupBy: Aggregate, numDurationBins: String, numMetricBins: Int): [JobsStatistics!]! + jobsMetricStats(filter: [JobFilter!], metrics: [String!]): [JobStats!]! + jobsFootprints(filter: [JobFilter!], metrics: [String!]!): Footprints rooflineHeatmap(filter: [JobFilter!]!, rows: Int!, cols: Int!, minX: Float!, minY: Float!, maxX: Float!, maxY: Float!): [[Float!]!]! @@ -2364,6 +2403,7 @@ type TimeRangeOutput { range: String, from: Time!, to: Time! } input JobFilter { tags: [ID!] jobId: StringInput + jobIds: [ID!] arrayJobId: Int user: StringInput project: StringInput @@ -3045,6 +3085,57 @@ func (ec *executionContext) field_Query_jobsFootprints_argsMetrics( return zeroVal, nil } +func (ec *executionContext) field_Query_jobsMetricStats_args(ctx context.Context, rawArgs map[string]any) (map[string]any, error) { + var err error + args := map[string]any{} + arg0, err := ec.field_Query_jobsMetricStats_argsFilter(ctx, rawArgs) + if err != nil { + return nil, err + } + args["filter"] = arg0 + arg1, err := ec.field_Query_jobsMetricStats_argsMetrics(ctx, rawArgs) + if err != nil { + return nil, err + } + args["metrics"] = arg1 + return args, nil +} +func (ec *executionContext) field_Query_jobsMetricStats_argsFilter( + ctx context.Context, + rawArgs map[string]any, +) ([]*model.JobFilter, error) { + if _, ok := rawArgs["filter"]; !ok { + var zeroVal []*model.JobFilter + return zeroVal, nil + } + + ctx = graphql.WithPathContext(ctx, graphql.NewPathWithField("filter")) + if tmp, ok := rawArgs["filter"]; ok { + return ec.unmarshalOJobFilter2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobFilterᚄ(ctx, tmp) + } + + var zeroVal []*model.JobFilter + return zeroVal, nil +} + +func (ec *executionContext) field_Query_jobsMetricStats_argsMetrics( + ctx context.Context, + rawArgs map[string]any, +) ([]string, error) { + if _, ok := rawArgs["metrics"]; !ok { + var zeroVal []string + return zeroVal, nil + } + + ctx = graphql.WithPathContext(ctx, graphql.NewPathWithField("metrics")) + if tmp, ok := rawArgs["metrics"]; ok { + return ec.unmarshalOString2ᚕstringᚄ(ctx, tmp) + } + + var zeroVal []string + return zeroVal, nil +} + func (ec *executionContext) field_Query_jobsStatistics_args(ctx context.Context, rawArgs map[string]any) (map[string]any, error) { var err error args := map[string]any{} @@ -7265,8 +7356,8 @@ func (ec *executionContext) fieldContext_JobResultList_hasNextPage(_ context.Con return fc, nil } -func (ec *executionContext) _JobStats_name(ctx context.Context, field graphql.CollectedField, obj *model.JobStats) (ret graphql.Marshaler) { - fc, err := ec.fieldContext_JobStats_name(ctx, field) +func (ec *executionContext) _JobStats_jobId(ctx context.Context, field graphql.CollectedField, obj *model.JobStats) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_JobStats_jobId(ctx, field) if err != nil { return graphql.Null } @@ -7279,7 +7370,7 @@ func (ec *executionContext) _JobStats_name(ctx context.Context, field graphql.Co }() resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { ctx = rctx // use context from middleware stack in children - return obj.Name, nil + return obj.JobID, nil }) if err != nil { ec.Error(ctx, err) @@ -7291,19 +7382,19 @@ func (ec *executionContext) _JobStats_name(ctx context.Context, field graphql.Co } return graphql.Null } - res := resTmp.(string) + res := resTmp.(int) fc.Result = res - return ec.marshalNString2string(ctx, field.Selections, res) + return ec.marshalNInt2int(ctx, field.Selections, res) } -func (ec *executionContext) fieldContext_JobStats_name(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { +func (ec *executionContext) fieldContext_JobStats_jobId(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { fc = &graphql.FieldContext{ Object: "JobStats", Field: field, IsMethod: false, IsResolver: false, Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { - return nil, errors.New("field of type String does not have child fields") + return nil, errors.New("field of type Int does not have child fields") }, } return fc, nil @@ -7335,9 +7426,9 @@ func (ec *executionContext) _JobStats_stats(ctx context.Context, field graphql.C } return graphql.Null } - res := resTmp.(*schema.MetricStatistics) + res := resTmp.([]*model.NamedStats) fc.Result = res - return ec.marshalNMetricStatistics2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋpkgᚋschemaᚐMetricStatistics(ctx, field.Selections, res) + return ec.marshalNNamedStats2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNamedStatsᚄ(ctx, field.Selections, res) } func (ec *executionContext) fieldContext_JobStats_stats(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { @@ -7348,154 +7439,12 @@ func (ec *executionContext) fieldContext_JobStats_stats(_ context.Context, field IsResolver: false, Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { switch field.Name { - case "avg": - return ec.fieldContext_MetricStatistics_avg(ctx, field) - case "min": - return ec.fieldContext_MetricStatistics_min(ctx, field) - case "max": - return ec.fieldContext_MetricStatistics_max(ctx, field) - } - return nil, fmt.Errorf("no field named %q was found under type MetricStatistics", field.Name) - }, - } - return fc, nil -} - -func (ec *executionContext) _JobStatsWithScope_name(ctx context.Context, field graphql.CollectedField, obj *model.JobStatsWithScope) (ret graphql.Marshaler) { - fc, err := ec.fieldContext_JobStatsWithScope_name(ctx, field) - if err != nil { - return graphql.Null - } - ctx = graphql.WithFieldContext(ctx, fc) - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = graphql.Null - } - }() - resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { - ctx = rctx // use context from middleware stack in children - return obj.Name, nil - }) - if err != nil { - ec.Error(ctx, err) - return graphql.Null - } - if resTmp == nil { - if !graphql.HasFieldError(ctx, fc) { - ec.Errorf(ctx, "must not be null") - } - return graphql.Null - } - res := resTmp.(string) - fc.Result = res - return ec.marshalNString2string(ctx, field.Selections, res) -} - -func (ec *executionContext) fieldContext_JobStatsWithScope_name(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { - fc = &graphql.FieldContext{ - Object: "JobStatsWithScope", - Field: field, - IsMethod: false, - IsResolver: false, - Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { - return nil, errors.New("field of type String does not have child fields") - }, - } - return fc, nil -} - -func (ec *executionContext) _JobStatsWithScope_scope(ctx context.Context, field graphql.CollectedField, obj *model.JobStatsWithScope) (ret graphql.Marshaler) { - fc, err := ec.fieldContext_JobStatsWithScope_scope(ctx, field) - if err != nil { - return graphql.Null - } - ctx = graphql.WithFieldContext(ctx, fc) - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = graphql.Null - } - }() - resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { - ctx = rctx // use context from middleware stack in children - return obj.Scope, nil - }) - if err != nil { - ec.Error(ctx, err) - return graphql.Null - } - if resTmp == nil { - if !graphql.HasFieldError(ctx, fc) { - ec.Errorf(ctx, "must not be null") - } - return graphql.Null - } - res := resTmp.(schema.MetricScope) - fc.Result = res - return ec.marshalNMetricScope2githubᚗcomᚋClusterCockpitᚋccᚑbackendᚋpkgᚋschemaᚐMetricScope(ctx, field.Selections, res) -} - -func (ec *executionContext) fieldContext_JobStatsWithScope_scope(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { - fc = &graphql.FieldContext{ - Object: "JobStatsWithScope", - Field: field, - IsMethod: false, - IsResolver: false, - Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { - return nil, errors.New("field of type MetricScope does not have child fields") - }, - } - return fc, nil -} - -func (ec *executionContext) _JobStatsWithScope_stats(ctx context.Context, field graphql.CollectedField, obj *model.JobStatsWithScope) (ret graphql.Marshaler) { - fc, err := ec.fieldContext_JobStatsWithScope_stats(ctx, field) - if err != nil { - return graphql.Null - } - ctx = graphql.WithFieldContext(ctx, fc) - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = graphql.Null - } - }() - resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { - ctx = rctx // use context from middleware stack in children - return obj.Stats, nil - }) - if err != nil { - ec.Error(ctx, err) - return graphql.Null - } - if resTmp == nil { - if !graphql.HasFieldError(ctx, fc) { - ec.Errorf(ctx, "must not be null") - } - return graphql.Null - } - res := resTmp.([]*model.ScopedStats) - fc.Result = res - return ec.marshalNScopedStats2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐScopedStatsᚄ(ctx, field.Selections, res) -} - -func (ec *executionContext) fieldContext_JobStatsWithScope_stats(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { - fc = &graphql.FieldContext{ - Object: "JobStatsWithScope", - Field: field, - IsMethod: false, - IsResolver: false, - Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { - switch field.Name { - case "hostname": - return ec.fieldContext_ScopedStats_hostname(ctx, field) - case "id": - return ec.fieldContext_ScopedStats_id(ctx, field) + case "name": + return ec.fieldContext_NamedStats_name(ctx, field) case "data": - return ec.fieldContext_ScopedStats_data(ctx, field) + return ec.fieldContext_NamedStats_data(ctx, field) } - return nil, fmt.Errorf("no field named %q was found under type ScopedStats", field.Name) + return nil, fmt.Errorf("no field named %q was found under type NamedStats", field.Name) }, } return fc, nil @@ -9840,6 +9789,242 @@ func (ec *executionContext) fieldContext_Mutation_updateConfiguration(ctx contex return fc, nil } +func (ec *executionContext) _NamedStats_name(ctx context.Context, field graphql.CollectedField, obj *model.NamedStats) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_NamedStats_name(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.Name, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(string) + fc.Result = res + return ec.marshalNString2string(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_NamedStats_name(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "NamedStats", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type String does not have child fields") + }, + } + return fc, nil +} + +func (ec *executionContext) _NamedStats_data(ctx context.Context, field graphql.CollectedField, obj *model.NamedStats) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_NamedStats_data(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.Data, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(*schema.MetricStatistics) + fc.Result = res + return ec.marshalNMetricStatistics2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋpkgᚋschemaᚐMetricStatistics(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_NamedStats_data(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "NamedStats", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "avg": + return ec.fieldContext_MetricStatistics_avg(ctx, field) + case "min": + return ec.fieldContext_MetricStatistics_min(ctx, field) + case "max": + return ec.fieldContext_MetricStatistics_max(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type MetricStatistics", field.Name) + }, + } + return fc, nil +} + +func (ec *executionContext) _NamedStatsWithScope_name(ctx context.Context, field graphql.CollectedField, obj *model.NamedStatsWithScope) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_NamedStatsWithScope_name(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.Name, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(string) + fc.Result = res + return ec.marshalNString2string(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_NamedStatsWithScope_name(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "NamedStatsWithScope", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type String does not have child fields") + }, + } + return fc, nil +} + +func (ec *executionContext) _NamedStatsWithScope_scope(ctx context.Context, field graphql.CollectedField, obj *model.NamedStatsWithScope) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_NamedStatsWithScope_scope(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.Scope, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(schema.MetricScope) + fc.Result = res + return ec.marshalNMetricScope2githubᚗcomᚋClusterCockpitᚋccᚑbackendᚋpkgᚋschemaᚐMetricScope(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_NamedStatsWithScope_scope(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "NamedStatsWithScope", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type MetricScope does not have child fields") + }, + } + return fc, nil +} + +func (ec *executionContext) _NamedStatsWithScope_stats(ctx context.Context, field graphql.CollectedField, obj *model.NamedStatsWithScope) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_NamedStatsWithScope_stats(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.Stats, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.([]*model.ScopedStats) + fc.Result = res + return ec.marshalNScopedStats2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐScopedStatsᚄ(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_NamedStatsWithScope_stats(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "NamedStatsWithScope", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "hostname": + return ec.fieldContext_ScopedStats_hostname(ctx, field) + case "id": + return ec.fieldContext_ScopedStats_id(ctx, field) + case "data": + return ec.fieldContext_ScopedStats_data(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type ScopedStats", field.Name) + }, + } + return fc, nil +} + func (ec *executionContext) _NodeMetrics_host(ctx context.Context, field graphql.CollectedField, obj *model.NodeMetrics) (ret graphql.Marshaler) { fc, err := ec.fieldContext_NodeMetrics_host(ctx, field) if err != nil { @@ -10715,9 +10900,9 @@ func (ec *executionContext) _Query_jobStats(ctx context.Context, field graphql.C } return graphql.Null } - res := resTmp.([]*model.JobStats) + res := resTmp.([]*model.NamedStats) fc.Result = res - return ec.marshalNJobStats2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobStatsᚄ(ctx, field.Selections, res) + return ec.marshalNNamedStats2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNamedStatsᚄ(ctx, field.Selections, res) } func (ec *executionContext) fieldContext_Query_jobStats(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { @@ -10729,11 +10914,11 @@ func (ec *executionContext) fieldContext_Query_jobStats(ctx context.Context, fie Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { switch field.Name { case "name": - return ec.fieldContext_JobStats_name(ctx, field) - case "stats": - return ec.fieldContext_JobStats_stats(ctx, field) + return ec.fieldContext_NamedStats_name(ctx, field) + case "data": + return ec.fieldContext_NamedStats_data(ctx, field) } - return nil, fmt.Errorf("no field named %q was found under type JobStats", field.Name) + return nil, fmt.Errorf("no field named %q was found under type NamedStats", field.Name) }, } defer func() { @@ -10776,9 +10961,9 @@ func (ec *executionContext) _Query_scopedJobStats(ctx context.Context, field gra } return graphql.Null } - res := resTmp.([]*model.JobStatsWithScope) + res := resTmp.([]*model.NamedStatsWithScope) fc.Result = res - return ec.marshalNJobStatsWithScope2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobStatsWithScopeᚄ(ctx, field.Selections, res) + return ec.marshalNNamedStatsWithScope2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNamedStatsWithScopeᚄ(ctx, field.Selections, res) } func (ec *executionContext) fieldContext_Query_scopedJobStats(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { @@ -10790,13 +10975,13 @@ func (ec *executionContext) fieldContext_Query_scopedJobStats(ctx context.Contex Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { switch field.Name { case "name": - return ec.fieldContext_JobStatsWithScope_name(ctx, field) + return ec.fieldContext_NamedStatsWithScope_name(ctx, field) case "scope": - return ec.fieldContext_JobStatsWithScope_scope(ctx, field) + return ec.fieldContext_NamedStatsWithScope_scope(ctx, field) case "stats": - return ec.fieldContext_JobStatsWithScope_stats(ctx, field) + return ec.fieldContext_NamedStatsWithScope_stats(ctx, field) } - return nil, fmt.Errorf("no field named %q was found under type JobStatsWithScope", field.Name) + return nil, fmt.Errorf("no field named %q was found under type NamedStatsWithScope", field.Name) }, } defer func() { @@ -10813,64 +10998,6 @@ func (ec *executionContext) fieldContext_Query_scopedJobStats(ctx context.Contex return fc, nil } -func (ec *executionContext) _Query_jobsFootprints(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { - fc, err := ec.fieldContext_Query_jobsFootprints(ctx, field) - if err != nil { - return graphql.Null - } - ctx = graphql.WithFieldContext(ctx, fc) - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = graphql.Null - } - }() - resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { - ctx = rctx // use context from middleware stack in children - return ec.resolvers.Query().JobsFootprints(rctx, fc.Args["filter"].([]*model.JobFilter), fc.Args["metrics"].([]string)) - }) - if err != nil { - ec.Error(ctx, err) - return graphql.Null - } - if resTmp == nil { - return graphql.Null - } - res := resTmp.(*model.Footprints) - fc.Result = res - return ec.marshalOFootprints2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐFootprints(ctx, field.Selections, res) -} - -func (ec *executionContext) fieldContext_Query_jobsFootprints(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { - fc = &graphql.FieldContext{ - Object: "Query", - Field: field, - IsMethod: true, - IsResolver: true, - Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { - switch field.Name { - case "timeWeights": - return ec.fieldContext_Footprints_timeWeights(ctx, field) - case "metrics": - return ec.fieldContext_Footprints_metrics(ctx, field) - } - return nil, fmt.Errorf("no field named %q was found under type Footprints", field.Name) - }, - } - defer func() { - if r := recover(); r != nil { - err = ec.Recover(ctx, r) - ec.Error(ctx, err) - } - }() - ctx = graphql.WithFieldContext(ctx, fc) - if fc.Args, err = ec.field_Query_jobsFootprints_args(ctx, field.ArgumentMap(ec.Variables)); err != nil { - ec.Error(ctx, err) - return fc, err - } - return fc, nil -} - func (ec *executionContext) _Query_jobs(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { fc, err := ec.fieldContext_Query_jobs(ctx, field) if err != nil { @@ -11029,6 +11156,125 @@ func (ec *executionContext) fieldContext_Query_jobsStatistics(ctx context.Contex return fc, nil } +func (ec *executionContext) _Query_jobsMetricStats(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_Query_jobsMetricStats(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return ec.resolvers.Query().JobsMetricStats(rctx, fc.Args["filter"].([]*model.JobFilter), fc.Args["metrics"].([]string)) + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.([]*model.JobStats) + fc.Result = res + return ec.marshalNJobStats2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobStatsᚄ(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_Query_jobsMetricStats(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "Query", + Field: field, + IsMethod: true, + IsResolver: true, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "jobId": + return ec.fieldContext_JobStats_jobId(ctx, field) + case "stats": + return ec.fieldContext_JobStats_stats(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type JobStats", field.Name) + }, + } + defer func() { + if r := recover(); r != nil { + err = ec.Recover(ctx, r) + ec.Error(ctx, err) + } + }() + ctx = graphql.WithFieldContext(ctx, fc) + if fc.Args, err = ec.field_Query_jobsMetricStats_args(ctx, field.ArgumentMap(ec.Variables)); err != nil { + ec.Error(ctx, err) + return fc, err + } + return fc, nil +} + +func (ec *executionContext) _Query_jobsFootprints(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_Query_jobsFootprints(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return ec.resolvers.Query().JobsFootprints(rctx, fc.Args["filter"].([]*model.JobFilter), fc.Args["metrics"].([]string)) + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + return graphql.Null + } + res := resTmp.(*model.Footprints) + fc.Result = res + return ec.marshalOFootprints2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐFootprints(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_Query_jobsFootprints(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "Query", + Field: field, + IsMethod: true, + IsResolver: true, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "timeWeights": + return ec.fieldContext_Footprints_timeWeights(ctx, field) + case "metrics": + return ec.fieldContext_Footprints_metrics(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type Footprints", field.Name) + }, + } + defer func() { + if r := recover(); r != nil { + err = ec.Recover(ctx, r) + ec.Error(ctx, err) + } + }() + ctx = graphql.WithFieldContext(ctx, fc) + if fc.Args, err = ec.field_Query_jobsFootprints_args(ctx, field.ArgumentMap(ec.Variables)); err != nil { + ec.Error(ctx, err) + return fc, err + } + return fc, nil +} + func (ec *executionContext) _Query_rooflineHeatmap(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { fc, err := ec.fieldContext_Query_rooflineHeatmap(ctx, field) if err != nil { @@ -15822,7 +16068,7 @@ func (ec *executionContext) unmarshalInputJobFilter(ctx context.Context, obj any asMap[k] = v } - fieldsInOrder := [...]string{"tags", "jobId", "arrayJobId", "user", "project", "jobName", "cluster", "partition", "duration", "energy", "minRunningFor", "numNodes", "numAccelerators", "numHWThreads", "startTime", "state", "metricStats", "exclusive", "node"} + fieldsInOrder := [...]string{"tags", "jobId", "jobIds", "arrayJobId", "user", "project", "jobName", "cluster", "partition", "duration", "energy", "minRunningFor", "numNodes", "numAccelerators", "numHWThreads", "startTime", "state", "metricStats", "exclusive", "node"} for _, k := range fieldsInOrder { v, ok := asMap[k] if !ok { @@ -15843,6 +16089,13 @@ func (ec *executionContext) unmarshalInputJobFilter(ctx context.Context, obj any return it, err } it.JobID = data + case "jobIds": + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("jobIds")) + data, err := ec.unmarshalOID2ᚕstringᚄ(ctx, v) + if err != nil { + return it, err + } + it.JobIds = data case "arrayJobId": ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("arrayJobId")) data, err := ec.unmarshalOInt2ᚖint(ctx, v) @@ -17269,8 +17522,8 @@ func (ec *executionContext) _JobStats(ctx context.Context, sel ast.SelectionSet, switch field.Name { case "__typename": out.Values[i] = graphql.MarshalString("JobStats") - case "name": - out.Values[i] = ec._JobStats_name(ctx, field, obj) + case "jobId": + out.Values[i] = ec._JobStats_jobId(ctx, field, obj) if out.Values[i] == graphql.Null { out.Invalids++ } @@ -17302,55 +17555,6 @@ func (ec *executionContext) _JobStats(ctx context.Context, sel ast.SelectionSet, return out } -var jobStatsWithScopeImplementors = []string{"JobStatsWithScope"} - -func (ec *executionContext) _JobStatsWithScope(ctx context.Context, sel ast.SelectionSet, obj *model.JobStatsWithScope) graphql.Marshaler { - fields := graphql.CollectFields(ec.OperationContext, sel, jobStatsWithScopeImplementors) - - out := graphql.NewFieldSet(fields) - deferred := make(map[string]*graphql.FieldSet) - for i, field := range fields { - switch field.Name { - case "__typename": - out.Values[i] = graphql.MarshalString("JobStatsWithScope") - case "name": - out.Values[i] = ec._JobStatsWithScope_name(ctx, field, obj) - if out.Values[i] == graphql.Null { - out.Invalids++ - } - case "scope": - out.Values[i] = ec._JobStatsWithScope_scope(ctx, field, obj) - if out.Values[i] == graphql.Null { - out.Invalids++ - } - case "stats": - out.Values[i] = ec._JobStatsWithScope_stats(ctx, field, obj) - if out.Values[i] == graphql.Null { - out.Invalids++ - } - default: - panic("unknown field " + strconv.Quote(field.Name)) - } - } - out.Dispatch(ctx) - if out.Invalids > 0 { - return graphql.Null - } - - atomic.AddInt32(&ec.deferred, int32(len(deferred))) - - for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ - Label: label, - Path: graphql.GetPath(ctx), - FieldSet: dfs, - Context: ctx, - }) - } - - return out -} - var jobsStatisticsImplementors = []string{"JobsStatistics"} func (ec *executionContext) _JobsStatistics(ctx context.Context, sel ast.SelectionSet, obj *model.JobsStatistics) graphql.Marshaler { @@ -17897,6 +18101,99 @@ func (ec *executionContext) _Mutation(ctx context.Context, sel ast.SelectionSet) return out } +var namedStatsImplementors = []string{"NamedStats"} + +func (ec *executionContext) _NamedStats(ctx context.Context, sel ast.SelectionSet, obj *model.NamedStats) graphql.Marshaler { + fields := graphql.CollectFields(ec.OperationContext, sel, namedStatsImplementors) + + out := graphql.NewFieldSet(fields) + deferred := make(map[string]*graphql.FieldSet) + for i, field := range fields { + switch field.Name { + case "__typename": + out.Values[i] = graphql.MarshalString("NamedStats") + case "name": + out.Values[i] = ec._NamedStats_name(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + case "data": + out.Values[i] = ec._NamedStats_data(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + default: + panic("unknown field " + strconv.Quote(field.Name)) + } + } + out.Dispatch(ctx) + if out.Invalids > 0 { + return graphql.Null + } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + + return out +} + +var namedStatsWithScopeImplementors = []string{"NamedStatsWithScope"} + +func (ec *executionContext) _NamedStatsWithScope(ctx context.Context, sel ast.SelectionSet, obj *model.NamedStatsWithScope) graphql.Marshaler { + fields := graphql.CollectFields(ec.OperationContext, sel, namedStatsWithScopeImplementors) + + out := graphql.NewFieldSet(fields) + deferred := make(map[string]*graphql.FieldSet) + for i, field := range fields { + switch field.Name { + case "__typename": + out.Values[i] = graphql.MarshalString("NamedStatsWithScope") + case "name": + out.Values[i] = ec._NamedStatsWithScope_name(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + case "scope": + out.Values[i] = ec._NamedStatsWithScope_scope(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + case "stats": + out.Values[i] = ec._NamedStatsWithScope_stats(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + default: + panic("unknown field " + strconv.Quote(field.Name)) + } + } + out.Dispatch(ctx) + if out.Invalids > 0 { + return graphql.Null + } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + + return out +} + var nodeMetricsImplementors = []string{"NodeMetrics"} func (ec *executionContext) _NodeMetrics(ctx context.Context, sel ast.SelectionSet, obj *model.NodeMetrics) graphql.Marshaler { @@ -18205,25 +18502,6 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) } - out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return rrm(innerCtx) }) - case "jobsFootprints": - field := field - - innerFunc := func(ctx context.Context, _ *graphql.FieldSet) (res graphql.Marshaler) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - } - }() - res = ec._Query_jobsFootprints(ctx, field) - return res - } - - rrm := func(ctx context.Context) graphql.Marshaler { - return ec.OperationContext.RootResolverMiddleware(ctx, - func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) - } - out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return rrm(innerCtx) }) case "jobs": field := field @@ -18268,6 +18546,47 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) } + out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return rrm(innerCtx) }) + case "jobsMetricStats": + field := field + + innerFunc := func(ctx context.Context, fs *graphql.FieldSet) (res graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + } + }() + res = ec._Query_jobsMetricStats(ctx, field) + if res == graphql.Null { + atomic.AddUint32(&fs.Invalids, 1) + } + return res + } + + rrm := func(ctx context.Context) graphql.Marshaler { + return ec.OperationContext.RootResolverMiddleware(ctx, + func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) + } + + out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return rrm(innerCtx) }) + case "jobsFootprints": + field := field + + innerFunc := func(ctx context.Context, _ *graphql.FieldSet) (res graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + } + }() + res = ec._Query_jobsFootprints(ctx, field) + return res + } + + rrm := func(ctx context.Context) graphql.Marshaler { + return ec.OperationContext.RootResolverMiddleware(ctx, + func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) + } + out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return rrm(innerCtx) }) case "rooflineHeatmap": field := field @@ -20195,60 +20514,6 @@ func (ec *executionContext) marshalNJobStats2ᚖgithubᚗcomᚋClusterCockpitᚋ return ec._JobStats(ctx, sel, v) } -func (ec *executionContext) marshalNJobStatsWithScope2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobStatsWithScopeᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.JobStatsWithScope) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNJobStatsWithScope2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobStatsWithScope(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() - - for _, e := range ret { - if e == graphql.Null { - return graphql.Null - } - } - - return ret -} - -func (ec *executionContext) marshalNJobStatsWithScope2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobStatsWithScope(ctx context.Context, sel ast.SelectionSet, v *model.JobStatsWithScope) graphql.Marshaler { - if v == nil { - if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { - ec.Errorf(ctx, "the requested element is null which the schema does not allow") - } - return graphql.Null - } - return ec._JobStatsWithScope(ctx, sel, v) -} - func (ec *executionContext) marshalNJobsStatistics2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobsStatisticsᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.JobsStatistics) graphql.Marshaler { ret := make(graphql.Array, len(v)) var wg sync.WaitGroup @@ -20498,6 +20763,114 @@ func (ec *executionContext) marshalNMetricValue2githubᚗcomᚋClusterCockpitᚋ return ec._MetricValue(ctx, sel, &v) } +func (ec *executionContext) marshalNNamedStats2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNamedStatsᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.NamedStats) graphql.Marshaler { + ret := make(graphql.Array, len(v)) + var wg sync.WaitGroup + isLen1 := len(v) == 1 + if !isLen1 { + wg.Add(len(v)) + } + for i := range v { + i := i + fc := &graphql.FieldContext{ + Index: &i, + Result: &v[i], + } + ctx := graphql.WithFieldContext(ctx, fc) + f := func(i int) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = nil + } + }() + if !isLen1 { + defer wg.Done() + } + ret[i] = ec.marshalNNamedStats2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNamedStats(ctx, sel, v[i]) + } + if isLen1 { + f(i) + } else { + go f(i) + } + + } + wg.Wait() + + for _, e := range ret { + if e == graphql.Null { + return graphql.Null + } + } + + return ret +} + +func (ec *executionContext) marshalNNamedStats2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNamedStats(ctx context.Context, sel ast.SelectionSet, v *model.NamedStats) graphql.Marshaler { + if v == nil { + if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { + ec.Errorf(ctx, "the requested element is null which the schema does not allow") + } + return graphql.Null + } + return ec._NamedStats(ctx, sel, v) +} + +func (ec *executionContext) marshalNNamedStatsWithScope2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNamedStatsWithScopeᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.NamedStatsWithScope) graphql.Marshaler { + ret := make(graphql.Array, len(v)) + var wg sync.WaitGroup + isLen1 := len(v) == 1 + if !isLen1 { + wg.Add(len(v)) + } + for i := range v { + i := i + fc := &graphql.FieldContext{ + Index: &i, + Result: &v[i], + } + ctx := graphql.WithFieldContext(ctx, fc) + f := func(i int) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = nil + } + }() + if !isLen1 { + defer wg.Done() + } + ret[i] = ec.marshalNNamedStatsWithScope2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNamedStatsWithScope(ctx, sel, v[i]) + } + if isLen1 { + f(i) + } else { + go f(i) + } + + } + wg.Wait() + + for _, e := range ret { + if e == graphql.Null { + return graphql.Null + } + } + + return ret +} + +func (ec *executionContext) marshalNNamedStatsWithScope2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNamedStatsWithScope(ctx context.Context, sel ast.SelectionSet, v *model.NamedStatsWithScope) graphql.Marshaler { + if v == nil { + if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { + ec.Errorf(ctx, "the requested element is null which the schema does not allow") + } + return graphql.Null + } + return ec._NamedStatsWithScope(ctx, sel, v) +} + func (ec *executionContext) marshalNNodeMetrics2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNodeMetricsᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.NodeMetrics) graphql.Marshaler { ret := make(graphql.Array, len(v)) var wg sync.WaitGroup diff --git a/internal/graph/model/models_gen.go b/internal/graph/model/models_gen.go index 43c4e37..fdd3bf3 100644 --- a/internal/graph/model/models_gen.go +++ b/internal/graph/model/models_gen.go @@ -51,6 +51,7 @@ type IntRangeOutput struct { type JobFilter struct { Tags []string `json:"tags,omitempty"` JobID *StringInput `json:"jobId,omitempty"` + JobIds []string `json:"jobIds,omitempty"` ArrayJobID *int `json:"arrayJobId,omitempty"` User *StringInput `json:"user,omitempty"` Project *StringInput `json:"project,omitempty"` @@ -96,14 +97,8 @@ type JobResultList struct { } type JobStats struct { - Name string `json:"name"` - Stats *schema.MetricStatistics `json:"stats"` -} - -type JobStatsWithScope struct { - Name string `json:"name"` - Scope schema.MetricScope `json:"scope"` - Stats []*ScopedStats `json:"stats"` + JobID int `json:"jobId"` + Stats []*NamedStats `json:"stats"` } type JobsStatistics struct { @@ -153,6 +148,17 @@ type MetricStatItem struct { type Mutation struct { } +type NamedStats struct { + Name string `json:"name"` + Data *schema.MetricStatistics `json:"data"` +} + +type NamedStatsWithScope struct { + Name string `json:"name"` + Scope schema.MetricScope `json:"scope"` + Stats []*ScopedStats `json:"stats"` +} + type NodeMetrics struct { Host string `json:"host"` SubCluster string `json:"subCluster"` diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index 10e1b55..2920e0e 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -400,7 +400,7 @@ func (r *queryResolver) JobMetrics(ctx context.Context, id string, metrics []str } // JobStats is the resolver for the jobStats field. -func (r *queryResolver) JobStats(ctx context.Context, id string, metrics []string) ([]*model.JobStats, error) { +func (r *queryResolver) JobStats(ctx context.Context, id string, metrics []string) ([]*model.NamedStats, error) { job, err := r.Query().Job(ctx, id) if err != nil { log.Warnf("Error while querying job %s for metadata", id) @@ -413,11 +413,11 @@ func (r *queryResolver) JobStats(ctx context.Context, id string, metrics []strin return nil, err } - res := []*model.JobStats{} + res := []*model.NamedStats{} for name, md := range data { - res = append(res, &model.JobStats{ - Name: name, - Stats: &md, + res = append(res, &model.NamedStats{ + Name: name, + Data: &md, }) } @@ -425,7 +425,7 @@ func (r *queryResolver) JobStats(ctx context.Context, id string, metrics []strin } // ScopedJobStats is the resolver for the scopedJobStats field. -func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope) ([]*model.JobStatsWithScope, error) { +func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope) ([]*model.NamedStatsWithScope, error) { job, err := r.Query().Job(ctx, id) if err != nil { log.Warnf("Error while querying job %s for metadata", id) @@ -438,7 +438,7 @@ func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics [ return nil, err } - res := make([]*model.JobStatsWithScope, 0) + res := make([]*model.NamedStatsWithScope, 0) for name, scoped := range data { for scope, stats := range scoped { @@ -451,7 +451,7 @@ func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics [ }) } - res = append(res, &model.JobStatsWithScope{ + res = append(res, &model.NamedStatsWithScope{ Name: name, Scope: scope, Stats: mdlStats, @@ -462,12 +462,6 @@ func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics [ return res, nil } -// JobsFootprints is the resolver for the jobsFootprints field. -func (r *queryResolver) JobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) { - // NOTE: Legacy Naming! This resolver is for normalized histograms in analysis view only - *Not* related to DB "footprint" column! - return r.jobsFootprints(ctx, filter, metrics) -} - // Jobs is the resolver for the jobs field. func (r *queryResolver) Jobs(ctx context.Context, filter []*model.JobFilter, page *model.PageRequest, order *model.OrderByInput) (*model.JobResultList, error) { if page == nil { @@ -589,6 +583,52 @@ func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobF return stats, nil } +// JobsMetricStats is the resolver for the jobsMetricStats field. +func (r *queryResolver) JobsMetricStats(ctx context.Context, filter []*model.JobFilter, metrics []string) ([]*model.JobStats, error) { + // No Paging, Fixed Order by StartTime ASC + order := &model.OrderByInput{ + Field: "startTime", + Type: "col", + Order: "ASC", + } + + jobs, err := r.Repo.QueryJobs(ctx, filter, nil, order) + if err != nil { + log.Warn("Error while querying jobs for comparison") + return nil, err + } + + res := []*model.JobStats{} + for _, job := range jobs { + data, err := metricDataDispatcher.LoadJobStats(job, metrics, ctx) + if err != nil { + log.Warnf("Error while loading comparison jobStats data for job id %d", job.JobID) + continue + // return nil, err + } + + sres := []*model.NamedStats{} + for name, md := range data { + sres = append(sres, &model.NamedStats{ + Name: name, + Data: &md, + }) + } + + res = append(res, &model.JobStats{ + JobID: int(job.JobID), + Stats: sres, + }) + } + return res, err +} + +// JobsFootprints is the resolver for the jobsFootprints field. +func (r *queryResolver) JobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) { + // NOTE: Legacy Naming! This resolver is for normalized histograms in analysis view only - *Not* related to DB "footprint" column! + return r.jobsFootprints(ctx, filter, metrics) +} + // RooflineHeatmap is the resolver for the rooflineHeatmap field. func (r *queryResolver) RooflineHeatmap(ctx context.Context, filter []*model.JobFilter, rows int, cols int, minX float64, minY float64, maxX float64, maxY float64) ([][]float64, error) { return r.rooflineHeatmap(ctx, filter, rows, cols, minX, minY, maxX, maxY) diff --git a/web/frontend/src/Jobs.root.svelte b/web/frontend/src/Jobs.root.svelte index 7faa8b8..718ccda 100644 --- a/web/frontend/src/Jobs.root.svelte +++ b/web/frontend/src/Jobs.root.svelte @@ -21,6 +21,7 @@ import { init } from "./generic/utils.js"; import Filters from "./generic/Filters.svelte"; import JobList from "./generic/JobList.svelte"; + import JobCompare from "./generic/JobCompare.svelte"; import TextFilter from "./generic/helper/TextFilter.svelte"; import Refresher from "./generic/helper/Refresher.svelte"; import Sorting from "./generic/select/SortSelection.svelte"; @@ -36,7 +37,9 @@ let filterComponent; // see why here: https://stackoverflow.com/questions/58287729/how-can-i-export-a-function-from-a-svelte-component-that-changes-a-value-in-the let jobList, - matchedJobs = null; + jobCompare, + matchedListJobs, + matchedCompareJobs = null; let sorting = { field: "startTime", type: "col", order: "DESC" }, isSortingOpen = false, isMetricsSelectionOpen = false; @@ -49,6 +52,7 @@ : !!ccconfig.plot_list_showFootprint; let selectedCluster = filterPresets?.cluster ? filterPresets.cluster : null; let presetProject = filterPresets?.project ? filterPresets.project : "" + let showCompare = false; // The filterPresets are handled by the Filters component, // so we need to wait for it to be ready before we can start a query. @@ -72,7 +76,7 @@ {/if} - + - + { selectedCluster = detail.filters[0]?.cluster ? detail.filters[0].cluster.eq : null; - jobList.queryJobs(detail.filters); + if (showCompare) { + jobCompare.queryJobs(detail.filters); + } else { + jobList.queryJobs(detail.filters); + } }} /> - + filterComponent.updateFilters(detail)} /> - + { - jobList.refreshJobs() - jobList.refreshAllMetrics() + if (showCompare) { + jobCompare.refreshJobs() + jobCompare.refreshAllMetrics() + } else { + jobList.refreshJobs() + jobList.refreshAllMetrics() + } }} /> + + + - + - + {#if !showCompare} + + {:else} + + {/if} diff --git a/web/frontend/src/generic/JobCompare.svelte b/web/frontend/src/generic/JobCompare.svelte new file mode 100644 index 0000000..9ba1bbf --- /dev/null +++ b/web/frontend/src/generic/JobCompare.svelte @@ -0,0 +1,156 @@ + + + + +{#if $compareData.fetching} + + + + + +{:else if $compareData.error} + + +

{$compareData.error.message}

 + + +{:else} + {#each $compareData.data.jobsMetricStats as job (job.jobId)} + + {job.jobId} + {#each job.stats as stat (stat.name)} + {stat.name} + Min {stat.data.min} + Avg {stat.data.avg} + Max {stat.data.max} + {/each} + +
+ {:else} +
No jobs found
+ {/each} +{/if} \ No newline at end of file diff --git a/web/frontend/src/generic/JobList.svelte b/web/frontend/src/generic/JobList.svelte index 89b8fad..b31a496 100644 --- a/web/frontend/src/generic/JobList.svelte +++ b/web/frontend/src/generic/JobList.svelte @@ -35,7 +35,7 @@ } export let sorting = { field: "startTime", type: "col", order: "DESC" }; - export let matchedJobs = 0; + export let matchedListJobs = 0; export let metrics = ccconfig.plot_list_selectedMetrics; export let showFootprint; @@ -141,7 +141,7 @@ } } - $: matchedJobs = $jobsStore.data != null ? $jobsStore.data.jobs.count : -1; + $: matchedListJobs = $jobsStore.data != null ? $jobsStore.data.jobs.count : -1; // Force refresh list with existing unchanged variables (== usually would not trigger reactivity) export function refreshJobs() { @@ -310,7 +310,7 @@ bind:page {itemsPerPage} itemText="Jobs" - totalItems={matchedJobs} + totalItems={matchedListJobs} on:update-paging={({ detail }) => { if (detail.itemsPerPage != itemsPerPage) { updateConfiguration(detail.itemsPerPage.toString(), detail.page); diff --git a/web/frontend/src/generic/plots/Polar.svelte b/web/frontend/src/generic/plots/Polar.svelte index 765667a..9ae693a 100644 --- a/web/frontend/src/generic/plots/Polar.svelte +++ b/web/frontend/src/generic/plots/Polar.svelte @@ -55,7 +55,7 @@ const getValues = (type) => labels.map(name => { // Peak is adapted and scaled for job shared state const peak = polarMetrics.find(m => m?.name == name)?.peak - const metric = polarData.find(m => m?.name == name)?.stats + const metric = polarData.find(m => m?.name == name)?.data const value = (peak && metric) ? (metric[type] / peak) : 0 return value <= 1. ? value : 1. }) diff --git a/web/frontend/src/job/jobsummary/JobFootprintPolar.svelte b/web/frontend/src/job/jobsummary/JobFootprintPolar.svelte index fe6693b..4048b2b 100644 --- a/web/frontend/src/job/jobsummary/JobFootprintPolar.svelte +++ b/web/frontend/src/job/jobsummary/JobFootprintPolar.svelte @@ -42,7 +42,7 @@ query ($dbid: ID!, $selectedMetrics: [String!]!) { jobStats(id: $dbid, metrics: $selectedMetrics) { name - stats { + data { min avg max From 1c84bcae35eebb91571c06d38f1c93a81564cf6c Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 29 Apr 2025 18:40:44 +0200 Subject: [PATCH 67/82] add filterBuffer for seamless view switch --- web/frontend/src/Jobs.root.svelte | 6 +++++- web/frontend/src/generic/JobCompare.svelte | 3 ++- web/frontend/src/generic/JobList.svelte | 3 ++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/web/frontend/src/Jobs.root.svelte b/web/frontend/src/Jobs.root.svelte index 718ccda..57f0b5b 100644 --- a/web/frontend/src/Jobs.root.svelte +++ b/web/frontend/src/Jobs.root.svelte @@ -36,6 +36,7 @@ export let roles; let filterComponent; // see why here: https://stackoverflow.com/questions/58287729/how-can-i-export-a-function-from-a-svelte-component-that-changes-a-value-in-the + let filterBuffer = []; let jobList, jobCompare, matchedListJobs, @@ -100,6 +101,7 @@ selectedCluster = detail.filters[0]?.cluster ? detail.filters[0].cluster.eq : null; + filterBuffer = [...detail.filters] if (showCompare) { jobCompare.queryJobs(detail.filters); } else { @@ -131,7 +133,7 @@ @@ -146,12 +148,14 @@ bind:sorting bind:matchedListJobs bind:showFootprint + {filterBuffer} /> {:else} {/if} diff --git a/web/frontend/src/generic/JobCompare.svelte b/web/frontend/src/generic/JobCompare.svelte index 9ba1bbf..17da18f 100644 --- a/web/frontend/src/generic/JobCompare.svelte +++ b/web/frontend/src/generic/JobCompare.svelte @@ -32,9 +32,10 @@ } export let matchedCompareJobs = 0; + export let filterBuffer = []; export let metrics = ccconfig.plot_list_selectedMetrics; - let filter = []; + let filter = [...filterBuffer]; const sorting = { field: "startTime", type: "col", order: "DESC" }; /* GQL */ diff --git a/web/frontend/src/generic/JobList.svelte b/web/frontend/src/generic/JobList.svelte index b31a496..03044f0 100644 --- a/web/frontend/src/generic/JobList.svelte +++ b/web/frontend/src/generic/JobList.svelte @@ -38,12 +38,13 @@ export let matchedListJobs = 0; export let metrics = ccconfig.plot_list_selectedMetrics; export let showFootprint; + export let filterBuffer = []; let usePaging = ccconfig.job_list_usePaging let itemsPerPage = usePaging ? ccconfig.plot_list_jobsPerPage : 10; let page = 1; let paging = { itemsPerPage, page }; - let filter = []; + let filter = [...filterBuffer]; let lastFilter = []; let lastSorting = null; let triggerMetricRefresh = false; From 1d13d3dccf3389ec0925c704d63fc551d5841eb8 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 5 May 2025 11:26:39 +0200 Subject: [PATCH 68/82] add and integrate job comparison plot component --- web/frontend/src/generic/JobCompare.svelte | 52 +++- .../src/generic/plots/Comparogram.svelte | 281 ++++++++++++++++++ 2 files changed, 331 insertions(+), 2 deletions(-) create mode 100644 web/frontend/src/generic/plots/Comparogram.svelte diff --git a/web/frontend/src/generic/JobCompare.svelte b/web/frontend/src/generic/JobCompare.svelte index 17da18f..345bfa8 100644 --- a/web/frontend/src/generic/JobCompare.svelte +++ b/web/frontend/src/generic/JobCompare.svelte @@ -19,12 +19,13 @@ queryStore, gql, getContextClient, - mutationStore, + // mutationStore, } from "@urql/svelte"; import { Row, Col, Card, Spinner } from "@sveltestrap/sveltestrap"; + import Comparogram from "./plots/Comparogram.svelte"; const ccconfig = getContext("cc-config"), - initialized = getContext("initialized"), + // initialized = getContext("initialized"), globalMetrics = getContext("globalMetrics"); const equalsCheck = (a, b) => { @@ -36,6 +37,8 @@ export let metrics = ccconfig.plot_list_selectedMetrics; let filter = [...filterBuffer]; + let comparePlotData = {}; + let jobIds = []; const sorting = { field: "startTime", type: "col", order: "DESC" }; /* GQL */ @@ -58,6 +61,8 @@ } `; + /* REACTIVES */ + $: compareData = queryStore({ client: client, query: compareQuery, @@ -65,6 +70,11 @@ }); $: matchedCompareJobs = $compareData.data != null ? $compareData.data.jobsMetricStats.length : -1; + $: if ($compareData.data != null) { + jobIds = []; + comparePlotData = {} + jobs2uplot($compareData.data.jobsMetricStats, metrics) + } /* FUNCTIONS */ // Force refresh list with existing unchanged variables (== usually would not trigger reactivity) @@ -96,6 +106,32 @@ } } + function jobs2uplot(jobs, metrics) { + // Prep + for (let m of metrics) { + // Get Unit + const rawUnit = globalMetrics.find((gm) => gm.name == m)?.unit + const metricUnit = (rawUnit?.prefix ? rawUnit.prefix : "") + (rawUnit?.base ? rawUnit.base : "") + // Init + comparePlotData[m] = {unit: metricUnit, data: [[],[],[],[]]} // data: [X, Y1, Y2, Y3] + } + + // Iterate jobs if exists + if (jobs) { + let plotIndex = 0 + jobs.forEach((j) => { + jobIds.push(j.jobId) + for (let s of j.stats) { + comparePlotData[s.name].data[0].push(plotIndex) + comparePlotData[s.name].data[1].push(s.data.min) + comparePlotData[s.name].data[2].push(s.data.avg) + comparePlotData[s.name].data[3].push(s.data.max) + } + plotIndex++ + }) + } +} + // Adapt for Persisting Job Selections in DB later down the line // const updateConfigurationMutation = ({ name, value }) => { // return mutationStore({ @@ -140,6 +176,18 @@ {:else} + {#each metrics as m} + + {/each} +
{#each $compareData.data.jobsMetricStats as job (job.jobId)} {job.jobId} diff --git a/web/frontend/src/generic/plots/Comparogram.svelte b/web/frontend/src/generic/plots/Comparogram.svelte new file mode 100644 index 0000000..92db086 --- /dev/null +++ b/web/frontend/src/generic/plots/Comparogram.svelte @@ -0,0 +1,281 @@ + + + + + +{#if data && data[0].length > 0} +
+{:else} + Cannot render plot: No series data returned for {metric} +{/if} From fd52fdd35bd87a876be3cae79431c89e3523ee24 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 5 May 2025 16:41:05 +0200 Subject: [PATCH 69/82] add job starttime to legend --- api/schema.graphqls | 1 + internal/graph/generated/generated.go | 64 ++++++++++++++++++- internal/graph/model/models_gen.go | 5 +- internal/graph/schema.resolvers.go | 5 +- web/frontend/src/generic/JobCompare.svelte | 11 +++- .../src/generic/plots/Comparogram.svelte | 3 +- 6 files changed, 80 insertions(+), 9 deletions(-) diff --git a/api/schema.graphqls b/api/schema.graphqls index 1942454..b911d07 100644 --- a/api/schema.graphqls +++ b/api/schema.graphqls @@ -172,6 +172,7 @@ type ScopedStats { type JobStats { jobId: Int! + startTime: Int! stats: [NamedStats!]! } diff --git a/internal/graph/generated/generated.go b/internal/graph/generated/generated.go index 14d1b57..0671d48 100644 --- a/internal/graph/generated/generated.go +++ b/internal/graph/generated/generated.go @@ -171,8 +171,9 @@ type ComplexityRoot struct { } JobStats struct { - JobID func(childComplexity int) int - Stats func(childComplexity int) int + JobID func(childComplexity int) int + StartTime func(childComplexity int) int + Stats func(childComplexity int) int } JobsStatistics struct { @@ -947,6 +948,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.JobStats.JobID(childComplexity), true + case "JobStats.startTime": + if e.complexity.JobStats.StartTime == nil { + break + } + + return e.complexity.JobStats.StartTime(childComplexity), true + case "JobStats.stats": if e.complexity.JobStats.Stats == nil { break @@ -2281,6 +2289,7 @@ type ScopedStats { type JobStats { jobId: Int! + startTime: Int! stats: [NamedStats!]! } @@ -7400,6 +7409,50 @@ func (ec *executionContext) fieldContext_JobStats_jobId(_ context.Context, field return fc, nil } +func (ec *executionContext) _JobStats_startTime(ctx context.Context, field graphql.CollectedField, obj *model.JobStats) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_JobStats_startTime(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.StartTime, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(int) + fc.Result = res + return ec.marshalNInt2int(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_JobStats_startTime(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "JobStats", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type Int does not have child fields") + }, + } + return fc, nil +} + func (ec *executionContext) _JobStats_stats(ctx context.Context, field graphql.CollectedField, obj *model.JobStats) (ret graphql.Marshaler) { fc, err := ec.fieldContext_JobStats_stats(ctx, field) if err != nil { @@ -11197,6 +11250,8 @@ func (ec *executionContext) fieldContext_Query_jobsMetricStats(ctx context.Conte switch field.Name { case "jobId": return ec.fieldContext_JobStats_jobId(ctx, field) + case "startTime": + return ec.fieldContext_JobStats_startTime(ctx, field) case "stats": return ec.fieldContext_JobStats_stats(ctx, field) } @@ -17527,6 +17582,11 @@ func (ec *executionContext) _JobStats(ctx context.Context, sel ast.SelectionSet, if out.Values[i] == graphql.Null { out.Invalids++ } + case "startTime": + out.Values[i] = ec._JobStats_startTime(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } case "stats": out.Values[i] = ec._JobStats_stats(ctx, field, obj) if out.Values[i] == graphql.Null { diff --git a/internal/graph/model/models_gen.go b/internal/graph/model/models_gen.go index fdd3bf3..4cadf22 100644 --- a/internal/graph/model/models_gen.go +++ b/internal/graph/model/models_gen.go @@ -97,8 +97,9 @@ type JobResultList struct { } type JobStats struct { - JobID int `json:"jobId"` - Stats []*NamedStats `json:"stats"` + JobID int `json:"jobId"` + StartTime int `json:"startTime"` + Stats []*NamedStats `json:"stats"` } type JobsStatistics struct { diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index 2920e0e..a93a67e 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -616,8 +616,9 @@ func (r *queryResolver) JobsMetricStats(ctx context.Context, filter []*model.Job } res = append(res, &model.JobStats{ - JobID: int(job.JobID), - Stats: sres, + JobID: int(job.JobID), + StartTime: int(job.StartTime.Unix()), + Stats: sres, }) } return res, err diff --git a/web/frontend/src/generic/JobCompare.svelte b/web/frontend/src/generic/JobCompare.svelte index 345bfa8..9c1ff94 100644 --- a/web/frontend/src/generic/JobCompare.svelte +++ b/web/frontend/src/generic/JobCompare.svelte @@ -39,6 +39,7 @@ let filter = [...filterBuffer]; let comparePlotData = {}; let jobIds = []; + let jobStarts = []; const sorting = { field: "startTime", type: "col", order: "DESC" }; /* GQL */ @@ -49,6 +50,7 @@ query ($filter: [JobFilter!]!, $metrics: [String!]!) { jobsMetricStats(filter: $filter, metrics: $metrics) { jobId + startTime stats { name data { @@ -72,6 +74,7 @@ $: matchedCompareJobs = $compareData.data != null ? $compareData.data.jobsMetricStats.length : -1; $: if ($compareData.data != null) { jobIds = []; + jobStarts = []; comparePlotData = {} jobs2uplot($compareData.data.jobsMetricStats, metrics) } @@ -121,7 +124,9 @@ let plotIndex = 0 jobs.forEach((j) => { jobIds.push(j.jobId) + jobStarts.push(j.startTime) for (let s of j.stats) { + // comparePlotData[s.name].data[0].push(j.startTime) comparePlotData[s.name].data[0].push(plotIndex) comparePlotData[s.name].data[1].push(s.data.min) comparePlotData[s.name].data[2].push(s.data.avg) @@ -181,6 +186,7 @@ title={'Compare '+ m} xlabel="JobIds" xticks={jobIds} + xtimes={jobStarts} ylabel={m} metric={m} yunit={comparePlotData[m].unit} @@ -188,9 +194,10 @@ /> {/each}
- {#each $compareData.data.jobsMetricStats as job (job.jobId)} + {#each $compareData.data.jobsMetricStats as job, jindex (job.jobId)} - {job.jobId} + {jindex}: {job.jobId} + {new Date(job.startTime * 1000)} {#each job.stats as stat (stat.name)} {stat.name} Min {stat.data.min} diff --git a/web/frontend/src/generic/plots/Comparogram.svelte b/web/frontend/src/generic/plots/Comparogram.svelte index 92db086..386434d 100644 --- a/web/frontend/src/generic/plots/Comparogram.svelte +++ b/web/frontend/src/generic/plots/Comparogram.svelte @@ -24,6 +24,7 @@ export let data; export let xlabel; export let xticks; + export let xtimes; export let ylabel; export let yunit; export let title; @@ -120,7 +121,7 @@ { label: "JobID", value: (u, ts, sidx, didx) => { - return xticks[didx]; + return xticks[didx] + ' (' + new Date(xtimes[didx] * 1000).toLocaleString() + ')'; }, } ]; From 33ecfe88ef52102f5796c84c30e8cefa5cc38fec Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 6 May 2025 09:58:28 +0200 Subject: [PATCH 70/82] add job duration, add starttime and duration to legend --- api/schema.graphqls | 1 + internal/graph/generated/generated.go | 60 ++++++++++++ internal/graph/model/models_gen.go | 1 + internal/graph/schema.resolvers.go | 1 + web/frontend/src/generic/JobCompare.svelte | 16 ++-- .../src/generic/plots/Comparogram.svelte | 94 +++++++++++++------ 6 files changed, 137 insertions(+), 36 deletions(-) diff --git a/api/schema.graphqls b/api/schema.graphqls index b911d07..ca8ab95 100644 --- a/api/schema.graphqls +++ b/api/schema.graphqls @@ -173,6 +173,7 @@ type ScopedStats { type JobStats { jobId: Int! startTime: Int! + duration: Int! stats: [NamedStats!]! } diff --git a/internal/graph/generated/generated.go b/internal/graph/generated/generated.go index 0671d48..1eaf841 100644 --- a/internal/graph/generated/generated.go +++ b/internal/graph/generated/generated.go @@ -171,6 +171,7 @@ type ComplexityRoot struct { } JobStats struct { + Duration func(childComplexity int) int JobID func(childComplexity int) int StartTime func(childComplexity int) int Stats func(childComplexity int) int @@ -941,6 +942,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.JobResultList.Offset(childComplexity), true + case "JobStats.duration": + if e.complexity.JobStats.Duration == nil { + break + } + + return e.complexity.JobStats.Duration(childComplexity), true + case "JobStats.jobId": if e.complexity.JobStats.JobID == nil { break @@ -2290,6 +2298,7 @@ type ScopedStats { type JobStats { jobId: Int! startTime: Int! + duration: Int! stats: [NamedStats!]! } @@ -7453,6 +7462,50 @@ func (ec *executionContext) fieldContext_JobStats_startTime(_ context.Context, f return fc, nil } +func (ec *executionContext) _JobStats_duration(ctx context.Context, field graphql.CollectedField, obj *model.JobStats) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_JobStats_duration(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.Duration, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(int) + fc.Result = res + return ec.marshalNInt2int(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_JobStats_duration(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "JobStats", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type Int does not have child fields") + }, + } + return fc, nil +} + func (ec *executionContext) _JobStats_stats(ctx context.Context, field graphql.CollectedField, obj *model.JobStats) (ret graphql.Marshaler) { fc, err := ec.fieldContext_JobStats_stats(ctx, field) if err != nil { @@ -11252,6 +11305,8 @@ func (ec *executionContext) fieldContext_Query_jobsMetricStats(ctx context.Conte return ec.fieldContext_JobStats_jobId(ctx, field) case "startTime": return ec.fieldContext_JobStats_startTime(ctx, field) + case "duration": + return ec.fieldContext_JobStats_duration(ctx, field) case "stats": return ec.fieldContext_JobStats_stats(ctx, field) } @@ -17587,6 +17642,11 @@ func (ec *executionContext) _JobStats(ctx context.Context, sel ast.SelectionSet, if out.Values[i] == graphql.Null { out.Invalids++ } + case "duration": + out.Values[i] = ec._JobStats_duration(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } case "stats": out.Values[i] = ec._JobStats_stats(ctx, field, obj) if out.Values[i] == graphql.Null { diff --git a/internal/graph/model/models_gen.go b/internal/graph/model/models_gen.go index 4cadf22..d4486fc 100644 --- a/internal/graph/model/models_gen.go +++ b/internal/graph/model/models_gen.go @@ -99,6 +99,7 @@ type JobResultList struct { type JobStats struct { JobID int `json:"jobId"` StartTime int `json:"startTime"` + Duration int `json:"duration"` Stats []*NamedStats `json:"stats"` } diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index a93a67e..771565b 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -618,6 +618,7 @@ func (r *queryResolver) JobsMetricStats(ctx context.Context, filter []*model.Job res = append(res, &model.JobStats{ JobID: int(job.JobID), StartTime: int(job.StartTime.Unix()), + Duration: int(job.Duration), Stats: sres, }) } diff --git a/web/frontend/src/generic/JobCompare.svelte b/web/frontend/src/generic/JobCompare.svelte index 9c1ff94..cd37a07 100644 --- a/web/frontend/src/generic/JobCompare.svelte +++ b/web/frontend/src/generic/JobCompare.svelte @@ -39,7 +39,6 @@ let filter = [...filterBuffer]; let comparePlotData = {}; let jobIds = []; - let jobStarts = []; const sorting = { field: "startTime", type: "col", order: "DESC" }; /* GQL */ @@ -51,6 +50,7 @@ jobsMetricStats(filter: $filter, metrics: $metrics) { jobId startTime + duration stats { name data { @@ -74,7 +74,6 @@ $: matchedCompareJobs = $compareData.data != null ? $compareData.data.jobsMetricStats.length : -1; $: if ($compareData.data != null) { jobIds = []; - jobStarts = []; comparePlotData = {} jobs2uplot($compareData.data.jobsMetricStats, metrics) } @@ -116,7 +115,7 @@ const rawUnit = globalMetrics.find((gm) => gm.name == m)?.unit const metricUnit = (rawUnit?.prefix ? rawUnit.prefix : "") + (rawUnit?.base ? rawUnit.base : "") // Init - comparePlotData[m] = {unit: metricUnit, data: [[],[],[],[]]} // data: [X, Y1, Y2, Y3] + comparePlotData[m] = {unit: metricUnit, data: [[],[],[],[],[],[]]} // data: [X, XST, XRT, YMIN, YAVG, YMAX] } // Iterate jobs if exists @@ -124,13 +123,13 @@ let plotIndex = 0 jobs.forEach((j) => { jobIds.push(j.jobId) - jobStarts.push(j.startTime) for (let s of j.stats) { - // comparePlotData[s.name].data[0].push(j.startTime) comparePlotData[s.name].data[0].push(plotIndex) - comparePlotData[s.name].data[1].push(s.data.min) - comparePlotData[s.name].data[2].push(s.data.avg) - comparePlotData[s.name].data[3].push(s.data.max) + comparePlotData[s.name].data[1].push(j.startTime) + comparePlotData[s.name].data[2].push(j.duration) + comparePlotData[s.name].data[3].push(s.data.min) + comparePlotData[s.name].data[4].push(s.data.avg) + comparePlotData[s.name].data[5].push(s.data.max) } plotIndex++ }) @@ -186,7 +185,6 @@ title={'Compare '+ m} xlabel="JobIds" xticks={jobIds} - xtimes={jobStarts} ylabel={m} metric={m} yunit={comparePlotData[m].unit} diff --git a/web/frontend/src/generic/plots/Comparogram.svelte b/web/frontend/src/generic/plots/Comparogram.svelte index 386434d..da31b5c 100644 --- a/web/frontend/src/generic/plots/Comparogram.svelte +++ b/web/frontend/src/generic/plots/Comparogram.svelte @@ -24,15 +24,15 @@ export let data; export let xlabel; export let xticks; - export let xtimes; export let ylabel; export let yunit; export let title; // export let cluster = ""; // export let subCluster = ""; - // $: console.log('LABEL:', metric, yunit) - // $: console.log('DATA:', data) + $: console.log('LABEL:', metric, yunit) + $: console.log('DATA:', data) + $: console.log('XTICKS:', xticks) const metricConfig = null // DEBUG FILLER // const metricConfig = getContext("getMetricConfig")(cluster, subCluster, metric); // Args woher @@ -40,6 +40,22 @@ const lineWidth = clusterCockpitConfig.plot_general_lineWidth / window.devicePixelRatio; const cbmode = clusterCockpitConfig?.plot_general_colorblindMode || false; + // Format Seconds to hh:mm + function formatTime(t) { + if (t !== null) { + if (isNaN(t)) { + return t; + } else { + const tAbs = Math.abs(t); + const h = Math.floor(tAbs / 3600); + const m = Math.floor((tAbs % 3600) / 60); + if (h == 0) return `${m}m`; + else if (m == 0) return `${h}h`; + else return `${h}:${m}h`; + } + } + } + // UPLOT PLUGIN // converts the legend into a simple tooltip function legendAsTooltipPlugin({ className, @@ -120,34 +136,48 @@ const plotSeries = [ { label: "JobID", + scale: "x", value: (u, ts, sidx, didx) => { - return xticks[didx] + ' (' + new Date(xtimes[didx] * 1000).toLocaleString() + ')'; + return xticks[didx]; }, + }, + { + label: "Starttime", + scale: "xst", + value: (u, ts, sidx, didx) => { + return new Date(ts * 1000).toLocaleString(); + }, + }, + { + label: "Duration", + scale: "xrt", + value: (u, ts, sidx, didx) => { + return formatTime(ts); + }, + }, + { + label: "Min", + scale: "y", + width: lineWidth, + stroke: cbmode ? "rgb(0,255,0)" : "red", + }, + { + label: "Avg", + scale: "y", + width: lineWidth, + stroke: "black", + }, + { + label: "Max", + scale: "y", + width: lineWidth, + stroke: cbmode ? "rgb(0,0,255)" : "green", } ]; - plotSeries.push({ - label: "min", - scale: "y", - width: lineWidth, - stroke: cbmode ? "rgb(0,255,0)" : "red", - }); - plotSeries.push({ - label: "avg", - scale: "y", - width: lineWidth, - stroke: "black", - }); - plotSeries.push({ - label: "max", - scale: "y", - width: lineWidth, - stroke: cbmode ? "rgb(0,0,255)" : "green", - }); - const plotBands = [ - { series: [3, 2], fill: cbmode ? "rgba(0,0,255,0.1)" : "rgba(0,255,0,0.1)" }, - { series: [2, 1], fill: cbmode ? "rgba(0,255,0,0.1)" : "rgba(255,0,0,0.1)" }, + { series: [5, 4], fill: cbmode ? "rgba(0,0,255,0.1)" : "rgba(0,255,0,0.1)" }, + { series: [4, 3], fill: cbmode ? "rgba(0,255,0,0.1)" : "rgba(255,0,0,0.1)" }, ]; const opts = { @@ -167,6 +197,14 @@ return splits.map(s => xticks[s]); } }, + { + scale: "xst", + show: false, + }, + { + scale: "xrt", + show: false, + }, { scale: "y", grid: { show: true }, @@ -180,8 +218,8 @@ draw: [ (u) => { // Draw plot type label: - let textl = "Jobs min/avg/max"; - let textr = ""; + let textl = "Metric Min/Avg/Max in Duration"; + let textr = "Earlier <- StartTime -> Later"; u.ctx.save(); u.ctx.textAlign = "start"; // 'end' u.ctx.fillStyle = "black"; @@ -216,6 +254,8 @@ }, scales: { x: { time: false }, + xst: { time: false }, + xrt: { time: false }, y: maxY ? { min: 0, max: (maxY * 1.1) } : {auto: true}, // Add some space to upper render limit }, legend: { From d3d752f90cae1bcd2bd1470c6e19d6cc07d5b426 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 6 May 2025 10:46:30 +0200 Subject: [PATCH 71/82] finalize compareplot prototype, move formattime to units.js --- web/frontend/src/generic/JobCompare.svelte | 4 ++- .../src/generic/plots/Comparogram.svelte | 35 +++++++------------ .../src/generic/plots/Histogram.svelte | 17 +-------- .../src/generic/plots/MetricPlot.svelte | 18 +--------- web/frontend/src/generic/units.js | 20 +++++++++++ 5 files changed, 37 insertions(+), 57 deletions(-) diff --git a/web/frontend/src/generic/JobCompare.svelte b/web/frontend/src/generic/JobCompare.svelte index cd37a07..ba40bcc 100644 --- a/web/frontend/src/generic/JobCompare.svelte +++ b/web/frontend/src/generic/JobCompare.svelte @@ -22,6 +22,7 @@ // mutationStore, } from "@urql/svelte"; import { Row, Col, Card, Spinner } from "@sveltestrap/sveltestrap"; + import { formatTime } from "./units.js"; import Comparogram from "./plots/Comparogram.svelte"; const ccconfig = getContext("cc-config"), @@ -195,7 +196,8 @@ {#each $compareData.data.jobsMetricStats as job, jindex (job.jobId)} {jindex}: {job.jobId} - {new Date(job.startTime * 1000)} + {new Date(job.startTime * 1000).toISOString()} + {formatTime(job.duration)} {#each job.stats as stat (stat.name)} {stat.name} Min {stat.data.min} diff --git a/web/frontend/src/generic/plots/Comparogram.svelte b/web/frontend/src/generic/plots/Comparogram.svelte index da31b5c..b4d8d77 100644 --- a/web/frontend/src/generic/plots/Comparogram.svelte +++ b/web/frontend/src/generic/plots/Comparogram.svelte @@ -14,7 +14,7 @@ @@ -80,7 +85,7 @@ - + + + + @@ -148,6 +162,7 @@ bind:sorting bind:matchedListJobs bind:showFootprint + bind:selectedJobs {filterBuffer} /> {:else} @@ -161,7 +176,7 @@ - + Math.floor(Date.parse(rfc3339) / 1000); - let opts = []; if (filters.cluster) opts.push(`cluster=${filters.cluster}`); if (filters.node) opts.push(`node=${filters.node}`); @@ -196,6 +198,11 @@ if (filters.startTime.range) { opts.push(`startTime=${filters.startTime.range}`) } + if (filters.dbId.length != 0) { + for (let dbi of filters.dbId) { + opts.push(`dbId=${dbi}`); + } + } if (filters.jobId.length != 0) if (filters.jobIdMatch != "in") { opts.push(`jobId=${filters.jobId}`); diff --git a/web/frontend/src/generic/JobList.svelte b/web/frontend/src/generic/JobList.svelte index 03044f0..e6ae6f6 100644 --- a/web/frontend/src/generic/JobList.svelte +++ b/web/frontend/src/generic/JobList.svelte @@ -39,6 +39,7 @@ export let metrics = ccconfig.plot_list_selectedMetrics; export let showFootprint; export let filterBuffer = []; + export let selectedJobs = []; let usePaging = ccconfig.job_list_usePaging let itemsPerPage = usePaging ? ccconfig.plot_list_jobsPerPage : 10; @@ -285,7 +286,10 @@ {:else} {#each jobs as job (job)} - + selectedJobs = [...selectedJobs, detail]} + on:unselect-job={({detail}) => selectedJobs = selectedJobs.filter(item => item !== detail)} + /> {:else} No jobs found diff --git a/web/frontend/src/generic/joblist/JobInfo.svelte b/web/frontend/src/generic/joblist/JobInfo.svelte index 8917653..f5cb066 100644 --- a/web/frontend/src/generic/joblist/JobInfo.svelte +++ b/web/frontend/src/generic/joblist/JobInfo.svelte @@ -18,6 +18,8 @@ export let username = null; export let authlevel= null; export let roles = null; + export let isSelected = null; + export let showSelect = false; function formatDuration(duration) { const hours = Math.floor(duration / 3600); @@ -76,18 +78,39 @@ {job.jobId} ({job.cluster}) - + + { 'Add or Remove Job to/from Comparison Selection' } + {/if} - - - { displayCheck ? 'Copied!' : 'Copy Job ID to Clipboard' } - + + + { displayCheck ? 'Copied!' : 'Copy Job ID to Clipboard' } + + {#if job.metaData?.jobName} {#if job.metaData?.jobName.length <= 25} diff --git a/web/frontend/src/generic/joblist/JobListRow.svelte b/web/frontend/src/generic/joblist/JobListRow.svelte index 82cf2ed..7d94943 100644 --- a/web/frontend/src/generic/joblist/JobListRow.svelte +++ b/web/frontend/src/generic/joblist/JobListRow.svelte @@ -12,7 +12,7 @@