Merge pull request #188 from ClusterCockpit/hotfix

Prepare minor release 1.1.0

Makefile

@@ -2,7 +2,7 @@ TARGET = ./cc-backend
 VAR = ./var
 CFG = config.json .env
 FRONTEND = ./web/frontend
-VERSION = 1.0.0
+VERSION = 1.1.0
 GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development')
 CURRENT_TIME = $(shell date +"%Y-%m-%d:T%H:%M:%S")
 LD_FLAGS = '-s -X main.date=${CURRENT_TIME} -X main.version=${VERSION} -X main.commit=${GIT_HASH}'
@@ -28,7 +28,7 @@ SVELTE_SRC = $(wildcard $(FRONTEND)/src/*.svelte)         \
 			 $(wildcard $(FRONTEND)/src/plots/*.svelte)   \
 			 $(wildcard $(FRONTEND)/src/joblist/*.svelte)
 
-.PHONY: clean test tags frontend $(TARGET)
+.PHONY: clean distclean test tags frontend $(TARGET)
 
 .NOTPARALLEL:
 

ReleaseNotes.md

@@ -1,11 +1,11 @@
-# `cc-backend` version 1.0.0
+# `cc-backend` version 1.1.0
 
 Supports job archive version 1 and database version 6.
 
-This is the initial release of `cc-backend`, the API backend and frontend
+This is a minor release of `cc-backend`, the API backend and frontend
 implementation of ClusterCockpit.
 
-** Breaking changes **
+** Breaking changes v1 **
 
 The aggregate job statistic core hours is now computed using the job table
 column `num_hwthreads`. In a future release this column will be renamed to
@@ -25,12 +25,3 @@ sqlite> PRAGMA foreign_keys = ON;
 
 ```
 Otherwise if you delete jobs the jobtag relation table will not be updated accordingly!
-
-**Notable changes**
-* Supports user roles admin, support, manager, user, and api.
-* Unified search bar supports job id, job name, project id, user name, and name
-* Performance improvements for sqlite db backend
-* Extended REST api supports to query job metrics
-* Better support for shared jobs
-* More flexible metric list configuration
-* Versioning and migration for database and job archive

docs/dev-authentication.md

@@ -0,0 +1,156 @@
+# Overview
+
+The implementation of authentication is not easy to understand by just looking
+at the code. The authentication is implemented in `internal/auth/`. In `auth.go`
+an interface is defined that any authentication provider must fulfill. It also
+acts as a dispatcher to delegate the calls to the available authentication
+providers.
+
+The most important routine are:
+* `CanLogin()` Check if the authentication method is supported for login attempt
+* `Login()` Handle POST request to login user and start a new session
+* `Auth()`  Authenticate user and put User Object in context of the request
+
+The http router calls auth in the following cases:
+* `r.Handle("/login", authentication.Login( ... )).Methods(http.MethodPost)`:
+  The POST request on the `/login` route will call the Login callback.
+* Any route in the secured subrouter will always call Auth(), on success it will
+  call the next handler in the chain, on failure it will render the login
+  template.
+```
+secured.Use(func(next http.Handler) http.Handler {
+	return authentication.Auth(
+		// On success;
+		next,
+
+		// On failure:
+		func(rw http.ResponseWriter, r *http.Request, err error) {
+               // Render login form
+		})
+})
+```
+
+For non API routes a JWT token can be used to initiate an authenticated user
+session. This can either happen by calling the login route with a token
+provided in a header or query URL or via the `Auth()` method on first access
+to a secured URL via a special cookie containing the JWT token.
+For API routes the access is authenticated on every request using the JWT token
+and no session is initiated.
+
+# Login
+
+The Login function (located in `auth.go`):
+* Extracts the user name and gets the user from the user database table. In case the
+  user is not found the user object is set to nil.
+* Iterates over all authenticators and:
+  - Calls the `CanLogin` function which checks if the authentication method is
+    supported for this user and the user object is valid.
+  - Calls the `Login` function to authenticate the user. On success a valid user
+    object is returned.
+  - Creates a new session object, stores the user attributes in the session and
+    saves the session.
+  - Starts the `onSuccess` http handler
+
+## Local authenticator
+
+This authenticator is applied if 
+```
+return user != nil && user.AuthSource == AuthViaLocalPassword
+```
+
+Compares the password provided by the login form to the password hash stored in
+the user database table:
+```
+if e := bcrypt.CompareHashAndPassword([]byte(user.Password), []byte(r.FormValue("password"))); e != nil {
+	log.Errorf("AUTH/LOCAL > Authentication for user %s failed!", user.Username)
+	return nil, fmt.Errorf("AUTH/LOCAL > Authentication failed")
+}
+```
+
+## LDAP authenticator
+
+This authenticator is applied if 
+```
+return user != nil && user.AuthSource == AuthViaLDAP
+```
+
+Gets the LDAP connection and tries a bind with the provided credentials:
+```
+if err := l.Bind(userDn, r.FormValue("password")); err != nil {
+	log.Errorf("AUTH/LOCAL > Authentication for user %s failed: %v", user.Username, err)
+	return nil, fmt.Errorf("AUTH/LDAP > Authentication failed")
+}
+```
+
+## JWT authenticator
+
+Login via JWT token will create a session without password.
+For login the `X-Auth-Token` header is not supported.
+This authenticator is applied if either user is not nil and auth source is
+`AuthViaToken` or the Authorization header is present or the URL query key
+login-token is present:
+```
+return (user != nil && user.AuthSource == AuthViaToken) ||
+        r.Header.Get("Authorization") != "" ||
+        r.URL.Query().Get("login-token") != ""
+```
+
+The Login function:
+* Parses the token
+* Check if the signing method is EdDSA or HS256 or HS512
+* Check if claims are valid and extracts the claims
+* The following claims have to be present:
+   - `sub`: The subject, in this case this is the username
+   - `exp`: Expiration in Unix epoch time
+   - `roles`: String array with roles of user
+* In case user is not yet set, which is usually the case:
+   - Try to fetch user from database
+   - In case user is not yet present add user to user database table with `AuthViaToken` AuthSource.
+* Return valid user object
+
+# Auth
+
+The Auth function (located in `auth.go`):
+* Returns a new http handler function that is defined right away
+* This handler iterates over all authenticators
+* Calls `Auth()` on every authenticator
+* If err is not nil and the user object is valid it puts the user object in the
+  request context and starts the onSuccess http handler
+* Otherwise it calls the onFailure handler
+
+## Local
+
+Calls the `AuthViaSession()` function in `auth.go`. This will extract username,
+projects and roles from the session and initialize a user object with those
+values.
+
+## LDAP
+
+Calls the `AuthViaSession()` function in `auth.go`. This will extract username,
+projects and roles from the session and initialize a user object with those
+values.
+
+# JWT
+
+Check for JWT token:
+* Is token passed in the `X-Auth-Token` or `Authorization` header
+* If no token is found in a header it tries to read the token from a configured
+cookie.
+
+Finally it calls AuthViaSession in `auth.go` if a valid session exists. This is
+true if a JWT token was previously used to initiate a session. In this case the
+user object initialized with the session is returned right away.
+
+In case a token was found extract and parse the token:
+* Check if signing method is Ed25519/EdDSA 
+* In case publicKeyCrossLogin is configured:
+   - Check if `iss` issuer claim matched trusted issuer from configuration
+   - Return public cross login key
+   - Otherwise return standard public key
+* Check if claims are valid
+* Depending on the option `ForceJWTValidationViaDatabase ` the roles are
+  extracted from JWT token or taken from user object fetched from database
+* In case the token was extracted from cookie create a new session and ask the
+  browser to delete the JWT cookie
+* Return valid user object
+

docs/dev-release.md

@@ -0,0 +1,13 @@
+# Steps to prepare a release
+
+1. On `hotfix` branch:
+   * Update ReleaseNotes.md
+   * Update version in Makefile
+   * Commit, push, and pull request
+   * Merge in master
+
+2. On Linux host:
+   * Pull master
+   * Ensure that GitHub Token environment variable `GITHUB_TOKEN` is set
+   * Create release tag: `git tag v1.1.0 -m release`
+   * Execute `goreleaser release`

internal/auth/auth.go

@@ -75,10 +75,7 @@ func getRoleEnum(roleStr string) Role {
 }
 
 func isValidRole(role string) bool {
-	if getRoleEnum(role) == RoleError {
+	return getRoleEnum(role) != RoleError
-		return false
-	}
-	return true
 }
 
 func (u *User) HasValidRole(role string) (hasRole bool, isValid bool) {
@@ -166,16 +163,16 @@ func GetValidRoles(user *User) ([]string, error) {
 	return vals, fmt.Errorf("%s: only admins are allowed to fetch a list of roles", user.Username)
 }
 
-// Called by routerConfig web.page setup in backend: Only requires known user and/or not API user
+// Called by routerConfig web.page setup in backend: Only requires known user
 func GetValidRolesMap(user *User) (map[string]Role, error) {
 	named := make(map[string]Role)
-	if user.HasNotRoles([]Role{RoleApi, RoleAnonymous}) {
+	if user.HasNotRoles([]Role{RoleAnonymous}) {
 		for i := RoleApi; i < RoleError; i++ {
 			named[GetRoleString(i)] = i
 		}
 		return named, nil
 	}
-	return named, fmt.Errorf("Only known users are allowed to fetch a list of roles")
+	return named, fmt.Errorf("only known users are allowed to fetch a list of roles")
 }
 
 // Find highest role
@@ -300,6 +297,7 @@ func (auth *Authentication) AuthViaSession(
 		return nil, nil
 	}
 
+	// TODO Check if keys are present in session?
 	username, _ := session.Values["username"].(string)
 	projects, _ := session.Values["projects"].([]string)
 	roles, _ := session.Values["roles"].([]string)
@@ -320,11 +318,9 @@ func (auth *Authentication) Login(
 		err := errors.New("no authenticator applied")
 		username := r.FormValue("username")
 		user := (*User)(nil)
+
 		if username != "" {
-			if user, _ = auth.GetUser(username); err != nil {
+			user, _ = auth.GetUser(username)
-				// log.Warnf("login of unkown user %v", username)
-				_ = err
-			}
 		}
 
 		for _, authenticator := range auth.authenticators {

internal/auth/jwt.go

@@ -92,7 +92,7 @@ func (ja *JWTAuthenticator) Init(auth *Authentication, conf interface{}) error {
 		}
 	} else {
 		ja.publicKeyCrossLogin = nil
-		log.Warn("environment variable 'CROSS_LOGIN_JWT_PUBLIC_KEY' not set (cross login token based authentication will not work)")
+		log.Debug("environment variable 'CROSS_LOGIN_JWT_PUBLIC_KEY' not set (cross login token based authentication will not work)")
 	}
 
 	return nil
@@ -103,7 +103,9 @@ func (ja *JWTAuthenticator) CanLogin(
 	rw http.ResponseWriter,
 	r *http.Request) bool {
 
-	return (user != nil && user.AuthSource == AuthViaToken) || r.Header.Get("Authorization") != "" || r.URL.Query().Get("login-token") != ""
+	return (user != nil && user.AuthSource == AuthViaToken) ||
+		r.Header.Get("Authorization") != "" ||
+		r.URL.Query().Get("login-token") != ""
 }
 
 func (ja *JWTAuthenticator) Login(
@@ -111,13 +113,9 @@ func (ja *JWTAuthenticator) Login(
 	rw http.ResponseWriter,
 	r *http.Request) (*User, error) {
 
-	rawtoken := r.Header.Get("X-Auth-Token")
+	rawtoken := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
 	if rawtoken == "" {
-		rawtoken = r.Header.Get("Authorization")
+		rawtoken = r.URL.Query().Get("login-token")
-		rawtoken = strings.TrimPrefix(rawtoken, "Bearer ")
-		if rawtoken == "" {
-			rawtoken = r.URL.Query().Get("login-token")
-		}
 	}
 
 	token, err := jwt.Parse(rawtoken, func(t *jwt.Token) (interface{}, error) {
@@ -134,7 +132,7 @@ func (ja *JWTAuthenticator) Login(
 		return nil, err
 	}
 
-	if err := token.Claims.Valid(); err != nil {
+	if err = token.Claims.Valid(); err != nil {
 		log.Warn("jwt token claims are not valid")
 		return nil, err
 	}
@@ -220,7 +218,10 @@ func (ja *JWTAuthenticator) Auth(
 		}
 
 		// Is there more than one public key?
-		if ja.publicKeyCrossLogin != nil && ja.config != nil && ja.config.TrustedExternalIssuer != "" {
+		if ja.publicKeyCrossLogin != nil &&
+			ja.config != nil &&
+			ja.config.TrustedExternalIssuer != "" {
+
 			// Determine whether to use the external public key
 			unvalidatedIssuer, success := t.Claims.(jwt.MapClaims)["iss"].(string)
 			if success && unvalidatedIssuer == ja.config.TrustedExternalIssuer {

internal/routerConfig/routes.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"os"
 	"strconv"
 	"strings"
 	"time"
@@ -15,6 +16,7 @@ import (
 	"github.com/ClusterCockpit/cc-backend/internal/auth"
 	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
+	"github.com/ClusterCockpit/cc-backend/internal/util"
 	"github.com/ClusterCockpit/cc-backend/pkg/log"
 	"github.com/ClusterCockpit/cc-backend/web"
 	"github.com/gorilla/mux"
@@ -60,6 +62,16 @@ func setupHomeRoute(i InfoType, r *http.Request) InfoType {
 	}
 
 	i["clusters"] = stats
+
+	if util.CheckFileExists("./var/notice.txt") {
+		msg, err := os.ReadFile("./var/notice.txt")
+		if err != nil {
+			log.Warnf("failed to read notice.txt file: %s", err.Error())
+		} else {
+			i["message"] = string(msg)
+		}
+	}
+
 	return i
 }
 
@@ -294,6 +306,8 @@ func HandleSearchBar(rw http.ResponseWriter, r *http.Request, buildInfo web.Buil
 				http.Redirect(rw, r, "/monitoring/jobs/?jobName="+url.QueryEscape(strings.Trim(splitSearch[1], " ")), http.StatusFound) // All Users: Redirect to Tablequery
 			case "projectId":
 				http.Redirect(rw, r, "/monitoring/jobs/?projectMatch=eq&project="+url.QueryEscape(strings.Trim(splitSearch[1], " ")), http.StatusFound) // All Users: Redirect to Tablequery
+			case "arrayJobId":
+				http.Redirect(rw, r, "/monitoring/jobs/?arrayJobId="+url.QueryEscape(strings.Trim(splitSearch[1], " ")), http.StatusFound) // All Users: Redirect to Tablequery
 			case "username":
 				if user.HasAnyRole([]auth.Role{auth.RoleAdmin, auth.RoleSupport, auth.RoleManager}) {
 					http.Redirect(rw, r, "/monitoring/users/?user="+url.QueryEscape(strings.Trim(splitSearch[1], " ")), http.StatusFound)

web/frontend/src/Header.svelte

@@ -93,7 +93,7 @@
             <InputGroup>
                 <Input type="text" placeholder="Search 'type:<query>' ..." name="searchId"/>
                 <Button outline type="submit"><Icon name="search"/></Button>
-                <InputGroupText style="cursor:help;" title={(authlevel >= roles.support) ? "Example: 'projectId:a100cd', Types are: jobId | jobName | projectId | username | name" : "Example: 'jobName:myjob', Types are jobId | jobName | projectId"}><Icon name="info-circle"/></InputGroupText>
+                <InputGroupText style="cursor:help;" title={(authlevel >= roles.support) ? "Example: 'projectId:a100cd', Types are: jobId | jobName | projectId | arrayJobId | username | name" : "Example: 'jobName:myjob', Types are jobId | jobName | projectId | arrayJobId "}><Icon name="info-circle"/></InputGroupText>
             </InputGroup>
         </form>
         {#if username}

web/frontend/src/Job.root.svelte

@@ -48,7 +48,8 @@
     `);
 
     const ccconfig = getContext("cc-config"),
-        clusters = getContext("clusters");
+          clusters = getContext("clusters"),
+          metrics  = getContext("metrics")
 
     let isMetricsSelectionOpen = false,
         selectedMetrics = [],
@@ -74,16 +75,26 @@
                 ccconfig[`job_view_nodestats_selectedMetrics`]),
         ]);
 
-        // Select default Scopes to load
+        // Select default Scopes to load: Check before if accelerator metrics are not on accelerator scope by default
-        if (job.numAcc === 0) {
+        const accMetrics = ['acc_utilization', 'acc_mem_used', 'acc_power', 'nv_mem_util', 'nv_sm_clock', 'nv_temp'] 
-            // No Accels
+        const accNodeOnly = [...toFetch].some(function(m) {
+            if (accMetrics.includes(m)) {
+                const mc = metrics(job.cluster, m)
+                return mc.scope !== 'accelerator'
+            } else {
+                return false
+            }
+        })
+
+        if (job.numAcc === 0 || accNodeOnly === true) {
+            // No Accels or Accels on Node Scope
             startFetching(
                 job,
                 [...toFetch],
                 job.numNodes > 2 ? ["node"] : ["node", "core"]
             );
         } else {
-            // Accels
+            // Accels and not on node scope
             startFetching(
                 job,
                 [...toFetch],

web/frontend/src/List.root.svelte

@@ -119,10 +119,10 @@
     <thead>
         <tr>
             <th scope="col">
-                <!-- {({ -->
+                {({
-                <!--     USER: "Username", -->
+                    USER: "Username",
-                <!--     PROJECT: "Project Name", -->
+                    PROJECT: "Project Name",
-                <!-- })[type]} -->
+                })[type]}
                 <Button
                     color={sorting.field == "id" ? "primary" : "light"}
                     size="sm"
@@ -216,14 +216,14 @@
                             >
                         {:else if type == "PROJECT"}
                             <a href="/monitoring/jobs/?project={row.id}"
-                                >{row.id}</a
+                                >{scrambleNames ? scramble(row.id) : row.id}</a
                             >
                         {:else}
                             {row.id}
                         {/if}
                     </td>
                     {#if type == "USER"}
-                        <td>{row?.name ? row.name : ""}</td>
+                        <td>{scrambleNames ? scramble(row?.name?row.name:"-") : row?.name?row.name:"-"}</td>
                     {/if}
                     <td>{row.totalJobs}</td>
                     <td>{row.totalWalltime}</td>

web/frontend/src/joblist/JobInfo.svelte

@@ -7,7 +7,10 @@
  -->
 <script context="module">
     export const scrambleNames = window.localStorage.getItem("cc-scramble-names")
-    export const scramble = (str) => [...str].reduce((x, c, i) => x * 7 + c.charCodeAt(0) * i * 21, 5).toString(32)
+    export const scramble = function(str) {
+        if (str === '-') return str
+        else return [...str].reduce((x, c, i) => x * 7 + c.charCodeAt(0) * i * 21, 5).toString(32).substr(0, 6)
+    }
 </script>
 <script>
     import Tag from '../Tag.svelte';

web/templates/home.tmpl

@@ -1,4 +1,20 @@
 {{define "content"}}
+{{if .Infos.message }}
+<div class="row justify-content-center">
+    <div class="col-6">
+        <div class="alert alert-info p-3" role="alert">
+            <div class="row  align-items-center">
+                <div class="col-2">
+                    <h2><i class="bi-info-circle-fill m-3"></i></h2>
+                </div>
+                <div class="col-10">
+                {{.Infos.message}}
+                </div>
+            </div>
+        </div>
+    </div>
+</div>
+{{end}}
 <div class="row">
     <div class="col">
         <h2>Clusters</h2>

web/web.go

@@ -9,11 +9,11 @@ import (
 	"html/template"
 	"io/fs"
 	"net/http"
-	"os"
 	"strings"
 
 	"github.com/ClusterCockpit/cc-backend/internal/auth"
 	"github.com/ClusterCockpit/cc-backend/internal/config"
+	"github.com/ClusterCockpit/cc-backend/internal/util"
 	"github.com/ClusterCockpit/cc-backend/pkg/log"
 	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 )
@@ -48,7 +48,7 @@ func init() {
 		}
 
 		if path == "templates/login.tmpl" {
-			if _, err := os.Stat("./var/login.tmpl"); err == nil {
+			if util.CheckFileExists("./var/login.tmpl") {
 				log.Info("overwrite login.tmpl with local file")
 				templates[strings.TrimPrefix(path, "templates/")] =
 					template.Must(template.Must(base.Clone()).ParseFiles("./var/login.tmpl"))
@@ -56,7 +56,7 @@ func init() {
 			}
 		}
 		if path == "templates/imprint.tmpl" {
-			if _, err := os.Stat("./var/imprint.tmpl"); err == nil {
+			if util.CheckFileExists("./var/imprint.tmpl") {
 				log.Info("overwrite imprint.tmpl with local file")
 				templates[strings.TrimPrefix(path, "templates/")] =
 					template.Must(template.Must(base.Clone()).ParseFiles("./var/imprint.tmpl"))
@@ -64,7 +64,7 @@ func init() {
 			}
 		}
 		if path == "templates/privacy.tmpl" {
-			if _, err := os.Stat("./var/privacy.tmpl"); err == nil {
+			if util.CheckFileExists("./var/privacy.tmpl") {
 				log.Info("overwrite privacy.tmpl with local file")
 				templates[strings.TrimPrefix(path, "templates/")] =
 					template.Must(template.Must(base.Clone()).ParseFiles("./var/privacy.tmpl"))