ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2025-07-23 04:51:39 +02:00
Code Issues Packages Releases Activity

Merge branch 'master' into 236-user-authentication-using-keycloak-or-any-openid-client-for-using-external-auth-providers-such-as-ldap-github-google

Browse Source
This commit is contained in:
Jan Eitzinger
2024-03-28 12:07:58 +01:00
parent e8fb5a0030 ddd3fad1c6
commit 642fd5cc91
63 changed files with 8867 additions and 7123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
internal/api/rest.go
View File

@@ -114,12 +114,11 @@ type UpdateUserApiResponse struct {
// StopJobApiRequest model
type StopJobApiRequest struct {
// Stop Time of job as epoch
JobId *int64 `json:"jobId" example:"123000"`
Cluster *string `json:"cluster" example:"fritz"`
StartTime *int64 `json:"startTime" example:"1649723812"`
State schema.JobState `json:"jobState" validate:"required" example:"completed"`
StopTime int64 `json:"stopTime" validate:"required" example:"1649763839"`
State schema.JobState `json:"jobState" validate:"required" example:"completed"` // Final job state
JobId *int64 `json:"jobId" example:"123000"` // Cluster Job ID of job
Cluster *string `json:"cluster" example:"fritz"` // Cluster of job
StartTime *int64 `json:"startTime" example:"1649723812"` // Start Time of job as epoch
}
// DeleteJobApiRequest model
@@ -176,9 +175,9 @@ type GetCompleteJobApiResponse struct {
}
type JobMetricWithName struct {
Metric *schema.JobMetric `json:"metric"`
Name string `json:"name"`
Scope schema.MetricScope `json:"scope"`
Metric *schema.JobMetric `json:"metric"`
}
type ApiReturnedUser struct {
@@ -482,6 +481,18 @@ func (api *RestApi) getCompleteJobById(rw http.ResponseWriter, r *http.Request)
return
}
job.Tags, err = api.JobRepository.GetTags(&job.ID)
if err != nil {
handleError(err, http.StatusInternalServerError, rw)
return
}
if _, err = api.JobRepository.FetchMetadata(job); err != nil {
handleError(err, http.StatusInternalServerError, rw)
return
}
var scopes []schema.MetricScope
if job.NumNodes == 1 {
@@ -492,7 +503,7 @@ func (api *RestApi) getCompleteJobById(rw http.ResponseWriter, r *http.Request)
var data schema.JobData
if r.URL.Query().Has("all-metrics") {
if r.URL.Query().Get("all-metrics") == "true" {
data, err = metricdata.LoadData(job, nil, scopes, r.Context())
if err != nil {
log.Warn("Error while loading job data")
@@ -564,6 +575,18 @@ func (api *RestApi) getJobById(rw http.ResponseWriter, r *http.Request) {
return
}
job.Tags, err = api.JobRepository.GetTags(&job.ID)
if err != nil {
handleError(err, http.StatusInternalServerError, rw)
return
}
if _, err = api.JobRepository.FetchMetadata(job); err != nil {
handleError(err, http.StatusInternalServerError, rw)
return
}
var metrics GetJobApiRequest
if err = decode(r.Body, &metrics); err != nil {
http.Error(rw, err.Error(), http.StatusBadRequest)
@@ -1218,7 +1241,7 @@ func (api *RestApi) createUser(rw http.ResponseWriter, r *http.Request) {
return
}
rw.Write([]byte(fmt.Sprintf("User %v successfully created!\n", username)))
fmt.Fprintf(rw, "User %v successfully created!\n", username)
}
// deleteUser godoc

12
internal/auth/jwt.go
View File

@@ -17,7 +17,7 @@ import (
"github.com/ClusterCockpit/cc-backend/internal/repository"
"github.com/ClusterCockpit/cc-backend/pkg/log"
"github.com/ClusterCockpit/cc-backend/pkg/schema"
"github.com/golang-jwt/jwt/v4"
"github.com/golang-jwt/jwt/v5"
)
type JWTAuthenticator struct {
@@ -49,8 +49,8 @@ func (ja *JWTAuthenticator) Init() error {
func (ja *JWTAuthenticator) AuthViaJWT(
rw http.ResponseWriter,
r *http.Request) (*schema.User, error) {
r *http.Request,
) (*schema.User, error) {
rawtoken := r.Header.Get("X-Auth-Token")
if rawtoken == "" {
rawtoken = r.Header.Get("Authorization")
@@ -73,9 +73,9 @@ func (ja *JWTAuthenticator) AuthViaJWT(
log.Warn("Error while parsing JWT token")
return nil, err
}
if err := token.Claims.Valid(); err != nil {
if !token.Valid {
log.Warn("jwt token claims are not valid")
return nil, err
return nil, errors.New("jwt token claims are not valid")
}
// Token is valid, extract payload
@@ -88,7 +88,6 @@ func (ja *JWTAuthenticator) AuthViaJWT(
if config.Keys.JwtConfig.ValidateUser {
ur := repository.GetUserRepository()
user, err := ur.GetUser(sub)
// Deny any logins for unknown usernames
if err != nil {
log.Warn("Could not find user from JWT in internal database.")
@@ -117,7 +116,6 @@ func (ja *JWTAuthenticator) AuthViaJWT(
// Generate a new JWT that can be used for authentication
func (ja *JWTAuthenticator) ProvideJWT(user *schema.User) (string, error) {
if ja.privateKey == nil {
return "", errors.New("environment variable 'JWT_PRIVATE_KEY' not set")
}

15
internal/auth/jwtCookieSession.go
View File

@@ -17,7 +17,7 @@ import (
"github.com/ClusterCockpit/cc-backend/internal/repository"
"github.com/ClusterCockpit/cc-backend/pkg/log"
"github.com/ClusterCockpit/cc-backend/pkg/schema"
"github.com/golang-jwt/jwt/v4"
"github.com/golang-jwt/jwt/v5"
)
type JWTCookieSessionAuthenticator struct {
@@ -90,8 +90,8 @@ func (ja *JWTCookieSessionAuthenticator) CanLogin(
user *schema.User,
username string,
rw http.ResponseWriter,
r *http.Request) (*schema.User, bool) {
r *http.Request,
) (*schema.User, bool) {
jc := config.Keys.JwtConfig
cookieName := ""
if jc.CookieName != "" {
@@ -113,8 +113,8 @@ func (ja *JWTCookieSessionAuthenticator) CanLogin(
func (ja *JWTCookieSessionAuthenticator) Login(
user *schema.User,
rw http.ResponseWriter,
r *http.Request) (*schema.User, error) {
r *http.Request,
) (*schema.User, error) {
jc := config.Keys.JwtConfig
jwtCookie, err := r.Cookie(jc.CookieName)
var rawtoken string
@@ -144,10 +144,9 @@ func (ja *JWTCookieSessionAuthenticator) Login(
return nil, err
}
// Check token validity and extract paypload
if err := token.Claims.Valid(); err != nil {
if !token.Valid {
log.Warn("jwt token claims are not valid")
return nil, err
return nil, errors.New("jwt token claims are not valid")
}
claims := token.Claims.(jwt.MapClaims)

14
internal/auth/jwtSession.go
View File

@@ -17,7 +17,7 @@ import (
"github.com/ClusterCockpit/cc-backend/internal/repository"
"github.com/ClusterCockpit/cc-backend/pkg/log"
"github.com/ClusterCockpit/cc-backend/pkg/schema"
"github.com/golang-jwt/jwt/v4"
"github.com/golang-jwt/jwt/v5"
)
type JWTSessionAuthenticator struct {
@@ -44,8 +44,8 @@ func (ja *JWTSessionAuthenticator) CanLogin(
user *schema.User,
username string,
rw http.ResponseWriter,
r *http.Request) (*schema.User, bool) {
r *http.Request,
) (*schema.User, bool) {
return user, r.Header.Get("Authorization") != "" ||
r.URL.Query().Get("login-token") != ""
}
@@ -53,8 +53,8 @@ func (ja *JWTSessionAuthenticator) CanLogin(
func (ja *JWTSessionAuthenticator) Login(
user *schema.User,
rw http.ResponseWriter,
r *http.Request) (*schema.User, error) {
r *http.Request,
) (*schema.User, error) {
rawtoken := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
if rawtoken == "" {
rawtoken = r.URL.Query().Get("login-token")
@@ -71,9 +71,9 @@ func (ja *JWTSessionAuthenticator) Login(
return nil, err
}
if err = token.Claims.Valid(); err != nil {
if !token.Valid {
log.Warn("jwt token claims are not valid")
return nil, err
return nil, errors.New("jwt token claims are not valid")
}
claims := token.Claims.(jwt.MapClaims)

1
internal/config/config.go
View File

@@ -31,6 +31,7 @@ var Keys schema.ProgramConfig = schema.ProgramConfig{
"job_view_nodestats_selectedMetrics": []string{"flops_any", "mem_bw", "mem_used"},
"job_view_polarPlotMetrics": []string{"flops_any", "mem_bw", "mem_used"},
"job_view_selectedMetrics": []string{"flops_any", "mem_bw", "mem_used"},
"job_view_showFootprint": true,
"plot_general_colorBackground": true,
"plot_general_colorscheme": []string{"#00bfff", "#0000ff", "#ff00ff", "#ff0000", "#ff8000", "#ffff00", "#80ff00"},
"plot_general_lineWidth": 3,

51
internal/metricdata/cc-metric-store.go
View File

@@ -32,32 +32,32 @@ type CCMetricStoreConfig struct {
}
type CCMetricStore struct {
here2there map[string]string
there2here map[string]string
client http.Client
jwt string
url string
queryEndpoint string
client http.Client
here2there map[string]string
there2here map[string]string
}
type ApiQueryRequest struct {
Cluster string `json:"cluster"`
Queries []ApiQuery `json:"queries"`
ForAllNodes []string `json:"for-all-nodes"`
From int64 `json:"from"`
To int64 `json:"to"`
WithStats bool `json:"with-stats"`
WithData bool `json:"with-data"`
Queries []ApiQuery `json:"queries"`
ForAllNodes []string `json:"for-all-nodes"`
}
type ApiQuery struct {
Type *string `json:"type,omitempty"`
SubType *string `json:"subtype,omitempty"`
Metric string `json:"metric"`
Hostname string `json:"host"`
Aggregate bool `json:"aggreg"`
Type *string `json:"type,omitempty"`
TypeIds []string `json:"type-ids,omitempty"`
SubType *string `json:"subtype,omitempty"`
SubTypeIds []string `json:"subtype-ids,omitempty"`
Aggregate bool `json:"aggreg"`
}
type ApiQueryResponse struct {
@@ -67,16 +67,15 @@ type ApiQueryResponse struct {
type ApiMetricData struct {
Error *string `json:"error"`
Data []schema.Float `json:"data"`
From int64 `json:"from"`
To int64 `json:"to"`
Data []schema.Float `json:"data"`
Avg schema.Float `json:"avg"`
Min schema.Float `json:"min"`
Max schema.Float `json:"max"`
}
func (ccms *CCMetricStore) Init(rawConfig json.RawMessage) error {
var config CCMetricStoreConfig
if err := json.Unmarshal(rawConfig, &config); err != nil {
log.Warn("Error while unmarshaling raw json config")
@@ -122,8 +121,8 @@ func (ccms *CCMetricStore) toLocalName(metric string) string {
func (ccms *CCMetricStore) doRequest(
ctx context.Context,
body *ApiQueryRequest) (*ApiQueryResponse, error) {
body *ApiQueryRequest,
) (*ApiQueryResponse, error) {
buf := &bytes.Buffer{}
if err := json.NewEncoder(buf).Encode(body); err != nil {
log.Warn("Error while encoding request body")
@@ -162,8 +161,8 @@ func (ccms *CCMetricStore) LoadData(
job *schema.Job,
metrics []string,
scopes []schema.MetricScope,
ctx context.Context) (schema.JobData, error) {
ctx context.Context,
) (schema.JobData, error) {
queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes)
if err != nil {
log.Warn("Error while building queries")
@@ -186,7 +185,7 @@ func (ccms *CCMetricStore) LoadData(
}
var errors []string
var jobData schema.JobData = make(schema.JobData)
jobData := make(schema.JobData)
for i, row := range resBody.Results {
query := req.Queries[i]
metric := ccms.toLocalName(query.Metric)
@@ -206,7 +205,7 @@ func (ccms *CCMetricStore) LoadData(
jobData[metric][scope] = jobMetric
}
for _, res := range row {
for ndx, res := range row {
if res.Error != nil {
/* Build list for "partial errors", if any */
errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
@@ -216,7 +215,7 @@ func (ccms *CCMetricStore) LoadData(
id := (*string)(nil)
if query.Type != nil {
id = new(string)
*id = query.TypeIds[0]
*id = query.TypeIds[ndx]
}
if res.Avg.IsNaN() || res.Min.IsNaN() || res.Max.IsNaN() {
@@ -267,8 +266,8 @@ var (
func (ccms *CCMetricStore) buildQueries(
job *schema.Job,
metrics []string,
scopes []schema.MetricScope) ([]ApiQuery, []schema.MetricScope, error) {
scopes []schema.MetricScope,
) ([]ApiQuery, []schema.MetricScope, error) {
queries := make([]ApiQuery, 0, len(metrics)*len(scopes)*len(job.Resources))
assignedScope := []schema.MetricScope{}
@@ -313,6 +312,11 @@ func (ccms *CCMetricStore) buildQueries(
// Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node)
if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) {
if scope != schema.MetricScopeAccelerator {
// Skip all other catched cases
continue
}
queries = append(queries, ApiQuery{
Metric: remoteName,
Hostname: host.Hostname,
@@ -504,8 +508,8 @@ func (ccms *CCMetricStore) buildQueries(
func (ccms *CCMetricStore) LoadStats(
job *schema.Job,
metrics []string,
ctx context.Context) (map[string]map[string]schema.MetricStatistics, error) {
ctx context.Context,
) (map[string]map[string]schema.MetricStatistics, error) {
queries, _, err := ccms.buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}) // #166 Add scope shere for analysis view accelerator normalization?
if err != nil {
log.Warn("Error while building query")
@@ -566,8 +570,8 @@ func (ccms *CCMetricStore) LoadNodeData(
metrics, nodes []string,
scopes []schema.MetricScope,
from, to time.Time,
ctx context.Context) (map[string]map[string][]*schema.JobMetric, error) {
ctx context.Context,
) (map[string]map[string][]*schema.JobMetric, error) {
req := ApiQueryRequest{
Cluster: cluster,
From: from.Unix(),
@@ -652,7 +656,6 @@ func (ccms *CCMetricStore) LoadNodeData(
}
func intToStringSlice(is []int) []string {
ss := make([]string, len(is))
for i, x := range is {
ss[i] = strconv.Itoa(x)

10
internal/repository/job.go
View File

@@ -30,13 +30,11 @@ var (
)
type JobRepository struct {
DB *sqlx.DB
driver string
stmtCache *sq.StmtCache
cache *lrucache.Cache
DB *sqlx.DB
stmtCache *sq.StmtCache
cache *lrucache.Cache
archiveChannel chan *schema.Job
driver string
archivePending sync.WaitGroup
}

2
internal/repository/migrations/mysql/07_fix-tag-id.down.sql
View File

@@ -1 +1,3 @@
SET FOREIGN_KEY_CHECKS = 0;
ALTER TABLE tag MODIFY id INTEGER;
SET FOREIGN_KEY_CHECKS = 1;

2
internal/repository/migrations/mysql/07_fix-tag-id.up.sql
View File

@@ -1 +1,3 @@
SET FOREIGN_KEY_CHECKS = 0;
ALTER TABLE tag MODIFY id INTEGER AUTO_INCREMENT;
SET FOREIGN_KEY_CHECKS = 1;