ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2024-12-26 13:29:05 +01:00
Code Issues Packages Releases Activity

Intermediate Save commit

Browse Source
This commit is contained in:
Jan Eitzinger 2022-10-04 10:12:35 +02:00
parent 7251344d4a
commit 29d215fcea
10 changed files with 125 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
internal/repository/init.go
View File

@ -17,6 +17,7 @@ import (
"github.com/ClusterCockpit/cc-backend/pkg/archive" "github.com/ClusterCockpit/cc-backend/pkg/archive"
"github.com/ClusterCockpit/cc-backend/pkg/log" "github.com/ClusterCockpit/cc-backend/pkg/log"
"github.com/ClusterCockpit/cc-backend/pkg/schema" "github.com/ClusterCockpit/cc-backend/pkg/schema"
"github.com/ClusterCockpit/cc-backend/pkg/units"
) )
// `AUTO_INCREMENT` is in a comment because of this hack: // `AUTO_INCREMENT` is in a comment because of this hack:
@ -103,11 +104,11 @@ func HandleImportFlag(flag string) error {
return err return err
} }
if config.Keys.Validate { // if config.Keys.Validate {
if err := schema.Validate(schema.Meta, bytes.NewReader(raw)); err != nil { if err := schema.Validate(schema.Meta, bytes.NewReader(raw)); err != nil {
return fmt.Errorf("validate job meta: %v", err) return fmt.Errorf("validate job meta: %v", err)
}
} }
// }
dec := json.NewDecoder(bytes.NewReader(raw)) dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields() dec.DisallowUnknownFields()
jobMeta := schema.JobMeta{BaseJob: schema.JobDefaults} jobMeta := schema.JobMeta{BaseJob: schema.JobDefaults}
@ -132,6 +133,7 @@ func HandleImportFlag(flag string) error {
return err return err
} }
checkJobData(&jobData)
SanityChecks(&jobMeta.BaseJob) SanityChecks(&jobMeta.BaseJob)
jobMeta.MonitoringStatus = schema.MonitoringStatusArchivingSuccessful jobMeta.MonitoringStatus = schema.MonitoringStatusArchivingSuccessful
if job, err := GetJobRepository().Find(&jobMeta.JobID, &jobMeta.Cluster, &jobMeta.StartTime); err != sql.ErrNoRows { if job, err := GetJobRepository().Find(&jobMeta.JobID, &jobMeta.Cluster, &jobMeta.StartTime); err != sql.ErrNoRows {
@ -368,14 +370,31 @@ func loadJobStat(job *schema.JobMeta, metric string) float64 {
} }
func checkJobData(d *schema.JobData) error { func checkJobData(d *schema.JobData) error {
// for name, scopes := range *d { for _, scopes := range *d {
var newUnit string
// Add node scope if missing
for _, metric := range scopes {
if strings.Contains(metric.Unit.Base, "B/s") ||
strings.Contains(metric.Unit.Base, "F/s") ||
strings.Contains(metric.Unit.Base, "B") {
// for scope, metric := range scopes { // First get overall avg
// // 1. Unit normalisation sum := 0.0
for _, s := range metric.Series {
sum += s.Statistics.Avg
}
// } avg := sum / float64(len(metric.Series))
// // 2. Add node scope if missing
// } for _, s := range metric.Series {
fp := schema.ConvertFloatToFloat64(s.Data)
// Normalize values with new unit prefix
units.NormalizeSeries(fp, avg, metric.Unit, &newUnit)
s.Data = schema.GetFloat64ToFloat(fp)
}
metric.Unit = newUnit
}
}
}
return nil return nil
} }

22
internal/repository/job.go
View File

@ -211,7 +211,13 @@ func (r *JobRepository) Stop(
} }
// TODO: Use node hours instead: SELECT job.user, sum(job.num_nodes * (CASE WHEN job.job_state = "running" THEN CAST(strftime('%s', 'now') AS INTEGER) - job.start_time ELSE job.duration END)) as x FROM job GROUP BY user ORDER BY x DESC; // TODO: Use node hours instead: SELECT job.user, sum(job.num_nodes * (CASE WHEN job.job_state = "running" THEN CAST(strftime('%s', 'now') AS INTEGER) - job.start_time ELSE job.duration END)) as x FROM job GROUP BY user ORDER BY x DESC;
func (r *JobRepository) CountGroupedJobs(ctx context.Context, aggreg model.Aggregate, filters []*model.JobFilter, weight *model.Weights, limit *int) (map[string]int, error) { func (r *JobRepository) CountGroupedJobs(
ctx context.Context,
aggreg model.Aggregate,
filters []*model.JobFilter,
weight *model.Weights,
limit *int) (map[string]int, error) {
if !aggreg.IsValid() { if !aggreg.IsValid() {
return nil, errors.New("invalid aggregate") return nil, errors.New("invalid aggregate")
} }
@ -301,10 +307,14 @@ func (r *JobRepository) Archive(
var ErrNotFound = errors.New("no such job or user") var ErrNotFound = errors.New("no such job or user")
// FindJobOrUser returns a job database ID or a username if a job or user machtes the search term. // FindJobOrUser returns a job database ID or a username if a job or user
// As 0 is a valid job id, check if username is "" instead in order to check what machted. // machtes the search term. As 0 is a valid job id, check if username is ""
// If nothing matches the search, `ErrNotFound` is returned. // instead in order to check what matched. If nothing matches the search,
func (r *JobRepository) FindJobOrUser(ctx context.Context, searchterm string) (job int64, username string, err error) { // `ErrNotFound` is returned.
func (r *JobRepository) FindJobOrUser(
ctx context.Context,
searchterm string) (job int64, username string, err error) {
user := auth.GetUser(ctx) user := auth.GetUser(ctx)
if id, err := strconv.Atoi(searchterm); err == nil { if id, err := strconv.Atoi(searchterm); err == nil {
qb := sq.Select("job.id").From("job").Where("job.job_id = ?", id) qb := sq.Select("job.id").From("job").Where("job.job_id = ?", id)
@ -353,6 +363,7 @@ func (r *JobRepository) Partitions(cluster string) ([]string, error) {
// AllocatedNodes returns a map of all subclusters to a map of hostnames to the amount of jobs running on that host. // AllocatedNodes returns a map of all subclusters to a map of hostnames to the amount of jobs running on that host.
// Hosts with zero jobs running on them will not show up! // Hosts with zero jobs running on them will not show up!
func (r *JobRepository) AllocatedNodes(cluster string) (map[string]map[string]int, error) { func (r *JobRepository) AllocatedNodes(cluster string) (map[string]map[string]int, error) {
subclusters := make(map[string]map[string]int) subclusters := make(map[string]map[string]int)
rows, err := sq.Select("resources", "subcluster").From("job"). rows, err := sq.Select("resources", "subcluster").From("job").
Where("job.job_state = 'running'"). Where("job.job_state = 'running'").
@ -390,6 +401,7 @@ func (r *JobRepository) AllocatedNodes(cluster string) (map[string]map[string]in
} }
func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error { func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error {
res, err := sq.Update("job"). res, err := sq.Update("job").
Set("monitoring_status", schema.MonitoringStatusArchivingFailed). Set("monitoring_status", schema.MonitoringStatusArchivingFailed).
Set("duration", 0). Set("duration", 0).

2
pkg/schema/cluster.go
View File

@ -45,7 +45,7 @@ type SubClusterConfig struct {
type MetricConfig struct { type MetricConfig struct {
Name string `json:"name"` Name string `json:"name"`
Unit string `json:"unit"` Unit Unit `json:"unit"`
Scope MetricScope `json:"scope"` Scope MetricScope `json:"scope"`
Aggregation *string `json:"aggregation"` Aggregation *string `json:"aggregation"`
Timestep int `json:"timestep"` Timestep int `json:"timestep"`

20
pkg/schema/float.go
View File

@ -107,3 +107,23 @@ func (s *Series) MarshalJSON() ([]byte, error) {
buf = append(buf, ']', '}') buf = append(buf, ']', '}')
return buf, nil return buf, nil
} }
func ConvertFloatToFloat64(s []Float) []float64 {
fp := make([]float64, len(s))
for i, val := range s {
fp[i] = float64(val)
}
return fp
}
func GetFloat64ToFloat(s []float64) []Float {
fp := make([]Float, len(s))
for i, val := range s {
fp[i] = Float(val)
}
return fp
}

78
pkg/schema/job.go
View File

@ -17,26 +17,26 @@ import (
type BaseJob struct { type BaseJob struct {
// The unique identifier of a job // The unique identifier of a job
JobID int64 `json:"jobId" db:"job_id" example:"123000"` JobID int64 `json:"jobId" db:"job_id" example:"123000"`
User string `json:"user" db:"user" example:"abcd100h"` // The unique identifier of a user User string `json:"user" db:"user" example:"abcd100h"` // The unique identifier of a user
Project string `json:"project" db:"project" example:"abcd200"` // The unique identifier of a project Project string `json:"project" db:"project" example:"abcd200"` // The unique identifier of a project
Cluster string `json:"cluster" db:"cluster" example:"fritz"` // The unique identifier of a cluster Cluster string `json:"cluster" db:"cluster" example:"fritz"` // The unique identifier of a cluster
SubCluster string `json:"subCluster" db:"subcluster" example:"main"` // The unique identifier of a sub cluster SubCluster string `json:"subCluster" db:"subcluster" example:"main"` // The unique identifier of a sub cluster
Partition string `json:"partition" db:"partition" example:"main"` // The Slurm partition to which the job was submitted Partition string `json:"partition" db:"partition" example:"main"` // The Slurm partition to which the job was submitted
ArrayJobId int64 `json:"arrayJobId" db:"array_job_id" example:"123000"` // The unique identifier of an array job ArrayJobId int64 `json:"arrayJobId" db:"array_job_id" example:"123000"` // The unique identifier of an array job
NumNodes int32 `json:"numNodes" db:"num_nodes" example:"2" minimum:"1"` // Number of nodes used (Min > 0) NumNodes int32 `json:"numNodes" db:"num_nodes" example:"2" minimum:"1"` // Number of nodes used (Min > 0)
NumHWThreads int32 `json:"numHwthreads" db:"num_hwthreads" example:"20" minimum:"1"` // Number of HWThreads used (Min > 0) NumHWThreads int32 `json:"numHwthreads" db:"num_hwthreads" example:"20" minimum:"1"` // Number of HWThreads used (Min > 0)
NumAcc int32 `json:"numAcc" db:"num_acc" example:"2" minimum:"1"` // Number of accelerators used (Min > 0) NumAcc int32 `json:"numAcc" db:"num_acc" example:"2" minimum:"1"` // Number of accelerators used (Min > 0)
Exclusive int32 `json:"exclusive" db:"exclusive" example:"1" minimum:"0" maximum:"2"` // Specifies how nodes are shared: 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple jobs of same user Exclusive int32 `json:"exclusive" db:"exclusive" example:"1" minimum:"0" maximum:"2"` // Specifies how nodes are shared: 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple jobs of same user
MonitoringStatus int32 `json:"monitoringStatus" db:"monitoring_status" example:"1" minimum:"0" maximum:"3"` // State of monitoring system during job run: 0 - Disabled, 1 - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull MonitoringStatus int32 `json:"monitoringStatus" db:"monitoring_status" example:"1" minimum:"0" maximum:"3"` // State of monitoring system during job run: 0 - Disabled, 1 - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull
SMT int32 `json:"smt" db:"smt" example:"4"` // SMT threads used by job SMT int32 `json:"smt" db:"smt" example:"4"` // SMT threads used by job
State JobState `json:"jobState" db:"job_state" example:"completed" enums:"completed,failed,cancelled,stopped,timeout,out_of_memory"` // Final state of job State JobState `json:"jobState" db:"job_state" example:"completed" enums:"completed,failed,cancelled,stopped,timeout,out_of_memory"` // Final state of job
Duration int32 `json:"duration" db:"duration" example:"43200" minimum:"1"` // Duration of job in seconds (Min > 0) Duration int32 `json:"duration" db:"duration" example:"43200" minimum:"1"` // Duration of job in seconds (Min > 0)
Walltime int64 `json:"walltime" db:"walltime" example:"86400" minimum:"1"` // Requested walltime of job in seconds (Min > 0) Walltime int64 `json:"walltime" db:"walltime" example:"86400" minimum:"1"` // Requested walltime of job in seconds (Min > 0)
Tags []*Tag `json:"tags"` // List of tags Tags []*Tag `json:"tags"` // List of tags
RawResources []byte `json:"-" db:"resources"` // Resources used by job [As Bytes] RawResources []byte `json:"-" db:"resources"` // Resources used by job [As Bytes]
Resources []*Resource `json:"resources"` // Resources used by job Resources []*Resource `json:"resources"` // Resources used by job
RawMetaData []byte `json:"-" db:"meta_data"` // Additional information about the job [As Bytes] RawMetaData []byte `json:"-" db:"meta_data"` // Additional information about the job [As Bytes]
MetaData map[string]string `json:"metaData"` // Additional information about the job MetaData map[string]string `json:"metaData"` // Additional information about the job
} }
// Non-Swaggered Comment: Job // Non-Swaggered Comment: Job
@ -49,15 +49,15 @@ type Job struct {
ID int64 `json:"id" db:"id"` ID int64 `json:"id" db:"id"`
BaseJob BaseJob
StartTimeUnix int64 `json:"-" db:"start_time" example:"1649723812"` // Start epoch time stamp in seconds StartTimeUnix int64 `json:"-" db:"start_time" example:"1649723812"` // Start epoch time stamp in seconds
StartTime time.Time `json:"startTime"` // Start time as 'time.Time' data type StartTime time.Time `json:"startTime"` // Start time as 'time.Time' data type
MemUsedMax float64 `json:"-" db:"mem_used_max"` // MemUsedMax as Float64 MemUsedMax float64 `json:"-" db:"mem_used_max"` // MemUsedMax as Float64
FlopsAnyAvg float64 `json:"-" db:"flops_any_avg"` // FlopsAnyAvg as Float64 FlopsAnyAvg float64 `json:"-" db:"flops_any_avg"` // FlopsAnyAvg as Float64
MemBwAvg float64 `json:"-" db:"mem_bw_avg"` // MemBwAvg as Float64 MemBwAvg float64 `json:"-" db:"mem_bw_avg"` // MemBwAvg as Float64
LoadAvg float64 `json:"-" db:"load_avg"` // LoadAvg as Float64 LoadAvg float64 `json:"-" db:"load_avg"` // LoadAvg as Float64
NetBwAvg float64 `json:"-" db:"net_bw_avg"` // NetBwAvg as Float64 NetBwAvg float64 `json:"-" db:"net_bw_avg"` // NetBwAvg as Float64
NetDataVolTotal float64 `json:"-" db:"net_data_vol_total"` // NetDataVolTotal as Float64 NetDataVolTotal float64 `json:"-" db:"net_data_vol_total"` // NetDataVolTotal as Float64
FileBwAvg float64 `json:"-" db:"file_bw_avg"` // FileBwAvg as Float64 FileBwAvg float64 `json:"-" db:"file_bw_avg"` // FileBwAvg as Float64
FileDataVolTotal float64 `json:"-" db:"file_data_vol_total"` // FileDataVolTotal as Float64 FileDataVolTotal float64 `json:"-" db:"file_data_vol_total"` // FileDataVolTotal as Float64
} }
// Non-Swaggered Comment: JobMeta // Non-Swaggered Comment: JobMeta
@ -70,11 +70,11 @@ type Job struct {
// JobMeta model // JobMeta model
// @Description Meta data information of a HPC job. // @Description Meta data information of a HPC job.
type JobMeta struct { type JobMeta struct {
// The unique identifier of a job in the database // The unique identifier of a job in the database
ID *int64 `json:"id,omitempty"` ID *int64 `json:"id,omitempty"`
BaseJob BaseJob
StartTime int64 `json:"startTime" db:"start_time" example:"1649723812" minimum:"1"` // Start epoch time stamp in seconds (Min > 0) StartTime int64 `json:"startTime" db:"start_time" example:"1649723812" minimum:"1"` // Start epoch time stamp in seconds (Min > 0)
Statistics map[string]JobStatistics `json:"statistics,omitempty"` // Metric statistics of job Statistics map[string]JobStatistics `json:"statistics,omitempty"` // Metric statistics of job
} }
const ( const (
@ -89,11 +89,15 @@ var JobDefaults BaseJob = BaseJob{
MonitoringStatus: MonitoringStatusRunningOrArchiving, MonitoringStatus: MonitoringStatusRunningOrArchiving,
} }
type Unit struct {
Base string `json:"base"`
Prefix string `json:"prefix"`
}
// JobStatistics model // JobStatistics model
// @Description Specification for job metric statistics. // @Description Specification for job metric statistics.
type JobStatistics struct { type JobStatistics struct {
// Metric unit (see schema/unit.schema.json) Unit Unit `json:"unit" example:"GHz"`
Unit string `json:"unit" example:"GHz"`
Avg float64 `json:"avg" example:"2500" minimum:"0"` // Job metric average Avg float64 `json:"avg" example:"2500" minimum:"0"` // Job metric average
Min float64 `json:"min" example:"2000" minimum:"0"` // Job metric minimum Min float64 `json:"min" example:"2000" minimum:"0"` // Job metric minimum
Max float64 `json:"max" example:"3000" minimum:"0"` // Job metric maximum Max float64 `json:"max" example:"3000" minimum:"0"` // Job metric maximum
@ -102,18 +106,18 @@ type JobStatistics struct {
// Tag model // Tag model
// @Description Defines a tag using name and type. // @Description Defines a tag using name and type.
type Tag struct { type Tag struct {
// The unique DB identifier of a tag // The unique DB identifier of a tag
ID int64 `json:"id" db:"id"` ID int64 `json:"id" db:"id"`
Type string `json:"type" db:"tag_type" example:"Debug"` // Tag Type Type string `json:"type" db:"tag_type" example:"Debug"` // Tag Type
Name string `json:"name" db:"tag_name" example:"Testjob"` // Tag Name Name string `json:"name" db:"tag_name" example:"Testjob"` // Tag Name
} }
// Resource model // Resource model
// @Description A resource used by a job // @Description A resource used by a job
type Resource struct { type Resource struct {
Hostname string `json:"hostname"` // Name of the host (= node) Hostname string `json:"hostname"` // Name of the host (= node)
HWThreads []int `json:"hwthreads,omitempty"` // List of OS processor ids HWThreads []int `json:"hwthreads,omitempty"` // List of OS processor ids
Accelerators []string `json:"accelerators,omitempty"` // List of of accelerator device ids Accelerators []string `json:"accelerators,omitempty"` // List of of accelerator device ids
Configuration string `json:"configuration,omitempty"` // The configuration options of the node Configuration string `json:"configuration,omitempty"` // The configuration options of the node
} }

2
pkg/schema/metrics.go
View File

@ -15,7 +15,7 @@ import (
type JobData map[string]map[MetricScope]*JobMetric type JobData map[string]map[MetricScope]*JobMetric
type JobMetric struct { type JobMetric struct {
Unit string `json:"unit"` Unit Unit `json:"unit"`
Scope MetricScope `json:"scope"` Scope MetricScope `json:"scope"`
Timestep int `json:"timestep"` Timestep int `json:"timestep"`
Series []Series `json:"series"` Series []Series `json:"series"`

3
pkg/schema/schemas/cluster.schema.json
View File

@ -21,7 +21,7 @@
}, },
"unit": { "unit": {
"description": "Metric unit", "description": "Metric unit",
"type": "string" "$ref": "embedfs://unit.schema.json"
}, },
"scope": { "scope": {
"description": "Native measurement resolution", "description": "Native measurement resolution",
@ -38,7 +38,6 @@
"sum", "sum",
"avg" "avg"
] ]
}, },
"subClusters": { "subClusters": {
"description": "Array of cluster hardware partition metric thresholds", "description": "Array of cluster hardware partition metric thresholds",

1
pkg/schema/schemas/job-meta.schema.json
View File

@ -349,7 +349,6 @@
"jobState", "jobState",
"duration", "duration",
"resources", "resources",
"tags",
"statistics" "statistics"
] ]
} }

4
pkg/schema/schemas/unit.schema.json
View File

@ -5,7 +5,7 @@
"description": "Format specification for job metric units", "description": "Format specification for job metric units",
"type": "object", "type": "object",
"properties": { "properties": {
"base_unit": { "base": {
"description": "Metric base unit", "description": "Metric base unit",
"type": "string", "type": "string",
"enum": [ "enum": [
@ -36,6 +36,6 @@
} }
}, },
"required": [ "required": [
"base_unit" "base"
] ]
} }

13
test/integration_test.go
View File

@ -602,4 +602,17 @@ func testImportFlag(t *testing.T) {
if len(data) != 8 { if len(data) != 8 {
t.Errorf("Job data length: Got %d, want 8", len(data)) t.Errorf("Job data length: Got %d, want 8", len(data))
} }
r := map[string]string{"mem_used": "GB", "net_bw": "KB/s",
"cpu_power": "W", "cpu_used": "cpu_used",
"file_bw": "KB/s", "flops_any": "Flops/s",
"mem_bw": "GB/s", "ipc": "IPC"}
for name, scopes := range data {
for _, metric := range scopes {
if metric.Unit != r[name] {
t.Errorf("Metric %s unit: Got %s, want %s", name, metric.Unit, r[name])
}
}
}
} }