mirror of
https://github.com/ClusterCockpit/cc-backend
synced 2024-12-25 21:09:05 +01:00
Merge branch 'dev' into 275_add_tag_scope
This commit is contained in:
Christoph Kluge
commit
31765ce0ef
20
Makefile
20
Makefile
@ -22,11 +22,21 @@ SVELTE_COMPONENTS = status \
|
||||
header
|
||||
|
||||
SVELTE_TARGETS = $(addprefix $(FRONTEND)/public/build/,$(addsuffix .js, $(SVELTE_COMPONENTS)))
|
||||
SVELTE_SRC = $(wildcard $(FRONTEND)/src/*.svelte) \
|
||||
$(wildcard $(FRONTEND)/src/*.js) \
|
||||
$(wildcard $(FRONTEND)/src/filters/*.svelte) \
|
||||
$(wildcard $(FRONTEND)/src/plots/*.svelte) \
|
||||
$(wildcard $(FRONTEND)/src/joblist/*.svelte)
|
||||
SVELTE_SRC = $(wildcard $(FRONTEND)/src/*.svelte) \
|
||||
$(wildcard $(FRONTEND)/src/*.js) \
|
||||
$(wildcard $(FRONTEND)/src/analysis/*.svelte) \
|
||||
$(wildcard $(FRONTEND)/src/config/*.svelte) \
|
||||
$(wildcard $(FRONTEND)/src/config/admin/*.svelte) \
|
||||
$(wildcard $(FRONTEND)/src/config/user/*.svelte) \
|
||||
$(wildcard $(FRONTEND)/src/generic/*.js) \
|
||||
$(wildcard $(FRONTEND)/src/generic/*.svelte) \
|
||||
$(wildcard $(FRONTEND)/src/generic/filters/*.svelte) \
|
||||
$(wildcard $(FRONTEND)/src/generic/plots/*.svelte) \
|
||||
$(wildcard $(FRONTEND)/src/generic/joblist/*.svelte) \
|
||||
$(wildcard $(FRONTEND)/src/generic/helper/*.svelte) \
|
||||
$(wildcard $(FRONTEND)/src/generic/select/*.svelte) \
|
||||
$(wildcard $(FRONTEND)/src/header/*.svelte) \
|
||||
$(wildcard $(FRONTEND)/src/job/*.svelte)
|
||||
|
||||
.PHONY: clean distclean test tags frontend swagger graphql $(TARGET)
|
||||
|
||||
|
||||
@ -38,6 +38,15 @@ var (
|
||||
apiHandle *api.RestApi
|
||||
)
|
||||
|
||||
func onFailureResponse(rw http.ResponseWriter, r *http.Request, err error) {
|
||||
rw.Header().Add("Content-Type", "application/json")
|
||||
rw.WriteHeader(http.StatusUnauthorized)
|
||||
json.NewEncoder(rw).Encode(map[string]string{
|
||||
"status": http.StatusText(http.StatusUnauthorized),
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
func serverInit() {
|
||||
// Setup the http.Handler/Router used by the server
|
||||
graph.Init()
|
||||
@ -166,64 +175,32 @@ func serverInit() {
|
||||
return authHandle.AuthApi(
|
||||
// On success;
|
||||
next,
|
||||
|
||||
// On failure: JSON Response
|
||||
func(rw http.ResponseWriter, r *http.Request, err error) {
|
||||
rw.Header().Add("Content-Type", "application/json")
|
||||
rw.WriteHeader(http.StatusUnauthorized)
|
||||
json.NewEncoder(rw).Encode(map[string]string{
|
||||
"status": http.StatusText(http.StatusUnauthorized),
|
||||
"error": err.Error(),
|
||||
})
|
||||
})
|
||||
onFailureResponse)
|
||||
})
|
||||
|
||||
userapi.Use(func(next http.Handler) http.Handler {
|
||||
return authHandle.AuthUserApi(
|
||||
// On success;
|
||||
next,
|
||||
|
||||
// On failure: JSON Response
|
||||
func(rw http.ResponseWriter, r *http.Request, err error) {
|
||||
rw.Header().Add("Content-Type", "application/json")
|
||||
rw.WriteHeader(http.StatusUnauthorized)
|
||||
json.NewEncoder(rw).Encode(map[string]string{
|
||||
"status": http.StatusText(http.StatusUnauthorized),
|
||||
"error": err.Error(),
|
||||
})
|
||||
})
|
||||
onFailureResponse)
|
||||
})
|
||||
|
||||
configapi.Use(func(next http.Handler) http.Handler {
|
||||
return authHandle.AuthConfigApi(
|
||||
// On success;
|
||||
next,
|
||||
|
||||
// On failure: JSON Response
|
||||
func(rw http.ResponseWriter, r *http.Request, err error) {
|
||||
rw.Header().Add("Content-Type", "application/json")
|
||||
rw.WriteHeader(http.StatusUnauthorized)
|
||||
json.NewEncoder(rw).Encode(map[string]string{
|
||||
"status": http.StatusText(http.StatusUnauthorized),
|
||||
"error": err.Error(),
|
||||
})
|
||||
})
|
||||
onFailureResponse)
|
||||
})
|
||||
|
||||
frontendapi.Use(func(next http.Handler) http.Handler {
|
||||
return authHandle.AuthFrontendApi(
|
||||
// On success;
|
||||
next,
|
||||
|
||||
// On failure: JSON Response
|
||||
func(rw http.ResponseWriter, r *http.Request, err error) {
|
||||
rw.Header().Add("Content-Type", "application/json")
|
||||
rw.WriteHeader(http.StatusUnauthorized)
|
||||
json.NewEncoder(rw).Encode(map[string]string{
|
||||
"status": http.StatusText(http.StatusUnauthorized),
|
||||
"error": err.Error(),
|
||||
})
|
||||
})
|
||||
onFailureResponse)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@ -307,6 +307,10 @@ func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.JobMeta, error) {
|
||||
scopes = append(scopes, schema.MetricScopeCore)
|
||||
}
|
||||
|
||||
if job.NumAcc > 0 {
|
||||
scopes = append(scopes, schema.MetricScopeAccelerator)
|
||||
}
|
||||
|
||||
jobData, err := LoadData(job, allMetrics, scopes, ctx)
|
||||
if err != nil {
|
||||
log.Error("Error wile loading job data for archiving")
|
||||
|
||||
@ -0,0 +1,21 @@
|
||||
ALTER TABLE job DROP energy;
|
||||
ALTER TABLE job DROP energy_footprint;
|
||||
ALTER TABLE job ADD COLUMN flops_any_avg;
|
||||
ALTER TABLE job ADD COLUMN mem_bw_avg;
|
||||
ALTER TABLE job ADD COLUMN mem_used_max;
|
||||
ALTER TABLE job ADD COLUMN load_avg;
|
||||
ALTER TABLE job ADD COLUMN net_bw_avg;
|
||||
ALTER TABLE job ADD COLUMN net_data_vol_total;
|
||||
ALTER TABLE job ADD COLUMN file_bw_avg;
|
||||
ALTER TABLE job ADD COLUMN file_data_vol_total;
|
||||
|
||||
UPDATE job SET flops_any_avg = json_extract(footprint, '$.flops_any_avg');
|
||||
UPDATE job SET mem_bw_avg = json_extract(footprint, '$.mem_bw_avg');
|
||||
UPDATE job SET mem_used_max = json_extract(footprint, '$.mem_used_max');
|
||||
UPDATE job SET load_avg = json_extract(footprint, '$.cpu_load_avg');
|
||||
UPDATE job SET net_bw_avg = json_extract(footprint, '$.net_bw_avg');
|
||||
UPDATE job SET net_data_vol_total = json_extract(footprint, '$.net_data_vol_total');
|
||||
UPDATE job SET file_bw_avg = json_extract(footprint, '$.file_bw_avg');
|
||||
UPDATE job SET file_data_vol_total = json_extract(footprint, '$.file_data_vol_total');
|
||||
|
||||
ALTER TABLE job DROP footprint;
|
||||
@ -1,12 +1,27 @@
|
||||
CREATE INDEX IF NOT EXISTS job_by_project ON job (project);
|
||||
CREATE INDEX IF NOT EXISTS job_list_projects ON job (project, job_state);
|
||||
|
||||
ALTER TABLE job ADD COLUMN energy REAL NOT NULL DEFAULT 0.0;
|
||||
ALTER TABLE job ADD COLUMN energy_footprint TEXT DEFAULT NULL;
|
||||
|
||||
ALTER TABLE job ADD COLUMN footprint TEXT DEFAULT NULL;
|
||||
ALTER TABLE tag ADD COLUMN tag_scope TEXT NOT NULL DEFAULT 'global';
|
||||
|
||||
UPDATE job SET footprint = '{"flops_any_avg": 0.0}';
|
||||
UPDATE job SET footprint = json_replace(footprint, '$.flops_any_avg', job.flops_any_avg);
|
||||
UPDATE job SET footprint = json_insert(footprint, '$.mem_bw_avg', job.mem_bw_avg);
|
||||
UPDATE job SET footprint = json_insert(footprint, '$.mem_used_max', job.mem_used_max);
|
||||
UPDATE job SET footprint = json_insert(footprint, '$.cpu_load_avg', job.load_avg);
|
||||
UPDATE job SET footprint = json_insert(footprint, '$.net_bw_avg', job.net_bw_avg) WHERE job.net_bw_avg != 0;
|
||||
UPDATE job SET footprint = json_insert(footprint, '$.net_data_vol_total', job.net_data_vol_total) WHERE job.net_data_vol_total != 0;
|
||||
UPDATE job SET footprint = json_insert(footprint, '$.file_bw_avg', job.file_bw_avg) WHERE job.file_bw_avg != 0;
|
||||
UPDATE job SET footprint = json_insert(footprint, '$.file_data_vol_total', job.file_data_vol_total) WHERE job.file_data_vol_total != 0;
|
||||
|
||||
ALTER TABLE job DROP flops_any_avg;
|
||||
ALTER TABLE job DROP mem_bw_avg;
|
||||
ALTER TABLE job DROP mem_used_max;
|
||||
ALTER TABLE job DROP load_avg;
|
||||
ALTER TABLE job DROP net_bw_avg;
|
||||
ALTER TABLE job DROP net_data_vol_total;
|
||||
ALTER TABLE job DROP file_bw_avg;
|
||||
ALTER TABLE job DROP file_data_vol_total;
|
||||
|
||||
@ -47,11 +47,11 @@ type SubCluster struct {
|
||||
|
||||
type SubClusterConfig struct {
|
||||
Name string `json:"name"`
|
||||
Footprint string `json:"footprint,omitempty"`
|
||||
Peak float64 `json:"peak"`
|
||||
Normal float64 `json:"normal"`
|
||||
Caution float64 `json:"caution"`
|
||||
Alert float64 `json:"alert"`
|
||||
Footprint string `json:"footprint,omitempty"`
|
||||
Remove bool `json:"remove"`
|
||||
LowerIsBetter bool `json:"lowerIsBetter"`
|
||||
Energy bool `json:"energy"`
|
||||
@ -62,14 +62,14 @@ type MetricConfig struct {
|
||||
Name string `json:"name"`
|
||||
Scope MetricScope `json:"scope"`
|
||||
Aggregation string `json:"aggregation"`
|
||||
Footprint string `json:"footprint,omitempty"`
|
||||
SubClusters []*SubClusterConfig `json:"subClusters,omitempty"`
|
||||
Timestep int `json:"timestep"`
|
||||
Peak float64 `json:"peak"`
|
||||
Normal float64 `json:"normal"`
|
||||
Caution float64 `json:"caution"`
|
||||
Alert float64 `json:"alert"`
|
||||
Timestep int `json:"timestep"`
|
||||
LowerIsBetter bool `json:"lowerIsBetter"`
|
||||
Footprint string `json:"footprint,omitempty"`
|
||||
Energy bool `json:"energy"`
|
||||
}
|
||||
|
||||
|
||||
@ -32,7 +32,7 @@ type BaseJob struct {
|
||||
Footprint map[string]float64 `json:"footprint"`
|
||||
MetaData map[string]string `json:"metaData"`
|
||||
ConcurrentJobs JobLinkResultList `json:"concurrentJobs"`
|
||||
Energy float64 `json:"energy"`
|
||||
Energy float64 `json:"energy" db:"energy"`
|
||||
ArrayJobId int64 `json:"arrayJobId,omitempty" db:"array_job_id" example:"123000"`
|
||||
Walltime int64 `json:"walltime,omitempty" db:"walltime" example:"86400" minimum:"1"`
|
||||
JobID int64 `json:"jobId" db:"job_id" example:"123000"`
|
||||
|
||||
@ -1,284 +1,319 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "embedfs://cluster.schema.json",
|
||||
"title": "HPC cluster description",
|
||||
"description": "Meta data information of a HPC cluster",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"description": "The unique identifier of a cluster",
|
||||
"type": "string"
|
||||
},
|
||||
"metricConfig": {
|
||||
"description": "Metric specifications",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"description": "Metric name",
|
||||
"type": "string"
|
||||
},
|
||||
"unit": {
|
||||
"description": "Metric unit",
|
||||
"$ref": "embedfs://unit.schema.json"
|
||||
},
|
||||
"scope": {
|
||||
"description": "Native measurement resolution",
|
||||
"type": "string"
|
||||
},
|
||||
"timestep": {
|
||||
"description": "Frequency of timeseries points",
|
||||
"type": "integer"
|
||||
},
|
||||
"aggregation": {
|
||||
"description": "How the metric is aggregated",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"sum",
|
||||
"avg"
|
||||
]
|
||||
},
|
||||
"peak": {
|
||||
"description": "Metric peak threshold (Upper metric limit)",
|
||||
"type": "number"
|
||||
},
|
||||
"normal": {
|
||||
"description": "Metric normal threshold",
|
||||
"type": "number"
|
||||
},
|
||||
"caution": {
|
||||
"description": "Metric caution threshold (Suspicious but does not require immediate action)",
|
||||
"type": "number"
|
||||
},
|
||||
"alert": {
|
||||
"description": "Metric alert threshold (Requires immediate action)",
|
||||
"type": "number"
|
||||
},
|
||||
"subClusters": {
|
||||
"description": "Array of cluster hardware partition metric thresholds",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"description": "Hardware partition name",
|
||||
"type": "string"
|
||||
},
|
||||
"peak": {
|
||||
"type": "number"
|
||||
},
|
||||
"normal": {
|
||||
"type": "number"
|
||||
},
|
||||
"caution": {
|
||||
"type": "number"
|
||||
},
|
||||
"alert": {
|
||||
"type": "number"
|
||||
},
|
||||
"remove": {
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name",
|
||||
"unit",
|
||||
"scope",
|
||||
"timestep",
|
||||
"aggregation",
|
||||
"peak",
|
||||
"normal",
|
||||
"caution",
|
||||
"alert"
|
||||
]
|
||||
},
|
||||
"minItems": 1
|
||||
},
|
||||
"subClusters": {
|
||||
"description": "Array of cluster hardware partitions",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"description": "Hardware partition name",
|
||||
"type": "string"
|
||||
},
|
||||
"processorType": {
|
||||
"description": "Processor type",
|
||||
"type": "string"
|
||||
},
|
||||
"socketsPerNode": {
|
||||
"description": "Number of sockets per node",
|
||||
"type": "integer"
|
||||
},
|
||||
"coresPerSocket": {
|
||||
"description": "Number of cores per socket",
|
||||
"type": "integer"
|
||||
},
|
||||
"threadsPerCore": {
|
||||
"description": "Number of SMT threads per core",
|
||||
"type": "integer"
|
||||
},
|
||||
"flopRateScalar": {
|
||||
"description": "Theoretical node peak flop rate for scalar code in GFlops/s",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"unit": {
|
||||
"description": "Metric unit",
|
||||
"$ref": "embedfs://unit.schema.json"
|
||||
},
|
||||
"value": {
|
||||
"type": "number"
|
||||
}
|
||||
}
|
||||
},
|
||||
"flopRateSimd": {
|
||||
"description": "Theoretical node peak flop rate for SIMD code in GFlops/s",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"unit": {
|
||||
"description": "Metric unit",
|
||||
"$ref": "embedfs://unit.schema.json"
|
||||
},
|
||||
"value": {
|
||||
"type": "number"
|
||||
}
|
||||
}
|
||||
},
|
||||
"memoryBandwidth": {
|
||||
"description": "Theoretical node peak memory bandwidth in GB/s",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"unit": {
|
||||
"description": "Metric unit",
|
||||
"$ref": "embedfs://unit.schema.json"
|
||||
},
|
||||
"value": {
|
||||
"type": "number"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nodes": {
|
||||
"description": "Node list expression",
|
||||
"type": "string"
|
||||
},
|
||||
"topology": {
|
||||
"description": "Node topology",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"node": {
|
||||
"description": "HwTread lists of node",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"socket": {
|
||||
"description": "HwTread lists of sockets",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"memoryDomain": {
|
||||
"description": "HwTread lists of memory domains",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"die": {
|
||||
"description": "HwTread lists of dies",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"core": {
|
||||
"description": "HwTread lists of cores",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"accelerators": {
|
||||
"type": "array",
|
||||
"description": "List of of accelerator devices",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string",
|
||||
"description": "The unique device id"
|
||||
},
|
||||
"type": {
|
||||
"type": "string",
|
||||
"description": "The accelerator type",
|
||||
"enum": [
|
||||
"Nvidia GPU",
|
||||
"AMD GPU",
|
||||
"Intel GPU"
|
||||
]
|
||||
},
|
||||
"model": {
|
||||
"type": "string",
|
||||
"description": "The accelerator model"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"type",
|
||||
"model"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"node",
|
||||
"socket",
|
||||
"memoryDomain"
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name",
|
||||
"nodes",
|
||||
"topology",
|
||||
"processorType",
|
||||
"socketsPerNode",
|
||||
"coresPerSocket",
|
||||
"threadsPerCore",
|
||||
"flopRateScalar",
|
||||
"flopRateSimd",
|
||||
"memoryBandwidth"
|
||||
]
|
||||
},
|
||||
"minItems": 1
|
||||
}
|
||||
"$schema": "http://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "embedfs://cluster.schema.json",
|
||||
"title": "HPC cluster description",
|
||||
"description": "Meta data information of a HPC cluster",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"description": "The unique identifier of a cluster",
|
||||
"type": "string"
|
||||
},
|
||||
"required": [
|
||||
"name",
|
||||
"metricConfig",
|
||||
"subClusters"
|
||||
]
|
||||
"metricConfig": {
|
||||
"description": "Metric specifications",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"description": "Metric name",
|
||||
"type": "string"
|
||||
},
|
||||
"unit": {
|
||||
"description": "Metric unit",
|
||||
"$ref": "embedfs://unit.schema.json"
|
||||
},
|
||||
"scope": {
|
||||
"description": "Native measurement resolution",
|
||||
"type": "string"
|
||||
},
|
||||
"timestep": {
|
||||
"description": "Frequency of timeseries points",
|
||||
"type": "integer"
|
||||
},
|
||||
"aggregation": {
|
||||
"description": "How the metric is aggregated",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"sum",
|
||||
"avg"
|
||||
]
|
||||
},
|
||||
"footprint": {
|
||||
"description": "Is it a footprint metric and what type",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"avg",
|
||||
"max",
|
||||
"min"
|
||||
]
|
||||
},
|
||||
"energy": {
|
||||
"description": "Is it used to calculate job energy",
|
||||
"type": "boolean"
|
||||
},
|
||||
"lowerIsBetter": {
|
||||
"description": "Is lower better.",
|
||||
"type": "boolean"
|
||||
},
|
||||
"peak": {
|
||||
"description": "Metric peak threshold (Upper metric limit)",
|
||||
"type": "number"
|
||||
},
|
||||
"normal": {
|
||||
"description": "Metric normal threshold",
|
||||
"type": "number"
|
||||
},
|
||||
"caution": {
|
||||
"description": "Metric caution threshold (Suspicious but does not require immediate action)",
|
||||
"type": "number"
|
||||
},
|
||||
"alert": {
|
||||
"description": "Metric alert threshold (Requires immediate action)",
|
||||
"type": "number"
|
||||
},
|
||||
"subClusters": {
|
||||
"description": "Array of cluster hardware partition metric thresholds",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"description": "Hardware partition name",
|
||||
"type": "string"
|
||||
},
|
||||
"footprint": {
|
||||
"description": "Is it a footprint metric and what type. Overwrite global setting",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"avg",
|
||||
"max",
|
||||
"min"
|
||||
]
|
||||
},
|
||||
"energy": {
|
||||
"description": "Is it used to calculate job energy. Overwrite global",
|
||||
"type": "boolean"
|
||||
},
|
||||
"lowerIsBetter": {
|
||||
"description": "Is lower better. Overwrite global",
|
||||
"type": "boolean"
|
||||
},
|
||||
"peak": {
|
||||
"type": "number"
|
||||
},
|
||||
"normal": {
|
||||
"type": "number"
|
||||
},
|
||||
"caution": {
|
||||
"type": "number"
|
||||
},
|
||||
"alert": {
|
||||
"type": "number"
|
||||
},
|
||||
"remove": {
|
||||
"description": "Remove this metric for this subcluster",
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name",
|
||||
"unit",
|
||||
"scope",
|
||||
"timestep",
|
||||
"aggregation",
|
||||
"peak",
|
||||
"normal",
|
||||
"caution",
|
||||
"alert"
|
||||
]
|
||||
},
|
||||
"minItems": 1
|
||||
},
|
||||
"subClusters": {
|
||||
"description": "Array of cluster hardware partitions",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"description": "Hardware partition name",
|
||||
"type": "string"
|
||||
},
|
||||
"processorType": {
|
||||
"description": "Processor type",
|
||||
"type": "string"
|
||||
},
|
||||
"socketsPerNode": {
|
||||
"description": "Number of sockets per node",
|
||||
"type": "integer"
|
||||
},
|
||||
"coresPerSocket": {
|
||||
"description": "Number of cores per socket",
|
||||
"type": "integer"
|
||||
},
|
||||
"threadsPerCore": {
|
||||
"description": "Number of SMT threads per core",
|
||||
"type": "integer"
|
||||
},
|
||||
"flopRateScalar": {
|
||||
"description": "Theoretical node peak flop rate for scalar code in GFlops/s",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"unit": {
|
||||
"description": "Metric unit",
|
||||
"$ref": "embedfs://unit.schema.json"
|
||||
},
|
||||
"value": {
|
||||
"type": "number"
|
||||
}
|
||||
}
|
||||
},
|
||||
"flopRateSimd": {
|
||||
"description": "Theoretical node peak flop rate for SIMD code in GFlops/s",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"unit": {
|
||||
"description": "Metric unit",
|
||||
"$ref": "embedfs://unit.schema.json"
|
||||
},
|
||||
"value": {
|
||||
"type": "number"
|
||||
}
|
||||
}
|
||||
},
|
||||
"memoryBandwidth": {
|
||||
"description": "Theoretical node peak memory bandwidth in GB/s",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"unit": {
|
||||
"description": "Metric unit",
|
||||
"$ref": "embedfs://unit.schema.json"
|
||||
},
|
||||
"value": {
|
||||
"type": "number"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nodes": {
|
||||
"description": "Node list expression",
|
||||
"type": "string"
|
||||
},
|
||||
"topology": {
|
||||
"description": "Node topology",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"node": {
|
||||
"description": "HwTread lists of node",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"socket": {
|
||||
"description": "HwTread lists of sockets",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"memoryDomain": {
|
||||
"description": "HwTread lists of memory domains",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"die": {
|
||||
"description": "HwTread lists of dies",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"core": {
|
||||
"description": "HwTread lists of cores",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"accelerators": {
|
||||
"type": "array",
|
||||
"description": "List of of accelerator devices",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string",
|
||||
"description": "The unique device id"
|
||||
},
|
||||
"type": {
|
||||
"type": "string",
|
||||
"description": "The accelerator type",
|
||||
"enum": [
|
||||
"Nvidia GPU",
|
||||
"AMD GPU",
|
||||
"Intel GPU"
|
||||
]
|
||||
},
|
||||
"model": {
|
||||
"type": "string",
|
||||
"description": "The accelerator model"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"type",
|
||||
"model"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"node",
|
||||
"socket",
|
||||
"memoryDomain"
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name",
|
||||
"nodes",
|
||||
"topology",
|
||||
"processorType",
|
||||
"socketsPerNode",
|
||||
"coresPerSocket",
|
||||
"threadsPerCore",
|
||||
"flopRateScalar",
|
||||
"flopRateSimd",
|
||||
"memoryBandwidth"
|
||||
]
|
||||
},
|
||||
"minItems": 1
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name",
|
||||
"metricConfig",
|
||||
"subClusters"
|
||||
]
|
||||
}
|
||||
|
||||
@ -75,7 +75,7 @@
|
||||
duration, numNodes, numHWThreads, numAcc,
|
||||
SMT, exclusive, partition, subCluster, arrayJobId,
|
||||
monitoringStatus, state, walltime,
|
||||
tags { id, type, name, scope },
|
||||
tags { id, type, name },
|
||||
resources { hostname, hwthreads, accelerators },
|
||||
metaData,
|
||||
userData { name, email },
|
||||
|
||||
@ -67,62 +67,74 @@
|
||||
export let height = "310px";
|
||||
|
||||
const footprintData = job?.footprint?.map((jf) => {
|
||||
// Unit
|
||||
const fmc = getContext("getMetricConfig")(job.cluster, job.subCluster, jf.name);
|
||||
const unit = (fmc?.unit?.prefix ? fmc.unit.prefix : "") + (fmc?.unit?.base ? fmc.unit.base : "")
|
||||
if (fmc) {
|
||||
// Unit
|
||||
const unit = (fmc?.unit?.prefix ? fmc.unit.prefix : "") + (fmc?.unit?.base ? fmc.unit.base : "")
|
||||
|
||||
// Threshold / -Differences
|
||||
const fmt = findJobThresholds(job, fmc);
|
||||
if (jf.name === "flops_any") fmt.peak = round(fmt.peak * 0.85, 0);
|
||||
// Threshold / -Differences
|
||||
const fmt = findJobThresholds(job, fmc);
|
||||
if (jf.name === "flops_any") fmt.peak = round(fmt.peak * 0.85, 0);
|
||||
|
||||
// Define basic data -> Value: Use as Provided
|
||||
const fmBase = {
|
||||
name: jf.name + ' (' + jf.stat + ')',
|
||||
avg: jf.value,
|
||||
unit: unit,
|
||||
max: fmt.peak,
|
||||
dir: fmc.lowerIsBetter
|
||||
};
|
||||
// Define basic data -> Value: Use as Provided
|
||||
const fmBase = {
|
||||
name: jf.name + ' (' + jf.stat + ')',
|
||||
avg: jf.value,
|
||||
unit: unit,
|
||||
max: fmt.peak,
|
||||
dir: fmc.lowerIsBetter
|
||||
};
|
||||
|
||||
if (evalFootprint(jf.value, fmt, fmc.lowerIsBetter, "alert")) {
|
||||
if (evalFootprint(jf.value, fmt, fmc.lowerIsBetter, "alert")) {
|
||||
return {
|
||||
...fmBase,
|
||||
color: "danger",
|
||||
message: `Metric average way ${fmc.lowerIsBetter ? "above" : "below"} expected normal thresholds.`,
|
||||
impact: 3
|
||||
};
|
||||
} else if (evalFootprint(jf.value, fmt, fmc.lowerIsBetter, "caution")) {
|
||||
return {
|
||||
...fmBase,
|
||||
color: "warning",
|
||||
message: `Metric average ${fmc.lowerIsBetter ? "above" : "below"} expected normal thresholds.`,
|
||||
impact: 2,
|
||||
};
|
||||
} else if (evalFootprint(jf.value, fmt, fmc.lowerIsBetter, "normal")) {
|
||||
return {
|
||||
...fmBase,
|
||||
color: "success",
|
||||
message: "Metric average within expected thresholds.",
|
||||
impact: 1,
|
||||
};
|
||||
} else if (evalFootprint(jf.value, fmt, fmc.lowerIsBetter, "peak")) {
|
||||
return {
|
||||
...fmBase,
|
||||
color: "info",
|
||||
message:
|
||||
"Metric average above expected normal thresholds: Check for artifacts recommended.",
|
||||
impact: 0,
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
...fmBase,
|
||||
color: "secondary",
|
||||
message:
|
||||
"Metric average above expected peak threshold: Check for artifacts!",
|
||||
impact: -1,
|
||||
};
|
||||
}
|
||||
} else { // No matching metric config: display as single value
|
||||
return {
|
||||
...fmBase,
|
||||
color: "danger",
|
||||
message: `Metric average way ${fmc.lowerIsBetter ? "above" : "below"} expected normal thresholds.`,
|
||||
impact: 3
|
||||
};
|
||||
} else if (evalFootprint(jf.value, fmt, fmc.lowerIsBetter, "caution")) {
|
||||
return {
|
||||
...fmBase,
|
||||
color: "warning",
|
||||
message: `Metric average ${fmc.lowerIsBetter ? "above" : "below"} expected normal thresholds.`,
|
||||
impact: 2,
|
||||
};
|
||||
} else if (evalFootprint(jf.value, fmt, fmc.lowerIsBetter, "normal")) {
|
||||
return {
|
||||
...fmBase,
|
||||
color: "success",
|
||||
message: "Metric average within expected thresholds.",
|
||||
impact: 1,
|
||||
};
|
||||
} else if (evalFootprint(jf.value, fmt, fmc.lowerIsBetter, "peak")) {
|
||||
return {
|
||||
...fmBase,
|
||||
color: "info",
|
||||
name: jf.name + ' (' + jf.stat + ')',
|
||||
avg: jf.value,
|
||||
message:
|
||||
"Metric average above expected normal thresholds: Check for artifacts recommended.",
|
||||
impact: 0,
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
...fmBase,
|
||||
color: "secondary",
|
||||
message:
|
||||
"Metric average above expected peak threshold: Check for artifacts!",
|
||||
impact: -1,
|
||||
`No config for metric ${jf.name} found.`,
|
||||
impact: 4,
|
||||
};
|
||||
}
|
||||
});
|
||||
}).sort(function (a, b) { // Sort by impact value primarily, within impact sort name alphabetically
|
||||
return a.impact - b.impact || ((a.name > b.name) ? 1 : ((b.name > a.name) ? -1 : 0));
|
||||
});;
|
||||
|
||||
function evalFootprint(mean, thresholds, lowerIsBetter, level) {
|
||||
// Handle Metrics in which less value is better
|
||||
@ -159,37 +171,76 @@
|
||||
{/if}
|
||||
<CardBody>
|
||||
{#each footprintData as fpd, index}
|
||||
<div class="mb-1 d-flex justify-content-between">
|
||||
<div> <b>{fpd.name}</b></div>
|
||||
<!-- For symmetry, see below ...-->
|
||||
<div
|
||||
class="cursor-help d-inline-flex"
|
||||
id={`footprint-${job.jobId}-${index}`}
|
||||
>
|
||||
<div class="mx-1">
|
||||
<!-- Alerts Only -->
|
||||
{#if fpd.impact === 3 || fpd.impact === -1}
|
||||
<Icon name="exclamation-triangle-fill" class="text-danger" />
|
||||
{:else if fpd.impact === 2}
|
||||
<Icon name="exclamation-triangle" class="text-warning" />
|
||||
{/if}
|
||||
<!-- Emoji for all states-->
|
||||
{#if fpd.impact === 3}
|
||||
<Icon name="emoji-frown" class="text-danger" />
|
||||
{:else if fpd.impact === 2}
|
||||
<Icon name="emoji-neutral" class="text-warning" />
|
||||
{:else if fpd.impact === 1}
|
||||
<Icon name="emoji-smile" class="text-success" />
|
||||
{:else if fpd.impact === 0}
|
||||
<Icon name="emoji-laughing" class="text-info" />
|
||||
{:else if fpd.impact === -1}
|
||||
<Icon name="emoji-dizzy" class="text-danger" />
|
||||
{/if}
|
||||
{#if fpd.impact !== 4}
|
||||
<div class="mb-1 d-flex justify-content-between">
|
||||
<div> <b>{fpd.name}</b></div>
|
||||
<!-- For symmetry, see below ...-->
|
||||
<div
|
||||
class="cursor-help d-inline-flex"
|
||||
id={`footprint-${job.jobId}-${index}`}
|
||||
>
|
||||
<div class="mx-1">
|
||||
<!-- Alerts Only -->
|
||||
{#if fpd.impact === 3 || fpd.impact === -1}
|
||||
<Icon name="exclamation-triangle-fill" class="text-danger" />
|
||||
{:else if fpd.impact === 2}
|
||||
<Icon name="exclamation-triangle" class="text-warning" />
|
||||
{/if}
|
||||
<!-- Emoji for all states-->
|
||||
{#if fpd.impact === 3}
|
||||
<Icon name="emoji-frown" class="text-danger" />
|
||||
{:else if fpd.impact === 2}
|
||||
<Icon name="emoji-neutral" class="text-warning" />
|
||||
{:else if fpd.impact === 1}
|
||||
<Icon name="emoji-smile" class="text-success" />
|
||||
{:else if fpd.impact === 0}
|
||||
<Icon name="emoji-laughing" class="text-info" />
|
||||
{:else if fpd.impact === -1}
|
||||
<Icon name="emoji-dizzy" class="text-danger" />
|
||||
{/if}
|
||||
</div>
|
||||
<div>
|
||||
<!-- Print Values -->
|
||||
{fpd.avg} / {fpd.max}
|
||||
{fpd.unit} <!-- To increase margin to tooltip: No other way manageable ... -->
|
||||
</div>
|
||||
</div>
|
||||
<Tooltip
|
||||
target={`footprint-${job.jobId}-${index}`}
|
||||
placement="right"
|
||||
offset={[0, 20]}>{fpd.message}</Tooltip
|
||||
>
|
||||
</div>
|
||||
<Row cols={12} class="{(footprintData.length == (index + 1)) ? 'mb-0' : 'mb-2'}">
|
||||
{#if fpd.dir}
|
||||
<Col xs="1">
|
||||
<Icon name="caret-left-fill" />
|
||||
</Col>
|
||||
{/if}
|
||||
<Col xs="11" class="align-content-center">
|
||||
<Progress value={fpd.avg} max={fpd.max} color={fpd.color} />
|
||||
</Col>
|
||||
{#if !fpd.dir}
|
||||
<Col xs="1">
|
||||
<Icon name="caret-right-fill" />
|
||||
</Col>
|
||||
{/if}
|
||||
</Row>
|
||||
{:else}
|
||||
<div class="mb-1 d-flex justify-content-between">
|
||||
<div>
|
||||
<!-- Print Values -->
|
||||
{fpd.avg} / {fpd.max}
|
||||
{fpd.unit} <!-- To increase margin to tooltip: No other way manageable ... -->
|
||||
<b>{fpd.name}</b>
|
||||
</div>
|
||||
<div
|
||||
class="cursor-help d-inline-flex"
|
||||
id={`footprint-${job.jobId}-${index}`}
|
||||
>
|
||||
<div class="mx-1">
|
||||
<Icon name="info-circle"/>
|
||||
</div>
|
||||
<div>
|
||||
{fpd.avg}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<Tooltip
|
||||
@ -197,22 +248,7 @@
|
||||
placement="right"
|
||||
offset={[0, 20]}>{fpd.message}</Tooltip
|
||||
>
|
||||
</div>
|
||||
<Row cols={12} class="{(footprintData.length == (index + 1)) ? 'mb-0' : 'mb-2'}">
|
||||
{#if fpd.dir}
|
||||
<Col xs="1">
|
||||
<Icon name="caret-left-fill" />
|
||||
</Col>
|
||||
{/if}
|
||||
<Col xs="11" class="align-content-center">
|
||||
<Progress value={fpd.avg} max={fpd.max} color={fpd.color} />
|
||||
</Col>
|
||||
{#if !fpd.dir}
|
||||
<Col xs="1">
|
||||
<Icon name="caret-right-fill" />
|
||||
</Col>
|
||||
{/if}
|
||||
</Row>
|
||||
{/if}
|
||||
{/each}
|
||||
{#if job?.metaData?.message}
|
||||
<hr class="mt-1 mb-2" />
|
||||
|
||||
@ -301,7 +301,7 @@ export function stickyHeader(datatableHeaderSelector, updatePading) {
|
||||
onDestroy(() => document.removeEventListener("scroll", onscroll));
|
||||
}
|
||||
|
||||
export function checkMetricDisabled(m, c, s) { //[m]etric, [c]luster, [s]ubcluster
|
||||
export function checkMetricDisabled(m, c, s) { // [m]etric, [c]luster, [s]ubcluster
|
||||
const metrics = getContext("globalMetrics");
|
||||
const result = metrics?.find((gm) => gm.name === m)?.availability?.find((av) => av.cluster === c)?.subClusters?.includes(s)
|
||||
return !result
|
||||
@ -309,23 +309,22 @@ export function checkMetricDisabled(m, c, s) { //[m]etric, [c]luster, [s]ubclust
|
||||
|
||||
export function getStatsItems() {
|
||||
// console.time('stats')
|
||||
// console.log('getStatsItems ...')
|
||||
const globalMetrics = getContext("globalMetrics")
|
||||
const result = globalMetrics.map((gm) => {
|
||||
if (gm?.footprint) {
|
||||
// Footprint contains suffix naming the used stat-type
|
||||
// console.time('deep')
|
||||
// console.log('Deep Config for', gm.name)
|
||||
const mc = getMetricConfigDeep(gm.name, null, null)
|
||||
// console.timeEnd('deep')
|
||||
return {
|
||||
field: gm.name + '_' + gm.footprint,
|
||||
text: gm.name + ' (' + gm.footprint + ')',
|
||||
metric: gm.name,
|
||||
from: 0,
|
||||
to: mc.peak,
|
||||
peak: mc.peak,
|
||||
enabled: false
|
||||
if (mc) {
|
||||
return {
|
||||
field: gm.name + '_' + gm.footprint,
|
||||
text: gm.name + ' (' + gm.footprint + ')',
|
||||
metric: gm.name,
|
||||
from: 0,
|
||||
to: mc.peak,
|
||||
peak: mc.peak,
|
||||
enabled: false
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
@ -336,11 +335,9 @@ export function getStatsItems() {
|
||||
|
||||
export function getSortItems() {
|
||||
//console.time('sort')
|
||||
//console.log('getSortItems ...')
|
||||
const globalMetrics = getContext("globalMetrics")
|
||||
const result = globalMetrics.map((gm) => {
|
||||
if (gm?.footprint) {
|
||||
// Footprint contains suffix naming the used stat-type
|
||||
return {
|
||||
field: gm.name + '_' + gm.footprint,
|
||||
type: 'foot',
|
||||
@ -357,21 +354,22 @@ export function getSortItems() {
|
||||
function getMetricConfigDeep(metric, cluster, subCluster) {
|
||||
const clusters = getContext("clusters");
|
||||
if (cluster != null) {
|
||||
let c = clusters.find((c) => c.name == cluster);
|
||||
const c = clusters.find((c) => c.name == cluster);
|
||||
if (subCluster != null) {
|
||||
let sc = c.subClusters.find((sc) => sc.name == subCluster);
|
||||
const sc = c.subClusters.find((sc) => sc.name == subCluster);
|
||||
return sc.metricConfig.find((mc) => mc.name == metric)
|
||||
} else {
|
||||
let result;
|
||||
for (let sc of c.subClusters) {
|
||||
const mc = sc.metricConfig.find((mc) => mc.name == metric)
|
||||
if (result) { // If lowerIsBetter: Peak is still maximum value, no special case required
|
||||
if (result && mc) { // update result; If lowerIsBetter: Peak is still maximum value, no special case required
|
||||
result.alert = (mc.alert > result.alert) ? mc.alert : result.alert
|
||||
result.caution = (mc.caution > result.caution) ? mc.caution : result.caution
|
||||
result.normal = (mc.normal > result.normal) ? mc.normal : result.normal
|
||||
result.peak = (mc.peak > result.peak) ? mc.peak : result.peak
|
||||
} else {
|
||||
if (mc) result = {...mc};
|
||||
} else if (mc) {
|
||||
// start new result
|
||||
result = {...mc};
|
||||
}
|
||||
}
|
||||
return result
|
||||
@ -381,13 +379,14 @@ function getMetricConfigDeep(metric, cluster, subCluster) {
|
||||
for (let c of clusters) {
|
||||
for (let sc of c.subClusters) {
|
||||
const mc = sc.metricConfig.find((mc) => mc.name == metric)
|
||||
if (result) { // If lowerIsBetter: Peak is still maximum value, no special case required
|
||||
if (result && mc) { // update result; If lowerIsBetter: Peak is still maximum value, no special case required
|
||||
result.alert = (mc.alert > result.alert) ? mc.alert : result.alert
|
||||
result.caution = (mc.caution > result.caution) ? mc.caution : result.caution
|
||||
result.normal = (mc.normal > result.normal) ? mc.normal : result.normal
|
||||
result.peak = (mc.peak > result.peak) ? mc.peak : result.peak
|
||||
} else {
|
||||
if (mc) result = {...mc};
|
||||
} else if (mc) {
|
||||
// Start new result
|
||||
result = {...mc};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user