mirror of
https://github.com/ClusterCockpit/cc-backend
synced 2025-07-23 12:51:40 +02:00
Cleanup
Use external cc-units Lint code Fix import tests
This commit is contained in:
@@ -68,7 +68,7 @@ func HandleImportFlag(flag string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
//checkJobData(&jobData)
|
||||
// checkJobData(&jobData)
|
||||
// SanityChecks(&jobMeta.BaseJob)
|
||||
jobMeta.MonitoringStatus = schema.MonitoringStatusArchivingSuccessful
|
||||
if _, err = r.Find(&jobMeta.JobID, &jobMeta.Cluster, &jobMeta.StartTime); err != sql.ErrNoRows {
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
"github.com/ClusterCockpit/cc-backend/pkg/archive"
|
||||
"github.com/ClusterCockpit/cc-backend/pkg/log"
|
||||
"github.com/ClusterCockpit/cc-backend/pkg/schema"
|
||||
"github.com/ClusterCockpit/cc-backend/pkg/units"
|
||||
ccunits "github.com/ClusterCockpit/cc-units"
|
||||
)
|
||||
|
||||
// Delete the tables "job", "tag" and "jobtag" from the database and
|
||||
@@ -173,11 +173,28 @@ func getNormalizationFactor(v float64) (float64, int) {
|
||||
return math.Pow10(count * scale), count * scale
|
||||
}
|
||||
|
||||
func getExponent(p float64) int {
|
||||
count := 0
|
||||
|
||||
for p > 1.0 {
|
||||
p = p / 1000.0
|
||||
count++
|
||||
}
|
||||
|
||||
return count * 3
|
||||
}
|
||||
|
||||
func NewPrefixFromFactor(op ccunits.Prefix, e int) ccunits.Prefix {
|
||||
f := float64(op)
|
||||
exp := math.Pow10(getExponent(f) - e)
|
||||
return ccunits.Prefix(exp)
|
||||
}
|
||||
|
||||
func normalize(avg float64, p string) (float64, string) {
|
||||
f, e := getNormalizationFactor(avg)
|
||||
|
||||
if e != 0 {
|
||||
np := units.NewPrefixFromFactor(units.NewPrefix(p), e)
|
||||
np := NewPrefixFromFactor(ccunits.NewPrefix(p), e)
|
||||
return f, np.Prefix()
|
||||
}
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/ClusterCockpit/cc-backend/pkg/units"
|
||||
ccunits "github.com/ClusterCockpit/cc-units"
|
||||
)
|
||||
|
||||
func TestNormalizeFactor(t *testing.T) {
|
||||
@@ -27,11 +27,11 @@ func TestNormalizeFactor(t *testing.T) {
|
||||
|
||||
fmt.Printf("Factor %e Count %d\n", f, e)
|
||||
|
||||
np := units.NewPrefix("")
|
||||
np := ccunits.NewPrefix("")
|
||||
|
||||
fmt.Printf("Prefix %e Short %s\n", float64(np), np.Prefix())
|
||||
|
||||
p := units.NewPrefixFromFactor(np, e)
|
||||
p := NewPrefixFromFactor(np, e)
|
||||
|
||||
if p.Prefix() != "G" {
|
||||
t.Errorf("Failed Prefix or unit: Want G, Got %s", p.Prefix())
|
||||
@@ -52,11 +52,11 @@ func TestNormalizeKeep(t *testing.T) {
|
||||
|
||||
fmt.Printf("Factor %e Count %d\n", f, e)
|
||||
|
||||
np := units.NewPrefix("G")
|
||||
np := ccunits.NewPrefix("G")
|
||||
|
||||
fmt.Printf("Prefix %e Short %s\n", float64(np), np.Prefix())
|
||||
|
||||
p := units.NewPrefixFromFactor(np, e)
|
||||
p := NewPrefixFromFactor(np, e)
|
||||
|
||||
if p.Prefix() != "G" {
|
||||
t.Errorf("Failed Prefix or unit: Want G, Got %s", p.Prefix())
|
||||
|
||||
496
internal/importer/testdata/data-taurus.json
vendored
496
internal/importer/testdata/data-taurus.json
vendored
@@ -1,496 +0,0 @@
|
||||
{
|
||||
"cpu_used": {
|
||||
"core": {
|
||||
"unit": {
|
||||
"base": ""
|
||||
},
|
||||
"timestep": 30,
|
||||
"series": [
|
||||
{
|
||||
"hostname": "taurusi6489",
|
||||
"id": "0",
|
||||
"statistics": {
|
||||
"min": 0.09090909090909093,
|
||||
"avg": 0.9173553719008265,
|
||||
"max": 1.0000000000000002
|
||||
},
|
||||
"data": [
|
||||
0.09090909090909093,
|
||||
0.9999999999999999,
|
||||
1.0,
|
||||
1.0000000000000002,
|
||||
1.0,
|
||||
1.0000000000000002,
|
||||
0.9999999999999999,
|
||||
1.0,
|
||||
1.0,
|
||||
1.0,
|
||||
1.0
|
||||
]
|
||||
},
|
||||
{
|
||||
"hostname": "taurusi6489",
|
||||
"id": "1",
|
||||
"statistics": {
|
||||
"min": 0.03694102397926118,
|
||||
"avg": 0.045968409230268584,
|
||||
"max": 0.08809840425531917
|
||||
},
|
||||
"data": [
|
||||
0.08809840425531917,
|
||||
0.05710659898477157,
|
||||
0.04034861200774694,
|
||||
0.037962362102530824,
|
||||
0.03976721629485936,
|
||||
0.04163976759199483,
|
||||
0.03694102397926118,
|
||||
0.03821243523316062,
|
||||
0.03851132686084142,
|
||||
0.044752092723760455,
|
||||
0.04231266149870802
|
||||
]
|
||||
},
|
||||
{
|
||||
"hostname": "taurusi6490",
|
||||
"id": "10",
|
||||
"statistics": {
|
||||
"min": 0.10505319148936171,
|
||||
"avg": 0.9186411992263056,
|
||||
"max": 1.0000000000000002
|
||||
},
|
||||
"data": [
|
||||
0.10505319148936171,
|
||||
1.0000000000000002,
|
||||
1.0,
|
||||
1.0,
|
||||
1.0,
|
||||
0.9999999999999999,
|
||||
1.0,
|
||||
0.9999999999999999,
|
||||
1.0,
|
||||
1.0,
|
||||
1.0
|
||||
]
|
||||
},
|
||||
{
|
||||
"hostname": "taurusi6490",
|
||||
"id": "11",
|
||||
"statistics": {
|
||||
"min": 0.05286048845767815,
|
||||
"avg": 0.07053823838706144,
|
||||
"max": 0.075148113501715
|
||||
},
|
||||
"data": [
|
||||
0.05286048845767815,
|
||||
0.06936597614563718,
|
||||
0.07254534083802376,
|
||||
0.075148113501715,
|
||||
0.06909547738693468,
|
||||
0.07372696032489846,
|
||||
0.07077983088005012,
|
||||
0.07082419304293325,
|
||||
0.07424812030075188,
|
||||
0.07285803627267043,
|
||||
0.07446808510638298
|
||||
]
|
||||
}
|
||||
],
|
||||
"statisticsSeries": null
|
||||
}
|
||||
},
|
||||
"ipc": {
|
||||
"core": {
|
||||
"unit": {
|
||||
"base": "IPC"
|
||||
},
|
||||
"timestep": 60,
|
||||
"series": [
|
||||
{
|
||||
"hostname": "taurusi6489",
|
||||
"id": "0",
|
||||
"statistics": {
|
||||
"min": 1.3808406263195592,
|
||||
"avg": 1.3960848578375105,
|
||||
"max": 1.4485575599350569
|
||||
},
|
||||
"data": [
|
||||
1.4485575599350569,
|
||||
1.3808406263195592,
|
||||
1.3830284413690626,
|
||||
1.3836692663348698,
|
||||
1.3843283952290035
|
||||
]
|
||||
},
|
||||
{
|
||||
"hostname": "taurusi6489",
|
||||
"id": "1",
|
||||
"statistics": {
|
||||
"min": 0.30469640475234366,
|
||||
"avg": 0.8816944294664065,
|
||||
"max": 1.797623522191001
|
||||
},
|
||||
"data": [
|
||||
1.797623522191001,
|
||||
0.954395633726228,
|
||||
1.0019972349956185,
|
||||
0.30469640475234366,
|
||||
0.3497593516668412
|
||||
]
|
||||
},
|
||||
{
|
||||
"hostname": "taurusi6490",
|
||||
"id": "10",
|
||||
"statistics": {
|
||||
"min": 1.3791232173760588,
|
||||
"avg": 1.3850247295506815,
|
||||
"max": 1.386710405495511
|
||||
},
|
||||
"data": [
|
||||
1.3791232173760588,
|
||||
1.38619977419787,
|
||||
1.386397917938246,
|
||||
1.3866923327457215,
|
||||
1.386710405495511
|
||||
]
|
||||
},
|
||||
{
|
||||
"hostname": "taurusi6490",
|
||||
"id": "11",
|
||||
"statistics": {
|
||||
"min": 0.6424094604392216,
|
||||
"avg": 0.9544442638400293,
|
||||
"max": 1.2706704244636826
|
||||
},
|
||||
"data": [
|
||||
1.2706704244636826,
|
||||
0.6424094604392216,
|
||||
0.9249973908234796,
|
||||
0.6940110823242276,
|
||||
1.2401329611495353
|
||||
]
|
||||
}
|
||||
],
|
||||
"statisticsSeries": null
|
||||
}
|
||||
},
|
||||
"flops_any": {
|
||||
"core": {
|
||||
"unit": {
|
||||
"base": "F/s"
|
||||
},
|
||||
"timestep": 60,
|
||||
"series": [
|
||||
{
|
||||
"hostname": "taurusi6489",
|
||||
"id": "0",
|
||||
"statistics": {
|
||||
"min": 0.0,
|
||||
"avg": 184.2699002412084,
|
||||
"max": 921.3495012060421
|
||||
},
|
||||
"data": [
|
||||
921.3495012060421,
|
||||
0.0,
|
||||
0.0,
|
||||
0.0,
|
||||
0.0
|
||||
]
|
||||
},
|
||||
{
|
||||
"hostname": "taurusi6489",
|
||||
"id": "1",
|
||||
"statistics": {
|
||||
"min": 0.13559227208748068,
|
||||
"avg": 273.2997868356056,
|
||||
"max": 1355.9227390817396
|
||||
},
|
||||
"data": [
|
||||
1355.9227390817396,
|
||||
8.94908797747172,
|
||||
0.6779613312519499,
|
||||
0.13559227208748068,
|
||||
0.8135535154771758
|
||||
]
|
||||
},
|
||||
{
|
||||
"hostname": "taurusi6490",
|
||||
"id": "10",
|
||||
"statistics": {
|
||||
"min": 0.0,
|
||||
"avg": 1678.8419461262179,
|
||||
"max": 4346.591400350933
|
||||
},
|
||||
"data": [
|
||||
4346.591400350933,
|
||||
0.0,
|
||||
578.4248288199713,
|
||||
0.0,
|
||||
3469.193501460185
|
||||
]
|
||||
},
|
||||
{
|
||||
"hostname": "taurusi6490",
|
||||
"id": "11",
|
||||
"statistics": {
|
||||
"min": 45.28689133054866,
|
||||
"avg": 609.6644949204072,
|
||||
"max": 2582.7080822873186
|
||||
},
|
||||
"data": [
|
||||
2582.7080822873186,
|
||||
45.28689133054866,
|
||||
48.67663233623293,
|
||||
47.591911855555026,
|
||||
324.0589567923803
|
||||
]
|
||||
}
|
||||
],
|
||||
"statisticsSeries": null
|
||||
}
|
||||
},
|
||||
"mem_bw": {
|
||||
"socket": {
|
||||
"unit": {
|
||||
"base": "B/s"
|
||||
},
|
||||
"timestep": 60,
|
||||
"series": [
|
||||
{
|
||||
"hostname": "taurusi6489",
|
||||
"id": "0",
|
||||
"statistics": {
|
||||
"min": 653671812.1661415,
|
||||
"avg": 1637585527.5854635,
|
||||
"max": 2614718291.9554267
|
||||
},
|
||||
"data": [
|
||||
653671812.1661415,
|
||||
2614718291.9554267,
|
||||
1732453371.7073724,
|
||||
1612865229.8704093,
|
||||
1574218932.2279677
|
||||
]
|
||||
},
|
||||
{
|
||||
"hostname": "taurusi6490",
|
||||
"id": "0",
|
||||
"statistics": {
|
||||
"min": 1520190251.61048,
|
||||
"avg": 1572477682.3850098,
|
||||
"max": 1688960732.2760606
|
||||
},
|
||||
"data": [
|
||||
1688960732.2760606,
|
||||
1580140679.8216474,
|
||||
1520190251.61048,
|
||||
1541841829.6250021,
|
||||
1531254918.591859
|
||||
]
|
||||
}
|
||||
],
|
||||
"statisticsSeries": null
|
||||
}
|
||||
},
|
||||
"file_bw": {
|
||||
"node": {
|
||||
"unit": {
|
||||
"base": "B/s"
|
||||
},
|
||||
"timestep": 30,
|
||||
"series": [
|
||||
{
|
||||
"hostname": "taurusi6489",
|
||||
"statistics": {
|
||||
"min": 0.0,
|
||||
"avg": 190352.6328851857,
|
||||
"max": 2093878.361723524
|
||||
},
|
||||
"data": [
|
||||
0.0,
|
||||
0.0,
|
||||
0.0,
|
||||
0.6000135186380174,
|
||||
0.0,
|
||||
0.0,
|
||||
2093878.361723524,
|
||||
0.0,
|
||||
0.0,
|
||||
0.0,
|
||||
0.0
|
||||
]
|
||||
},
|
||||
{
|
||||
"hostname": "taurusi6490",
|
||||
"statistics": {
|
||||
"min": 0.0,
|
||||
"avg": 1050832.4509396513,
|
||||
"max": 11559156.360352296
|
||||
},
|
||||
"data": [
|
||||
0.0,
|
||||
0.0,
|
||||
0.0,
|
||||
0.0,
|
||||
0.0,
|
||||
0.0,
|
||||
11559156.360352296,
|
||||
0.0,
|
||||
0.5999838690326298,
|
||||
0.0,
|
||||
0.0
|
||||
]
|
||||
}
|
||||
],
|
||||
"statisticsSeries": null
|
||||
}
|
||||
},
|
||||
"net_bw": {
|
||||
"node": {
|
||||
"unit": {
|
||||
"base": "B/s"
|
||||
},
|
||||
"timestep": 30,
|
||||
"series": [
|
||||
{
|
||||
"hostname": "taurusi6489",
|
||||
"statistics": {
|
||||
"min": 126779.89655880642,
|
||||
"avg": 653834.5091507058,
|
||||
"max": 1285639.5107541133
|
||||
},
|
||||
"data": [
|
||||
1158202.7403032137,
|
||||
126779.89655880642,
|
||||
419017.91939583793,
|
||||
345766.3974972795,
|
||||
645419.3296982117,
|
||||
644667.7333333333,
|
||||
1285639.5107541133,
|
||||
643481.2108874657,
|
||||
640025.3562553325,
|
||||
643241.4875354709,
|
||||
639938.0184386979
|
||||
]
|
||||
},
|
||||
{
|
||||
"hostname": "taurusi6490",
|
||||
"statistics": {
|
||||
"min": 640156.9862985397,
|
||||
"avg": 872367.6551257868,
|
||||
"max": 1916309.7075416835
|
||||
},
|
||||
"data": [
|
||||
1774843.146788355,
|
||||
643218.3646426039,
|
||||
641681.1031071587,
|
||||
644690.1512268113,
|
||||
647183.5650609672,
|
||||
644439.3303402043,
|
||||
1916309.7075416835,
|
||||
643748.3241006166,
|
||||
757189.8273227927,
|
||||
642583.6999539217,
|
||||
640156.9862985397
|
||||
]
|
||||
}
|
||||
],
|
||||
"statisticsSeries": null
|
||||
}
|
||||
},
|
||||
"mem_used": {
|
||||
"node": {
|
||||
"unit": {
|
||||
"base": "B"
|
||||
},
|
||||
"timestep": 30,
|
||||
"series": [
|
||||
{
|
||||
"hostname": "taurusi6489",
|
||||
"statistics": {
|
||||
"min": 2779066368.0,
|
||||
"avg": 9282117259.636364,
|
||||
"max": 10202595328.0
|
||||
},
|
||||
"data": [
|
||||
2779066368.0,
|
||||
8518217728.0,
|
||||
9852760064.0,
|
||||
9979805696.0,
|
||||
10039619584.0,
|
||||
10087104512.0,
|
||||
10136084480.0,
|
||||
10202595328.0,
|
||||
10154196992.0,
|
||||
10177409024.0,
|
||||
10176430080.0
|
||||
]
|
||||
},
|
||||
{
|
||||
"hostname": "taurusi6490",
|
||||
"statistics": {
|
||||
"min": 9993277440.0,
|
||||
"avg": 10013080110.545454,
|
||||
"max": 10039676928.0
|
||||
},
|
||||
"data": [
|
||||
10001317888.0,
|
||||
10013028352.0,
|
||||
10006728704.0,
|
||||
10039676928.0,
|
||||
10035838976.0,
|
||||
10033356800.0,
|
||||
10006577152.0,
|
||||
10005659648.0,
|
||||
9993277440.0,
|
||||
9993564160.0,
|
||||
10014855168.0
|
||||
]
|
||||
}
|
||||
],
|
||||
"statisticsSeries": null
|
||||
}
|
||||
},
|
||||
"cpu_power": {
|
||||
"socket": {
|
||||
"unit": {
|
||||
"base": "W"
|
||||
},
|
||||
"timestep": 60,
|
||||
"series": [
|
||||
{
|
||||
"hostname": "taurusi6489",
|
||||
"id": "0",
|
||||
"statistics": {
|
||||
"min": 35.50647456742635,
|
||||
"avg": 72.08313211552377,
|
||||
"max": 83.33799371150049
|
||||
},
|
||||
"data": [
|
||||
35.50647456742635,
|
||||
75.65022009482759,
|
||||
83.33799371150049,
|
||||
83.00405043233219,
|
||||
82.9169217715322
|
||||
]
|
||||
},
|
||||
{
|
||||
"hostname": "taurusi6490",
|
||||
"id": "0",
|
||||
"statistics": {
|
||||
"min": 83.8466923147859,
|
||||
"avg": 85.18572681122097,
|
||||
"max": 85.83909286117324
|
||||
},
|
||||
"data": [
|
||||
83.8466923147859,
|
||||
85.58816979864088,
|
||||
85.31266819129794,
|
||||
85.83909286117324,
|
||||
85.34201089020692
|
||||
]
|
||||
}
|
||||
],
|
||||
"statisticsSeries": null
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"jobId": 398764,
|
||||
"jobId": 398955,
|
||||
"cluster": "fritz",
|
||||
"startTime": 1675954353,
|
||||
"duration": 3400
|
||||
"startTime": 1675956725,
|
||||
"duration": 260
|
||||
}
|
||||
|
||||
@@ -2,5 +2,5 @@
|
||||
"jobId": 398764,
|
||||
"cluster": "fritz",
|
||||
"startTime": 1675954353,
|
||||
"duration": 3400
|
||||
"duration": 177
|
||||
}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"jobId":398764,"user":"k106eb10","project":"k106eb","cluster":"fritz","subCluster":"main","partition":"singlenode","arrayJobId":0,"numNodes":1,"numHwthreads":72,"numAcc":0,"exclusive":1,"monitoringStatus":1,"smt":0,"jobState":"completed","duration":177,"walltime":86340,"resources":[{"hostname":"f0649"}],"metaData":{"jobName":"ams_pipeline","jobScript":"#!/bin/bash -l\n#SBATCH --job-name=ams_pipeline\n#SBATCH --time=23:59:00\n#SBATCH --partition=singlenode\n#SBATCH --ntasks=72\n#SBATCH --hint=multithread\n#SBATCH --chdir=/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/moreprototypesYury/gen_131_CuTe/cfg/Ni2Al2\n#SBATCH --export=NONE\nunset SLURM_EXPORT_ENV\nuss=$(whoami)\nfind /dev/shm/ -user $uss -type f -mmin +30 -delete\ncd \"/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/moreprototypesYury/gen_131_CuTe/cfg/Ni2Al2\"\nams_pipeline pipeline.json \u003e \"/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/moreprototypesYury/gen_131_CuTe/cfg/Ni2Al2/ams_pipeline_job.sh.out\" 2\u003e \"/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/moreprototypesYury/gen_131_CuTe/cfg/Ni2Al2/ams_pipeline_job.sh.err\"\n","slurmInfo":"\nJobId=398764 JobName=ams_pipeline\n UserId=k106eb10(210387) GroupId=80111\n Account=k106eb QOS=normal \n Requeue=False Restarts=0 BatchFlag=True \n TimeLimit=1439\n SubmitTime=2023-02-09T14:10:20\n Partition=singlenode \n NodeList=f0649\n NumNodes=1 NumCPUs=72 NumTasks=72 CPUs/Task=1\n NTasksPerNode:Socket:Core=0:None:None\n TRES_req=cpu=72,mem=250000M,node=1,billing=72\n TRES_alloc=cpu=72,node=1,billing=72\n Command=/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/moreprototypesYury/gen_131_CuTe/cfg/Ni2Al2/ams_pipeline_job.sh\n WorkDir=/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/moreprototypesYury/gen_131_CuTe/cfg/Ni2Al2\n StdErr=\n StdOut=ams_pipeline.o%j\n"},"startTime":1675954353,"statistics":{"clock":{"unit":{"base":"Hz","prefix":"M"},"avg":1336.519,"min":801.564,"max":2348.215},"cpu_load":{"unit":{"base":""},"avg":31.64,"min":17.36,"max":45.54},"cpu_power":{"unit":{"base":"W"},"avg":150.018,"min":93.672,"max":261.592},"cpu_user":{"unit":{"base":""},"avg":28.518,"min":0.09,"max":57.343},"flops_any":{"unit":{"base":"F/s","prefix":"G"},"avg":45.012,"min":0,"max":135.037},"flops_dp":{"unit":{"base":"F/s","prefix":"G"},"avg":22.496,"min":0,"max":67.488},"flops_sp":{"unit":{"base":"F/s","prefix":"G"},"avg":0.02,"min":0,"max":0.061},"ib_recv":{"unit":{"base":"B/s"},"avg":14442.82,"min":219.998,"max":42581.368},"ib_recv_pkts":{"unit":{"base":"packets/s"},"avg":201.532,"min":1.25,"max":601.345},"ib_xmit":{"unit":{"base":"B/s"},"avg":282.098,"min":56.2,"max":569.363},"ib_xmit_pkts":{"unit":{"base":"packets/s"},"avg":1.228,"min":0.433,"max":2},"ipc":{"unit":{"base":"IPC"},"avg":0.77,"min":0.564,"max":0.906},"mem_bw":{"unit":{"base":"B/s","prefix":"G"},"avg":4.872,"min":0.025,"max":14.552},"mem_power":{"unit":{"base":"W"},"avg":7.725,"min":6.286,"max":10.556},"mem_used":{"unit":{"base":"B","prefix":"G"},"avg":6.162,"min":6.103,"max":6.226},"nfs4_read":{"unit":{"base":"B/s","prefix":"M"},"avg":1045.333,"min":311,"max":1525},"nfs4_total":{"unit":{"base":"B/s","prefix":"M"},"avg":6430,"min":2796,"max":11518},"nfs4_write":{"unit":{"base":"B/s","prefix":"M"},"avg":24.333,"min":0,"max":38},"vectorization_ratio":{"unit":{"base":"%"},"avg":25.528,"min":0,"max":76.585}}}
|
||||
{"jobId":398764,"user":"k106eb10","project":"k106eb","cluster":"fritz","subCluster":"main","numNodes":1,"exclusive":1,"jobState":"completed","duration":177,"resources":[{"hostname":"f0649"}],"startTime":1675954353,"statistics":{"clock":{"unit":{"base":"Hz","prefix":"M"},"avg":1336.519,"min":801.564,"max":2348.215},"cpu_load":{"unit":{"base":""},"avg":31.64,"min":17.36,"max":45.54},"cpu_power":{"unit":{"base":"W"},"avg":150.018,"min":93.672,"max":261.592},"cpu_user":{"unit":{"base":""},"avg":28.518,"min":0.09,"max":57.343},"flops_any":{"unit":{"base":"F/s","prefix":"G"},"avg":45.012,"min":0,"max":135.037},"flops_dp":{"unit":{"base":"F/s","prefix":"G"},"avg":22.496,"min":0,"max":67.488},"flops_sp":{"unit":{"base":"F/s","prefix":"G"},"avg":0.02,"min":0,"max":0.061},"ib_recv":{"unit":{"base":"B/s"},"avg":14442.82,"min":219.998,"max":42581.368},"ib_recv_pkts":{"unit":{"base":"packets/s"},"avg":201.532,"min":1.25,"max":601.345},"ib_xmit":{"unit":{"base":"B/s"},"avg":282.098,"min":56.2,"max":569.363},"ib_xmit_pkts":{"unit":{"base":"packets/s"},"avg":1.228,"min":0.433,"max":2},"ipc":{"unit":{"base":"IPC"},"avg":0.77,"min":0.564,"max":0.906},"mem_bw":{"unit":{"base":"B/s","prefix":"G"},"avg":4.872,"min":0.025,"max":14.552},"mem_power":{"unit":{"base":"W"},"avg":7.725,"min":6.286,"max":10.556},"mem_used":{"unit":{"base":"B","prefix":"G"},"avg":6.162,"min":6.103,"max":6.226},"nfs4_read":{"unit":{"base":"B/s","prefix":"M"},"avg":1045.333,"min":311,"max":1525},"nfs4_total":{"unit":{"base":"B/s","prefix":"M"},"avg":6430,"min":2796,"max":11518},"nfs4_write":{"unit":{"base":"B/s","prefix":"M"},"avg":24.333,"min":0,"max":38},"vectorization_ratio":{"unit":{"base":"%"},"avg":25.528,"min":0,"max":76.585}}}
|
||||
|
||||
98
internal/importer/testdata/meta-taurus.input
vendored
98
internal/importer/testdata/meta-taurus.input
vendored
@@ -1,98 +0,0 @@
|
||||
{
|
||||
"jobId": 20639587,
|
||||
"user": "s3804552",
|
||||
"project": "p_speichersysteme",
|
||||
"cluster": "taurus",
|
||||
"subCluster": "haswell",
|
||||
"partition": "haswell64",
|
||||
"numNodes": 2,
|
||||
"numHwthreads": 4,
|
||||
"exclusive": 0,
|
||||
"startTime": 1635856524,
|
||||
"jobState": "completed",
|
||||
"duration": 310,
|
||||
"walltime": 3600,
|
||||
"smt": 0,
|
||||
"resources": [
|
||||
{
|
||||
"hostname": "taurusi6489",
|
||||
"hwthreads": [
|
||||
0,
|
||||
1
|
||||
]
|
||||
},
|
||||
{
|
||||
"hostname": "taurusi6490",
|
||||
"hwthreads": [
|
||||
10,
|
||||
11
|
||||
]
|
||||
}
|
||||
],
|
||||
"statistics": {
|
||||
"cpu_used": {
|
||||
"min": 0.03694102397926118,
|
||||
"avg": 0.48812580468611544,
|
||||
"max": 1.0000000000000002,
|
||||
"unit": {
|
||||
"base": ""
|
||||
}
|
||||
},
|
||||
"ipc": {
|
||||
"min": 0.30469640475234366,
|
||||
"avg": 1.154312070173657,
|
||||
"max": 1.797623522191001,
|
||||
"unit": {
|
||||
"base": "IPC"
|
||||
}
|
||||
},
|
||||
"flops_any": {
|
||||
"min": 0.0,
|
||||
"avg": 686.5190320308598,
|
||||
"max": 4346.591400350933,
|
||||
"unit": {
|
||||
"base": "F/s"
|
||||
}
|
||||
},
|
||||
"mem_bw": {
|
||||
"min": 653671812.1661415,
|
||||
"avg": 1605031604.9852366,
|
||||
"max": 2614718291.9554267,
|
||||
"unit": {
|
||||
"base": "B/s"
|
||||
}
|
||||
},
|
||||
"file_bw": {
|
||||
"min": 0.0,
|
||||
"avg": 620592.5419124186,
|
||||
"max": 11559156.360352296,
|
||||
"unit": {
|
||||
"base": "B/s"
|
||||
}
|
||||
},
|
||||
"net_bw": {
|
||||
"min": 126779.89655880642,
|
||||
"avg": 763101.082138246,
|
||||
"max": 1916309.7075416835,
|
||||
"unit": {
|
||||
"base": "B/s"
|
||||
}
|
||||
},
|
||||
"mem_used": {
|
||||
"min": 2779066368.0,
|
||||
"avg": 9647598685.09091,
|
||||
"max": 10202595328.0,
|
||||
"unit": {
|
||||
"base": "B"
|
||||
}
|
||||
},
|
||||
"cpu_power": {
|
||||
"min": 35.50647456742635,
|
||||
"avg": 78.63442946337237,
|
||||
"max": 85.83909286117324,
|
||||
"unit": {
|
||||
"base": "W"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"jobId": 398764,
|
||||
"cluster": "fritz",
|
||||
"startTime": 1675954353,
|
||||
"duration": 3400
|
||||
}
|
||||
@@ -96,6 +96,19 @@ func scanJob(row interface{ Scan(...interface{}) error }) (*schema.Job, error) {
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (r *JobRepository) Flush() error {
|
||||
var err error
|
||||
|
||||
switch r.driver {
|
||||
case "sqlite3":
|
||||
_, err = r.DB.Exec(`DELETE FROM job`)
|
||||
case "mysql":
|
||||
_, err = r.DB.Exec(`TRUNCATE TABLE job`)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *JobRepository) FetchJobName(job *schema.Job) (*string, error) {
|
||||
start := time.Now()
|
||||
cachekey := fmt.Sprintf("metadata:%d", job.ID)
|
||||
@@ -695,9 +708,10 @@ func (r *JobRepository) JobsStatistics(ctx context.Context,
|
||||
stats := map[string]*model.JobsStatistics{}
|
||||
var castType string
|
||||
|
||||
if r.driver == "sqlite3" {
|
||||
switch r.driver {
|
||||
case "sqlite3":
|
||||
castType = "int"
|
||||
} else if r.driver == "mysql" {
|
||||
case "mysql":
|
||||
castType = "unsigned"
|
||||
}
|
||||
|
||||
|
||||
@@ -24,8 +24,8 @@ var migrationFiles embed.FS
|
||||
func checkDBVersion(backend string, db *sql.DB) error {
|
||||
var m *migrate.Migrate
|
||||
|
||||
if backend == "sqlite3" {
|
||||
|
||||
switch backend {
|
||||
case "sqlite3":
|
||||
driver, err := sqlite3.WithInstance(db, &sqlite3.Config{})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -39,7 +39,7 @@ func checkDBVersion(backend string, db *sql.DB) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else if backend == "mysql" {
|
||||
case "mysql":
|
||||
driver, err := mysql.WithInstance(db, &mysql.Config{})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -65,11 +65,11 @@ func checkDBVersion(backend string, db *sql.DB) error {
|
||||
}
|
||||
|
||||
if v < Version {
|
||||
return fmt.Errorf("Unsupported database version %d, need %d.\nPlease backup your database file and run cc-backend --migrate-db", v, Version)
|
||||
return fmt.Errorf("unsupported database version %d, need %d.\nPlease backup your database file and run cc-backend --migrate-db", v, Version)
|
||||
}
|
||||
|
||||
if v > Version {
|
||||
return fmt.Errorf("Unsupported database version %d, need %d.\nPlease refer to documentation how to downgrade db with external migrate tool!", v, Version)
|
||||
return fmt.Errorf("unsupported database version %d, need %d.\nPlease refer to documentation how to downgrade db with external migrate tool", v, Version)
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -78,7 +78,8 @@ func checkDBVersion(backend string, db *sql.DB) error {
|
||||
func MigrateDB(backend string, db string) error {
|
||||
var m *migrate.Migrate
|
||||
|
||||
if backend == "sqlite3" {
|
||||
switch backend {
|
||||
case "sqlite3":
|
||||
d, err := iofs.New(migrationFiles, "migrations/sqlite3")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
@@ -88,7 +89,7 @@ func MigrateDB(backend string, db string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else if backend == "mysql" {
|
||||
case "mysql":
|
||||
d, err := iofs.New(migrationFiles, "migrations/mysql")
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -34,11 +34,13 @@ func (r *JobRepository) QueryJobs(
|
||||
|
||||
if order != nil {
|
||||
field := toSnakeCase(order.Field)
|
||||
if order.Order == model.SortDirectionEnumAsc {
|
||||
|
||||
switch order.Order {
|
||||
case model.SortDirectionEnumAsc:
|
||||
query = query.OrderBy(fmt.Sprintf("job.%s ASC", field))
|
||||
} else if order.Order == model.SortDirectionEnumDesc {
|
||||
case model.SortDirectionEnumDesc:
|
||||
query = query.OrderBy(fmt.Sprintf("job.%s DESC", field))
|
||||
} else {
|
||||
default:
|
||||
return nil, errors.New("REPOSITORY/QUERY > invalid sorting order")
|
||||
}
|
||||
}
|
||||
@@ -117,7 +119,7 @@ func SecurityCheck(ctx context.Context, query sq.SelectBuilder) (queryOut sq.Sel
|
||||
return query.Where("job.user = ?", user.Username), nil
|
||||
} else { // Unauthorized : Error
|
||||
var qnil sq.SelectBuilder
|
||||
return qnil, errors.New(fmt.Sprintf("User '%s' with unknown roles! [%#v]\n", user.Username, user.Roles))
|
||||
return qnil, fmt.Errorf("user '%s' with unknown roles [%#v]", user.Username, user.Roles)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user