From 5c715b6ec24e6157e7173197b4c2a6c700d52289 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Fri, 23 Sep 2022 15:23:45 +0200 Subject: [PATCH] Add test for import job handler. Improve error messages. --- cmd/cc-backend/main.go | 2 +- internal/config/config.go | 2 +- internal/repository/init.go | 4 +- internal/repository/init_test.go | 266 +++++++++++++++++ internal/repository/job.go | 3 +- pkg/archive/fsBackend.go | 2 +- test/data.json | 488 +++++++++++++++++++++++++++++++ test/meta.json | 78 +++++ 8 files changed, 838 insertions(+), 7 deletions(-) create mode 100644 internal/repository/init_test.go create mode 100644 test/data.json create mode 100644 test/meta.json diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index 37a7418..b343066 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -54,7 +54,7 @@ func main() { flag.BoolVar(&flagServer, "server", false, "Start a server, continues listening on port after initialization and argument handling") flag.BoolVar(&flagGops, "gops", false, "Listen via github.com/google/gops/agent (for debugging)") flag.BoolVar(&flagDev, "dev", false, "Enable development components: GraphQL Playground and Swagger UI") - flag.StringVar(&flagConfigFile, "config", "./config.json", "Overwrite the global config options by those specified in `config.json`") + flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`") flag.StringVar(&flagNewUser, "add-user", "", "Add a new user. Argument format: `:[admin,support,api,user]:`") flag.StringVar(&flagDelUser, "del-user", "", "Remove user by `username`") flag.StringVar(&flagGenJWT, "jwt", "", "Generate and print a JWT for the user specified by its `username`") diff --git a/internal/config/config.go b/internal/config/config.go index 2ee882e..ab996ff 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -53,7 +53,7 @@ func Init(flagConfigFile string) { } } else { if err := schema.Validate(schema.Config, bytes.NewReader(raw)); err != nil { - log.Fatal(err) + log.Fatalf("Validate config: %v\n", err) } dec := json.NewDecoder(bytes.NewReader(raw)) dec.DisallowUnknownFields() diff --git a/internal/repository/init.go b/internal/repository/init.go index fcc2c31..76973eb 100644 --- a/internal/repository/init.go +++ b/internal/repository/init.go @@ -105,7 +105,7 @@ func HandleImportFlag(flag string) error { if config.Keys.Validate { if err := schema.Validate(schema.Meta, bytes.NewReader(raw)); err != nil { - return err + return fmt.Errorf("validate job meta: %v", err) } } dec := json.NewDecoder(bytes.NewReader(raw)) @@ -122,7 +122,7 @@ func HandleImportFlag(flag string) error { if config.Keys.Validate { if err := schema.Validate(schema.Data, bytes.NewReader(raw)); err != nil { - return err + return fmt.Errorf("validate job data: %v", err) } } dec = json.NewDecoder(bytes.NewReader(raw)) diff --git a/internal/repository/init_test.go b/internal/repository/init_test.go new file mode 100644 index 0000000..8b31484 --- /dev/null +++ b/internal/repository/init_test.go @@ -0,0 +1,266 @@ +// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +package repository + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/ClusterCockpit/cc-backend/internal/config" + "github.com/ClusterCockpit/cc-backend/internal/metricdata" + "github.com/ClusterCockpit/cc-backend/pkg/archive" + _ "github.com/mattn/go-sqlite3" +) + +func init() { + Connect("sqlite3", "../../test/test.db") +} + +const testconfig = `{ + "addr": "0.0.0.0:8080", + "validate": false, + "archive": { + "kind": "file", + "path": "./var/job-archive" + }, + "clusters": [ + { + "name": "taurus", + "metricDataRepository": {"kind": "test", "url": "bla:8081"}, + "filterRanges": { + "numNodes": { "from": 1, "to": 4000 }, + "duration": { "from": 0, "to": 604800 }, + "startTime": { "from": "2010-01-01T00:00:00Z", "to": null } + } + } + ] +}` +const testclusterJson = `{ + "name": "taurus", + "SubClusters": [ + { + "name": "haswell", + "processorType": "Intel Haswell", + "socketsPerNode": 2, + "coresPerSocket": 12, + "threadsPerCore": 1, + "flopRateScalar": 32, + "flopRateSimd": 512, + "memoryBandwidth": 60, + "topology": { + "node": [ 0, 1 ], + "socket": [ + [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], + [ 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 ] + ], + "memoryDomain": [ + [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], + [ 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 ] + ], + "core": [ [ 0 ], [ 1 ], [ 2 ], [ 3 ], [ 4 ], [ 5 ], [ 6 ], [ 7 ], [ 8 ], [ 9 ], [ 10 ], [ 11 ], [ 12 ], [ 13 ], [ 14 ], [ 15 ], [ 16 ], [ 17 ], [ 18 ], [ 19 ], [ 20 ], [ 21 ], [ 22 ], [ 23 ] ] + } + } + ], + "metricConfig": [ + { + "name": "cpu_used", + "scope": "core", + "unit": "", + "timestep": 30, + "subClusters": [ + { + "name": "haswell", + "peak": 1, + "normal": 0.5, + "caution": 2e-07, + "alert": 1e-07 + } + ] + }, + { + "name": "ipc", + "scope": "core", + "unit": "IPC", + "timestep": 60, + "subClusters": [ + { + "name": "haswell", + "peak": 2, + "normal": 1, + "caution": 0.1, + "alert": 0.5 + } + ] + }, + { + "name": "flops_any", + "scope": "core", + "unit": "F/s", + "timestep": 60, + "subClusters": [ + { + "name": "haswell", + "peak": 40000000000, + "normal": 20000000000, + "caution": 30000000000, + "alert": 35000000000 + } + ] + }, + { + "name": "mem_bw", + "scope": "socket", + "unit": "B/s", + "timestep": 60, + "subClusters": [ + { + "name": "haswell", + "peak": 58800000000, + "normal": 28800000000, + "caution": 38800000000, + "alert": 48800000000 + } + ] + }, + { + "name": "file_bw", + "scope": "node", + "unit": "B/s", + "timestep": 30, + "subClusters": [ + { + "name": "haswell", + "peak": 20000000000, + "normal": 5000000000, + "caution": 9000000000, + "alert": 19000000000 + } + ] + }, + { + "name": "net_bw", + "scope": "node", + "unit": "B/s", + "timestep": 30, + "subClusters": [ + { + "name": "haswell", + "peak": 7000000000, + "normal": 5000000000, + "caution": 6000000000, + "alert": 6500000000 + } + ] + }, + { + "name": "mem_used", + "scope": "node", + "unit": "B", + "timestep": 30, + "subClusters": [ + { + "name": "haswell", + "peak": 32000000000, + "normal": 2000000000, + "caution": 31000000000, + "alert": 30000000000 + } + ] + }, + { + "name": "cpu_power", + "scope": "socket", + "unit": "W", + "timestep": 60, + "subClusters": [ + { + "name": "haswell", + "peak": 100, + "normal": 80, + "caution": 90, + "alert": 90 + } + ] + } + ] + }` + +func TestImportFlag(t *testing.T) { + tmpdir := t.TempDir() + jobarchive := filepath.Join(tmpdir, "job-archive") + if err := os.Mkdir(jobarchive, 0777); err != nil { + t.Fatal(err) + } + + if err := os.Mkdir(filepath.Join(jobarchive, "testcluster"), 0777); err != nil { + t.Fatal(err) + } + + if err := os.WriteFile(filepath.Join(jobarchive, "testcluster", "cluster.json"), []byte(testclusterJson), 0666); err != nil { + t.Fatal(err) + } + + dbfilepath := filepath.Join(tmpdir, "test.db") + f, err := os.Create(dbfilepath) + if err != nil { + t.Fatal(err) + } + f.Close() + + cfgFilePath := filepath.Join(tmpdir, "config.json") + if err := os.WriteFile(cfgFilePath, []byte(testconfig), 0666); err != nil { + t.Fatal(err) + } + + config.Init(cfgFilePath) + archiveCfg := fmt.Sprintf("{\"kind\": \"file\",\"path\": \"%s\"}", jobarchive) + + Connect("sqlite3", dbfilepath) + db := GetConnection() + + if err := archive.Init(json.RawMessage(archiveCfg)); err != nil { + t.Fatal(err) + } + + if err := metricdata.Init(config.Keys.DisableArchive); err != nil { + t.Fatal(err) + } + + if _, err := db.DB.Exec(JobsDBSchema); err != nil { + t.Fatal(err) + } + + t.Run("Job 20639587", func(t *testing.T) { + if err := HandleImportFlag("../../test/meta.json:../../test/data.json"); err != nil { + t.Fatal(err) + } + + repo := GetJobRepository() + jobId := int64(20639587) + cluster := "taurus" + startTime := int64(1635856524) + job, err := repo.Find(&jobId, &cluster, &startTime) + if err != nil { + t.Fatal(err) + } + + if job.NumNodes != 2 { + t.Errorf("NumNode: Received %d, expected 2", job.NumNodes) + } + + ar := archive.GetHandle() + data, err := ar.LoadJobData(job) + if err != nil { + t.Fatal(err) + } + + if len(data) != 8 { + t.Errorf("Job data length: Got %d, want 8", len(data)) + } + }) +} diff --git a/internal/repository/job.go b/internal/repository/job.go index e546471..d28fdfd 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -160,8 +160,7 @@ func (r *JobRepository) Find( // The job is queried using the database id. // It returns a pointer to a schema.Job data structure and an error variable. // To check if no job was found test err == sql.ErrNoRows -func (r *JobRepository) FindById( - jobId int64) (*schema.Job, error) { +func (r *JobRepository) FindById(jobId int64) (*schema.Job, error) { q := sq.Select(jobColumns...). From("job").Where("job.id = ?", jobId) return scanJob(q.RunWith(r.stmtCache).QueryRow()) diff --git a/pkg/archive/fsBackend.go b/pkg/archive/fsBackend.go index 80352a4..d2726a8 100644 --- a/pkg/archive/fsBackend.go +++ b/pkg/archive/fsBackend.go @@ -109,7 +109,7 @@ func (fsa *FsArchive) LoadClusterCfg(name string) (*schema.Cluster, error) { } if config.Keys.Validate { if err := schema.Validate(schema.ClusterCfg, bytes.NewReader(b)); err != nil { - return &schema.Cluster{}, err + return &schema.Cluster{}, fmt.Errorf("Validate cluster config: %v\n", err) } } return DecodeCluster(bytes.NewReader(b)) diff --git a/test/data.json b/test/data.json new file mode 100644 index 0000000..5a31685 --- /dev/null +++ b/test/data.json @@ -0,0 +1,488 @@ +{ + "cpu_used": { + "core": { + "unit": "cpu used", + "scope": "core", + "timestep": 30, + "series": [ + { + "hostname": "taurusi6489", + "id": 0, + "statistics": { + "min": 0.09090909090909093, + "avg": 0.9173553719008265, + "max": 1.0000000000000002 + }, + "data": [ + 0.09090909090909093, + 0.9999999999999999, + 1.0, + 1.0000000000000002, + 1.0, + 1.0000000000000002, + 0.9999999999999999, + 1.0, + 1.0, + 1.0, + 1.0 + ] + }, + { + "hostname": "taurusi6489", + "id": 1, + "statistics": { + "min": 0.03694102397926118, + "avg": 0.045968409230268584, + "max": 0.08809840425531917 + }, + "data": [ + 0.08809840425531917, + 0.05710659898477157, + 0.04034861200774694, + 0.037962362102530824, + 0.03976721629485936, + 0.04163976759199483, + 0.03694102397926118, + 0.03821243523316062, + 0.03851132686084142, + 0.044752092723760455, + 0.04231266149870802 + ] + }, + { + "hostname": "taurusi6490", + "id": 10, + "statistics": { + "min": 0.10505319148936171, + "avg": 0.9186411992263056, + "max": 1.0000000000000002 + }, + "data": [ + 0.10505319148936171, + 1.0000000000000002, + 1.0, + 1.0, + 1.0, + 0.9999999999999999, + 1.0, + 0.9999999999999999, + 1.0, + 1.0, + 1.0 + ] + }, + { + "hostname": "taurusi6490", + "id": 11, + "statistics": { + "min": 0.05286048845767815, + "avg": 0.07053823838706144, + "max": 0.075148113501715 + }, + "data": [ + 0.05286048845767815, + 0.06936597614563718, + 0.07254534083802376, + 0.075148113501715, + 0.06909547738693468, + 0.07372696032489846, + 0.07077983088005012, + 0.07082419304293325, + 0.07424812030075188, + 0.07285803627267043, + 0.07446808510638298 + ] + } + ], + "statisticsSeries": null + } + }, + "ipc": { + "core": { + "unit": "IPC", + "scope": "core", + "timestep": 60, + "series": [ + { + "hostname": "taurusi6489", + "id": 0, + "statistics": { + "min": 1.3808406263195592, + "avg": 1.3960848578375105, + "max": 1.4485575599350569 + }, + "data": [ + 1.4485575599350569, + 1.3808406263195592, + 1.3830284413690626, + 1.3836692663348698, + 1.3843283952290035 + ] + }, + { + "hostname": "taurusi6489", + "id": 1, + "statistics": { + "min": 0.30469640475234366, + "avg": 0.8816944294664065, + "max": 1.797623522191001 + }, + "data": [ + 1.797623522191001, + 0.954395633726228, + 1.0019972349956185, + 0.30469640475234366, + 0.3497593516668412 + ] + }, + { + "hostname": "taurusi6490", + "id": 10, + "statistics": { + "min": 1.3791232173760588, + "avg": 1.3850247295506815, + "max": 1.386710405495511 + }, + "data": [ + 1.3791232173760588, + 1.38619977419787, + 1.386397917938246, + 1.3866923327457215, + 1.386710405495511 + ] + }, + { + "hostname": "taurusi6490", + "id": 11, + "statistics": { + "min": 0.6424094604392216, + "avg": 0.9544442638400293, + "max": 1.2706704244636826 + }, + "data": [ + 1.2706704244636826, + 0.6424094604392216, + 0.9249973908234796, + 0.6940110823242276, + 1.2401329611495353 + ] + } + ], + "statisticsSeries": null + } + }, + "flops_any": { + "core": { + "unit": "F/s", + "scope": "core", + "timestep": 60, + "series": [ + { + "hostname": "taurusi6489", + "id": 0, + "statistics": { + "min": 0.0, + "avg": 184.2699002412084, + "max": 921.3495012060421 + }, + "data": [ + 921.3495012060421, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + { + "hostname": "taurusi6489", + "id": 1, + "statistics": { + "min": 0.13559227208748068, + "avg": 273.2997868356056, + "max": 1355.9227390817396 + }, + "data": [ + 1355.9227390817396, + 8.94908797747172, + 0.6779613312519499, + 0.13559227208748068, + 0.8135535154771758 + ] + }, + { + "hostname": "taurusi6490", + "id": 10, + "statistics": { + "min": 0.0, + "avg": 1678.8419461262179, + "max": 4346.591400350933 + }, + "data": [ + 4346.591400350933, + 0.0, + 578.4248288199713, + 0.0, + 3469.193501460185 + ] + }, + { + "hostname": "taurusi6490", + "id": 11, + "statistics": { + "min": 45.28689133054866, + "avg": 609.6644949204072, + "max": 2582.7080822873186 + }, + "data": [ + 2582.7080822873186, + 45.28689133054866, + 48.67663233623293, + 47.591911855555026, + 324.0589567923803 + ] + } + ], + "statisticsSeries": null + } + }, + "mem_bw": { + "socket": { + "unit": "B/s", + "scope": "socket", + "timestep": 60, + "series": [ + { + "hostname": "taurusi6489", + "id": 0, + "statistics": { + "min": 653671812.1661415, + "avg": 1637585527.5854635, + "max": 2614718291.9554267 + }, + "data": [ + 653671812.1661415, + 2614718291.9554267, + 1732453371.7073724, + 1612865229.8704093, + 1574218932.2279677 + ] + }, + { + "hostname": "taurusi6490", + "id": 0, + "statistics": { + "min": 1520190251.61048, + "avg": 1572477682.3850098, + "max": 1688960732.2760606 + }, + "data": [ + 1688960732.2760606, + 1580140679.8216474, + 1520190251.61048, + 1541841829.6250021, + 1531254918.591859 + ] + } + ], + "statisticsSeries": null + } + }, + "file_bw": { + "node": { + "unit": "B/s", + "scope": "node", + "timestep": 30, + "series": [ + { + "hostname": "taurusi6489", + "statistics": { + "min": 0.0, + "avg": 190352.6328851857, + "max": 2093878.361723524 + }, + "data": [ + 0.0, + 0.0, + 0.0, + 0.6000135186380174, + 0.0, + 0.0, + 2093878.361723524, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + { + "hostname": "taurusi6490", + "statistics": { + "min": 0.0, + "avg": 1050832.4509396513, + "max": 11559156.360352296 + }, + "data": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 11559156.360352296, + 0.0, + 0.5999838690326298, + 0.0, + 0.0 + ] + } + ], + "statisticsSeries": null + } + }, + "net_bw": { + "node": { + "unit": "B/s", + "scope": "node", + "timestep": 30, + "series": [ + { + "hostname": "taurusi6489", + "statistics": { + "min": 126779.89655880642, + "avg": 653834.5091507058, + "max": 1285639.5107541133 + }, + "data": [ + 1158202.7403032137, + 126779.89655880642, + 419017.91939583793, + 345766.3974972795, + 645419.3296982117, + 644667.7333333333, + 1285639.5107541133, + 643481.2108874657, + 640025.3562553325, + 643241.4875354709, + 639938.0184386979 + ] + }, + { + "hostname": "taurusi6490", + "statistics": { + "min": 640156.9862985397, + "avg": 872367.6551257868, + "max": 1916309.7075416835 + }, + "data": [ + 1774843.146788355, + 643218.3646426039, + 641681.1031071587, + 644690.1512268113, + 647183.5650609672, + 644439.3303402043, + 1916309.7075416835, + 643748.3241006166, + 757189.8273227927, + 642583.6999539217, + 640156.9862985397 + ] + } + ], + "statisticsSeries": null + } + }, + "mem_used": { + "node": { + "unit": "B", + "scope": "node", + "timestep": 30, + "series": [ + { + "hostname": "taurusi6489", + "statistics": { + "min": 2779066368.0, + "avg": 9282117259.636364, + "max": 10202595328.0 + }, + "data": [ + 2779066368.0, + 8518217728.0, + 9852760064.0, + 9979805696.0, + 10039619584.0, + 10087104512.0, + 10136084480.0, + 10202595328.0, + 10154196992.0, + 10177409024.0, + 10176430080.0 + ] + }, + { + "hostname": "taurusi6490", + "statistics": { + "min": 9993277440.0, + "avg": 10013080110.545454, + "max": 10039676928.0 + }, + "data": [ + 10001317888.0, + 10013028352.0, + 10006728704.0, + 10039676928.0, + 10035838976.0, + 10033356800.0, + 10006577152.0, + 10005659648.0, + 9993277440.0, + 9993564160.0, + 10014855168.0 + ] + } + ], + "statisticsSeries": null + } + }, + "cpu_power": { + "socket": { + "unit": "W", + "scope": "socket", + "timestep": 60, + "series": [ + { + "hostname": "taurusi6489", + "id": 0, + "statistics": { + "min": 35.50647456742635, + "avg": 72.08313211552377, + "max": 83.33799371150049 + }, + "data": [ + 35.50647456742635, + 75.65022009482759, + 83.33799371150049, + 83.00405043233219, + 82.9169217715322 + ] + }, + { + "hostname": "taurusi6490", + "id": 0, + "statistics": { + "min": 83.8466923147859, + "avg": 85.18572681122097, + "max": 85.83909286117324 + }, + "data": [ + 83.8466923147859, + 85.58816979864088, + 85.31266819129794, + 85.83909286117324, + 85.34201089020692 + ] + } + ], + "statisticsSeries": null + } + } + } \ No newline at end of file diff --git a/test/meta.json b/test/meta.json new file mode 100644 index 0000000..094cb84 --- /dev/null +++ b/test/meta.json @@ -0,0 +1,78 @@ +{ + "jobId": 20639587, + "user": "s3804552", + "project": "p_speichersysteme", + "cluster": "taurus", + "subCluster": "haswell", + "partition": "haswell64", + "arrayJobId": 0, + "numNodes": 2, + "numHwthreads": 4, + "numAcc": 0, + "exclusive": 0, + "startTime": 1635856524, + "jobState": "completed", + "duration": 310, + "walltime": 3600, + "smt": 0, + "resources": [ + { + "hostname": "taurusi6489", + "hwthreads": [ 0, 1 ] + }, + { + "hostname": "taurusi6490", + "hwthreads": [ 10, 11 ] + } + ], + "statistics": { + "cpu_used": { + "min": 0.03694102397926118, + "avg": 0.48812580468611544, + "max": 1.0000000000000002, + "unit": "cpu used" + }, + "ipc": { + "min": 0.30469640475234366, + "avg": 1.154312070173657, + "max": 1.797623522191001, + "unit": "IPC" + }, + "flops_any": { + "min": 0.0, + "avg": 686.5190320308598, + "max": 4346.591400350933, + "unit": "F/s" + }, + "mem_bw": { + "min": 653671812.1661415, + "avg": 1605031604.9852366, + "max": 2614718291.9554267, + "unit": "B/s" + }, + "file_bw": { + "min": 0.0, + "avg": 620592.5419124186, + "max": 11559156.360352296, + "unit": "B/s" + }, + "net_bw": { + "min": 126779.89655880642, + "avg": 763101.082138246, + "max": 1916309.7075416835, + "unit": "B/s" + }, + "mem_used": { + "min": 2779066368.0, + "avg": 9647598685.09091, + "max": 10202595328.0, + "unit": "B" + }, + "cpu_power": { + "min": 35.50647456742635, + "avg": 78.63442946337237, + "max": 85.83909286117324, + "unit": "W" + } + } + } \ No newline at end of file