diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e1f1b7b..db99fb2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,11 +5,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Install Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v4 with: - go-version: 1.17.x + go-version: 1.19.x - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Build, Vet & Test run: | go build ./... diff --git a/Makefile b/Makefile index 3901ea6..3a03b45 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,16 @@ TARGET = ./cc-backend VAR = ./var -DB = ./var/job.db +CFG = config.json .env FRONTEND = ./web/frontend VERSION = 1 GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development') CURRENT_TIME = $(shell date +"%Y-%m-%d:T%H:%M:%S") LD_FLAGS = '-s -X main.buildTime=${CURRENT_TIME} -X main.version=${VERSION} -X main.hash=${GIT_HASH}' +EXECUTABLES = go npm +K := $(foreach exec,$(EXECUTABLES),\ + $(if $(shell which $(exec)),some string,$(error "No $(exec) in PATH"))) + SVELTE_COMPONENTS = status \ analysis \ node \ @@ -28,17 +32,24 @@ SVELTE_SRC = $(wildcard $(FRONTEND)/src/*.svelte) \ .NOTPARALLEL: -$(TARGET): $(VAR) $(DB) $(SVELTE_TARGETS) +$(TARGET): $(VAR) $(CFG) $(SVELTE_TARGETS) $(info ===> BUILD cc-backend) @go build -ldflags=${LD_FLAGS} ./cmd/cc-backend clean: $(info ===> CLEAN) @go clean - @rm $(TARGET) + @rm -f $(TARGET) + +distclean: + @$(MAKE) clean + $(info ===> DISTCLEAN) + @rm -rf $(FRONTEND)/node_modules + @rm -rf $(VAR) test: $(info ===> TESTING) + @go clean -testcache @go build ./... @go vet ./... @go test ./... @@ -49,15 +60,18 @@ tags: $(VAR): @mkdir $(VAR) - @touch ./var/job.db - cd web/frontend && yarn install -$(DB): - ./cc-backend --migrate-db +config.json: + $(info ===> Initialize config.json file) + @cp configs/config.json config.json + +.env: + $(info ===> Initialize .env file) + @cp configs/env-template.txt .env $(SVELTE_TARGETS): $(SVELTE_SRC) $(info ===> BUILD frontend) - cd web/frontend && yarn build + cd web/frontend && npm install && npm run build install: $(TARGET) @WORKSPACE=$(PREFIX) diff --git a/README.md b/README.md index 9823694..b09f6fd 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ versions of third party packages. ## Demo Setup We provide a shell skript that downloads demo data and automatically builds and starts cc-backend. -You need `wget`, `go`, and `yarn` in your path to start the demo. The demo will download 32MB of data (223MB on disk). +You need `wget`, `go`, `node`, `rollup` and `yarn` in your path to start the demo. The demo will download 32MB of data (223MB on disk). ```sh git clone https://github.com/ClusterCockpit/cc-backend.git diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index 360d6d7..638aab7 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -7,6 +7,7 @@ package main import ( "context" "crypto/tls" + "encoding/json" "errors" "flag" "fmt" @@ -29,13 +30,16 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/graph" "github.com/ClusterCockpit/cc-backend/internal/graph/generated" + "github.com/ClusterCockpit/cc-backend/internal/importer" "github.com/ClusterCockpit/cc-backend/internal/metricdata" "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/internal/routerConfig" "github.com/ClusterCockpit/cc-backend/internal/runtimeEnv" "github.com/ClusterCockpit/cc-backend/pkg/archive" "github.com/ClusterCockpit/cc-backend/pkg/log" + "github.com/ClusterCockpit/cc-backend/pkg/schema" "github.com/ClusterCockpit/cc-backend/web" + "github.com/go-co-op/gocron" "github.com/google/gops/agent" "github.com/gorilla/handlers" "github.com/gorilla/mux" @@ -116,7 +120,10 @@ func main() { } if flagMigrateDB { - repository.MigrateDB(config.Keys.DBDriver, config.Keys.DB) + err := repository.MigrateDB(config.Keys.DBDriver, config.Keys.DB) + if err != nil { + log.Fatal(err) + } os.Exit(0) } @@ -196,13 +203,13 @@ func main() { } if flagReinitDB { - if err := repository.InitDB(); err != nil { + if err := importer.InitDB(); err != nil { log.Fatalf("failed to re-initialize repository DB: %s", err.Error()) } } if flagImportJob != "" { - if err := repository.HandleImportFlag(flagImportJob); err != nil { + if err := importer.HandleImportFlag(flagImportJob); err != nil { log.Fatalf("job import failed: %s", err.Error()) } } @@ -412,18 +419,95 @@ func main() { api.JobRepository.WaitForArchiving() }() + s := gocron.NewScheduler(time.Local) + if config.Keys.StopJobsExceedingWalltime > 0 { - go func() { - for range time.Tick(30 * time.Minute) { - err := jobRepo.StopJobsExceedingWalltimeBy(config.Keys.StopJobsExceedingWalltime) - if err != nil { - log.Warnf("Error while looking for jobs exceeding their walltime: %s", err.Error()) - } - runtime.GC() + log.Info("Register undead jobs service") + + s.Every(1).Day().At("3:00").Do(func() { + err := jobRepo.StopJobsExceedingWalltimeBy(config.Keys.StopJobsExceedingWalltime) + if err != nil { + log.Warnf("Error while looking for jobs exceeding their walltime: %s", err.Error()) } - }() + runtime.GC() + }) } + var cfg struct { + Compression int `json:"compression"` + Retention schema.Retention `json:"retention"` + } + + cfg.Retention.IncludeDB = true + + if err := json.Unmarshal(config.Keys.Archive, &cfg); err != nil { + log.Warn("Error while unmarshaling raw config json") + } + + switch cfg.Retention.Policy { + case "delete": + log.Info("Register retention delete service") + + s.Every(1).Day().At("4:00").Do(func() { + startTime := time.Now().Unix() - int64(cfg.Retention.Age*24*3600) + jobs, err := jobRepo.FindJobsBefore(startTime) + if err != nil { + log.Warnf("Error while looking for retention jobs: %s", err.Error()) + } + archive.GetHandle().CleanUp(jobs) + + if cfg.Retention.IncludeDB { + cnt, err := jobRepo.DeleteJobsBefore(startTime) + if err != nil { + log.Errorf("Error while deleting retention jobs from db: %s", err.Error()) + } else { + log.Infof("Retention: Removed %d jobs from db", cnt) + } + if err = jobRepo.Optimize(); err != nil { + log.Errorf("Error occured in db optimization: %s", err.Error()) + } + } + }) + case "move": + log.Info("Register retention move service") + + s.Every(1).Day().At("4:00").Do(func() { + startTime := time.Now().Unix() - int64(cfg.Retention.Age*24*3600) + jobs, err := jobRepo.FindJobsBefore(startTime) + if err != nil { + log.Warnf("Error while looking for retention jobs: %s", err.Error()) + } + archive.GetHandle().Move(jobs, cfg.Retention.Location) + + if cfg.Retention.IncludeDB { + cnt, err := jobRepo.DeleteJobsBefore(startTime) + if err != nil { + log.Errorf("Error while deleting retention jobs from db: %s", err.Error()) + } else { + log.Infof("Retention: Removed %d jobs from db", cnt) + } + if err = jobRepo.Optimize(); err != nil { + log.Errorf("Error occured in db optimization: %s", err.Error()) + } + } + }) + } + + if cfg.Compression > 0 { + log.Info("Register compression service") + + s.Every(1).Day().At("5:00").Do(func() { + startTime := time.Now().Unix() - int64(cfg.Compression*24*3600) + jobs, err := jobRepo.FindJobsBefore(startTime) + if err != nil { + log.Warnf("Error while looking for retention jobs: %s", err.Error()) + } + archive.GetHandle().Compress(jobs) + }) + } + + s.StartAsync() + if os.Getenv("GOGC") == "" { debug.SetGCPercent(25) } diff --git a/docs/dev-frontend.md b/docs/dev-frontend.md new file mode 100644 index 0000000..f1bffd4 --- /dev/null +++ b/docs/dev-frontend.md @@ -0,0 +1,33 @@ +## Tips for frontend development + +The frontend assets including the Svelte js files are per default embedded in +the bgo binary. To enable a quick turnaround cycle for web development of the +frontend disable embedding of static assets in `config.json`: +``` +"embed-static-files": false, +"static-files": "./web/frontend/public/", + +``` + +Start the node build process (in directory `./web/frontend`) in development mode: +``` +$ npm run dev +``` + +This will start the build process in listen mode. Whenever you change a source +files the depending javascript targets will be automatically rebuild. +In case the javascript files are minified you may need to set the production +flag by hand to false in `./web/frontend/rollup.config.mjs`: +``` +const production = false +``` + +Usually this should work automatically. + +Because the files are still served by ./cc-backend you have to reload the view +explicitly in your browser. + +A common setup is to have three terminals open: +* One running cc-backend (working directory repository root): `./cc-backend -server -dev` +* Another running npm in developer mode (working directory `./web/frontend`): `npm run dev` +* And the last one editing the frontend source files diff --git a/docs/dev-testing.md b/docs/dev-testing.md new file mode 100644 index 0000000..9ca39c3 --- /dev/null +++ b/docs/dev-testing.md @@ -0,0 +1,34 @@ +## Overview + +We use the standard golang testing environment. + +The following conventions are used: + +* *White box unit tests*: Tests for internal functionality are placed in files +* *Black box unit tests*: Tests for public interfaces are placed in files +with `_test.go` and belong to the package `_test`. +There only exists one package test file per package. +* *Integration tests*: Tests that use multiple componenents are placed in a +package test file. These are named `_test.go` and belong to the +package `_test`. +* *Test assets*: Any required files are placed in a directory `./testdata` +within each package directory. + +## Executing tests + +Visual Studio Code has a very good golang test integration. +For debugging a test this is the recommended solution. + +The Makefile provided by us has a `test` target that executes: +``` +$ go clean -testcache +$ go build ./... +$ go vet ./... +$ go test ./... +``` + +Of course the commands can also be used on the command line. +For details about golang testing refer to the standard documentation: + +* [Testing package](https://pkg.go.dev/testing) +* [go test command](https://pkg.go.dev/cmd/go#hdr-Test_packages) diff --git a/go.mod b/go.mod index 672a15b..f684bf8 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.18 require ( github.com/99designs/gqlgen v0.17.24 + github.com/ClusterCockpit/cc-units v0.4.0 github.com/Masterminds/squirrel v1.5.3 github.com/go-ldap/ldap/v3 v3.4.4 github.com/go-sql-driver/mysql v1.7.0 @@ -39,6 +40,7 @@ require ( github.com/felixge/httpsnoop v1.0.3 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect + github.com/go-co-op/gocron v1.25.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/spec v0.20.8 // indirect @@ -54,6 +56,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kr/pretty v0.3.0 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -66,11 +69,15 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/rogpeppe/go-internal v1.8.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/stretchr/testify v1.8.2 // indirect github.com/swaggo/files v1.0.0 // indirect github.com/urfave/cli/v2 v2.24.4 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.uber.org/atomic v1.10.0 // indirect + golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea // indirect golang.org/x/mod v0.8.0 // indirect golang.org/x/net v0.7.0 // indirect golang.org/x/oauth2 v0.5.0 // indirect diff --git a/go.sum b/go.sum index 275b165..0e12d13 100644 --- a/go.sum +++ b/go.sum @@ -81,6 +81,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= +github.com/ClusterCockpit/cc-units v0.4.0 h1:zP5DOu99GmErW0tCDf0gcLrlWt42RQ9dpoONEOh4cI0= +github.com/ClusterCockpit/cc-units v0.4.0/go.mod h1:3S3PAhAayS3pbgcT4q9Vn9VJw22Op51X0YimtG77zBw= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/Masterminds/squirrel v1.5.3 h1:YPpoceAcxuzIljlr5iWpNKaql7hLeG1KLSrhvdHpkZc= @@ -444,6 +446,8 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A= github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= +github.com/go-co-op/gocron v1.25.0 h1:pzAdtily1JVIf6lGby6K0JKzhishgLOllQgNxoYbR+8= +github.com/go-co-op/gocron v1.25.0/go.mod h1:JHrQDY4iE1HZPkgTyoccY4xtDgLbrUwL+xODIbEQdnc= github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= @@ -822,8 +826,9 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= @@ -1005,6 +1010,7 @@ github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/browser v0.0.0-20210706143420-7d21f8c997e2/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -1060,11 +1066,16 @@ github.com/qustavo/sqlhooks/v2 v2.1.0 h1:54yBemHnGHp/7xgT+pxwmIlMSDNYKx5JW5dfRAi github.com/qustavo/sqlhooks/v2 v2.1.0/go.mod h1:aMREyKo7fOKTwiLuWPsaHRXEmtqG4yREztO0idF83AU= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= @@ -1142,8 +1153,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/swaggo/files v1.0.0 h1:1gGXVIeUFCS/dta17rnP0iOpr6CXFwKD7EO5ID233e4= github.com/swaggo/files v1.0.0/go.mod h1:N59U6URJLyU1PQgFqPM7wXLMhJx7QAolnvfQkqO13kc= @@ -1302,6 +1314,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea h1:vLCWI/yYrdEHyN2JzIzPO3aaQJHQdp89IZBA/+azVC4= +golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= diff --git a/test/integration_test.go b/internal/api/api_test.go similarity index 60% rename from test/integration_test.go rename to internal/api/api_test.go index f6c52c0..9e11383 100644 --- a/test/integration_test.go +++ b/internal/api/api_test.go @@ -1,4 +1,8 @@ -package test +// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +package api_test import ( "bytes" @@ -44,16 +48,7 @@ func setup(t *testing.T) *api.RestApi { "duration": { "from": 0, "to": 86400 }, "startTime": { "from": "2022-01-01T00:00:00Z", "to": null } } - }, - { - "name": "taurus", - "metricDataRepository": {"kind": "test", "url": "bla:8081"}, - "filterRanges": { - "numNodes": { "from": 1, "to": 4000 }, - "duration": { "from": 0, "to": 604800 }, - "startTime": { "from": "2010-01-01T00:00:00Z", "to": null } - } - } + } ] }` const testclusterJson = `{ @@ -111,215 +106,6 @@ func setup(t *testing.T) *api.RestApi { } ] }` - const taurusclusterJson = `{ - "name": "taurus", - "subClusters": [ - { - "name": "haswell", - "processorType": "Intel Haswell", - "socketsPerNode": 2, - "coresPerSocket": 12, - "threadsPerCore": 1, - "flopRateScalar": { - "unit": { - "prefix": "G", - "base": "F/s" - }, - "value": 14 - }, - "flopRateSimd": { - "unit": { - "prefix": "G", - "base": "F/s" - }, - "value": 112 - }, - "memoryBandwidth": { - "unit": { - "prefix": "G", - "base": "B/s" - }, - "value": 24 - }, - "numberOfNodes": 70, - "nodes": "w11[27-45,49-63,69-72]", - "topology": { - "node": [ 0, 1 ], - "socket": [ - [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], - [ 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 ] - ], - "memoryDomain": [ - [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ], - [ 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 ] - ], - "core": [ [ 0 ], [ 1 ], [ 2 ], [ 3 ], [ 4 ], [ 5 ], [ 6 ], [ 7 ], [ 8 ], [ 9 ], [ 10 ], [ 11 ], [ 12 ], [ 13 ], [ 14 ], [ 15 ], [ 16 ], [ 17 ], [ 18 ], [ 19 ], [ 20 ], [ 21 ], [ 22 ], [ 23 ] ] - } - } - ], - "metricConfig": [ - { - "name": "cpu_used", - "scope": "core", - "unit": {"base": ""}, - "aggregation": "avg", - "timestep": 30, - "peak": 1, - "normal": 0.5, - "caution": 2e-07, - "alert": 1e-07, - "subClusters": [ - { - "name": "haswell", - "peak": 1, - "normal": 0.5, - "caution": 2e-07, - "alert": 1e-07 - } - ] - }, - { - "name": "ipc", - "scope": "core", - "unit": { "base": "IPC"}, - "aggregation": "avg", - "timestep": 60, - "peak": 2, - "normal": 1, - "caution": 0.1, - "alert": 0.5, - "subClusters": [ - { - "name": "haswell", - "peak": 2, - "normal": 1, - "caution": 0.1, - "alert": 0.5 - } - ] - }, - { - "name": "flops_any", - "scope": "core", - "unit": { "base": "F/s"}, - "aggregation": "sum", - "timestep": 60, - "peak": 40000000000, - "normal": 20000000000, - "caution": 30000000000, - "alert": 35000000000, - "subClusters": [ - { - "name": "haswell", - "peak": 40000000000, - "normal": 20000000000, - "caution": 30000000000, - "alert": 35000000000 - } - ] - }, - { - "name": "mem_bw", - "scope": "socket", - "unit": { "base": "B/s"}, - "aggregation": "sum", - "timestep": 60, - "peak": 58800000000, - "normal": 28800000000, - "caution": 38800000000, - "alert": 48800000000, - "subClusters": [ - { - "name": "haswell", - "peak": 58800000000, - "normal": 28800000000, - "caution": 38800000000, - "alert": 48800000000 - } - ] - }, - { - "name": "file_bw", - "scope": "node", - "unit": { "base": "B/s"}, - "aggregation": "sum", - "timestep": 30, - "peak": 20000000000, - "normal": 5000000000, - "caution": 9000000000, - "alert": 19000000000, - "subClusters": [ - { - "name": "haswell", - "peak": 20000000000, - "normal": 5000000000, - "caution": 9000000000, - "alert": 19000000000 - } - ] - }, - { - "name": "net_bw", - "scope": "node", - "unit": { "base": "B/s"}, - "timestep": 30, - "aggregation": "sum", - "peak": 7000000000, - "normal": 5000000000, - "caution": 6000000000, - "alert": 6500000000, - "subClusters": [ - { - "name": "haswell", - "peak": 7000000000, - "normal": 5000000000, - "caution": 6000000000, - "alert": 6500000000 - } - ] - }, - { - "name": "mem_used", - "scope": "node", - "unit": {"base": "B"}, - "aggregation": "sum", - "timestep": 30, - "peak": 32000000000, - "normal": 2000000000, - "caution": 31000000000, - "alert": 30000000000, - "subClusters": [ - { - "name": "haswell", - "peak": 32000000000, - "normal": 2000000000, - "caution": 31000000000, - "alert": 30000000000 - } - ] - }, - { - "name": "cpu_power", - "scope": "socket", - "unit": {"base": "W"}, - "aggregation": "sum", - "timestep": 60, - "peak": 100, - "normal": 80, - "caution": 90, - "alert": 90, - "subClusters": [ - { - "name": "haswell", - "peak": 100, - "normal": 80, - "caution": 90, - "alert": 90 - } - ] - } - ] - }` log.Init("info", true) tmpdir := t.TempDir() @@ -340,15 +126,11 @@ func setup(t *testing.T) *api.RestApi { t.Fatal(err) } - if err := os.Mkdir(filepath.Join(jobarchive, "taurus"), 0777); err != nil { - t.Fatal(err) - } - - if err := os.WriteFile(filepath.Join(jobarchive, "taurus", "cluster.json"), []byte(taurusclusterJson), 0666); err != nil { - t.Fatal(err) - } dbfilepath := filepath.Join(tmpdir, "test.db") - repository.MigrateDB("sqlite3", dbfilepath) + err := repository.MigrateDB("sqlite3", dbfilepath) + if err != nil { + t.Fatal(err) + } cfgFilePath := filepath.Join(tmpdir, "config.json") if err := os.WriteFile(cfgFilePath, []byte(testconfig), 0666); err != nil { @@ -385,7 +167,7 @@ func cleanup() { /* * This function starts a job, stops it, and then reads its data from the job-archive. * Do not run sub-tests in parallel! Tests should not be run in parallel at all, because -* at least `setup` modifies global state. Log-Output is redirected to /dev/null on purpose. +* at least `setup` modifies global state. */ func TestRestApi(t *testing.T) { restapi := setup(t) @@ -470,15 +252,15 @@ func TestRestApi(t *testing.T) { job.Project != "testproj" || job.Cluster != "testcluster" || job.SubCluster != "sc1" || - *job.Partition != "default" || - *job.Walltime != 3600 || - *job.ArrayJobId != 0 || + job.Partition != "default" || + job.Walltime != 3600 || + job.ArrayJobId != 0 || job.NumNodes != 1 || - *job.NumHWThreads != 8 || - *job.NumAcc != 0 || + job.NumHWThreads != 8 || + job.NumAcc != 0 || job.Exclusive != 1 || job.MonitoringStatus != 1 || - *job.SMT != 1 || + job.SMT != 1 || !reflect.DeepEqual(job.Resources, []*schema.Resource{{Hostname: "host123", HWThreads: []int{0, 1, 2, 3, 4, 5, 6, 7}}}) || job.StartTime.Unix() != 123456789 { t.Fatalf("unexpected job properties: %#v", job) @@ -566,17 +348,7 @@ func TestRestApi(t *testing.T) { } }) - // t.Run("FailedJob", func(t *testing.T) { - // subtestLetJobFail(t, restapi, r) - // }) - - // t.Run("ImportJob", func(t *testing.T) { - // testImportFlag(t) - // }) -} - -func subtestLetJobFail(t *testing.T, restapi *api.RestApi, r *mux.Router) { - const startJobBody string = `{ + const startJobBodyFailed string = `{ "jobId": 12345, "user": "testuser", "project": "testproj", @@ -595,8 +367,8 @@ func subtestLetJobFail(t *testing.T, restapi *api.RestApi, r *mux.Router) { "startTime": 12345678 }` - ok := t.Run("StartJob", func(t *testing.T) { - req := httptest.NewRequest(http.MethodPost, "/api/jobs/start_job/", bytes.NewBuffer([]byte(startJobBody))) + ok := t.Run("StartJobFailed", func(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "/api/jobs/start_job/", bytes.NewBuffer([]byte(startJobBodyFailed))) recorder := httptest.NewRecorder() r.ServeHTTP(recorder, req) @@ -609,7 +381,7 @@ func subtestLetJobFail(t *testing.T, restapi *api.RestApi, r *mux.Router) { t.Fatal("subtest failed") } - const stopJobBody string = `{ + const stopJobBodyFailed string = `{ "jobId": 12345, "cluster": "testcluster", @@ -617,8 +389,8 @@ func subtestLetJobFail(t *testing.T, restapi *api.RestApi, r *mux.Router) { "stopTime": 12355678 }` - ok = t.Run("StopJob", func(t *testing.T) { - req := httptest.NewRequest(http.MethodPost, "/api/jobs/stop_job/", bytes.NewBuffer([]byte(stopJobBody))) + ok = t.Run("StopJobFailed", func(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "/api/jobs/stop_job/", bytes.NewBuffer([]byte(stopJobBodyFailed))) recorder := httptest.NewRecorder() r.ServeHTTP(recorder, req) @@ -642,45 +414,3 @@ func subtestLetJobFail(t *testing.T, restapi *api.RestApi, r *mux.Router) { t.Fatal("subtest failed") } } - -func testImportFlag(t *testing.T) { - if err := repository.HandleImportFlag("meta.json:data.json"); err != nil { - t.Fatal(err) - } - - repo := repository.GetJobRepository() - jobId := int64(20639587) - cluster := "taurus" - startTime := int64(1635856524) - job, err := repo.Find(&jobId, &cluster, &startTime) - if err != nil { - t.Fatal(err) - } - - if job.NumNodes != 2 { - t.Errorf("NumNode: Received %d, expected 2", job.NumNodes) - } - - ar := archive.GetHandle() - data, err := ar.LoadJobData(job) - if err != nil { - t.Fatal(err) - } - - if len(data) != 8 { - t.Errorf("Job data length: Got %d, want 8", len(data)) - } - - r := map[string]string{"mem_used": "GB", "net_bw": "KB/s", - "cpu_power": "W", "cpu_used": "", - "file_bw": "KB/s", "flops_any": "F/s", - "mem_bw": "GB/s", "ipc": "IPC"} - - for name, scopes := range data { - for _, metric := range scopes { - if metric.Unit.Base != r[name] { - t.Errorf("Metric %s unit: Got %s, want %s", name, metric.Unit.Base, r[name]) - } - } - } -} diff --git a/internal/api/rest.go b/internal/api/rest.go index 5bad0ad..ae9e8e9 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -22,6 +22,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/auth" "github.com/ClusterCockpit/cc-backend/internal/graph" "github.com/ClusterCockpit/cc-backend/internal/graph/model" + "github.com/ClusterCockpit/cc-backend/internal/importer" "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/archive" "github.com/ClusterCockpit/cc-backend/pkg/log" @@ -252,7 +253,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { results := make([]*schema.JobMeta, 0, len(jobs)) for _, job := range jobs { if withMetadata { - if _, err := api.JobRepository.FetchMetadata(job); err != nil { + if _, err = api.JobRepository.FetchMetadata(job); err != nil { handleError(err, http.StatusInternalServerError, rw) return } @@ -396,7 +397,7 @@ func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) { if req.State == "" { req.State = schema.JobStateRunning } - if err := repository.SanityChecks(&req.BaseJob); err != nil { + if err := importer.SanityChecks(&req.BaseJob); err != nil { handleError(err, http.StatusBadRequest, rw) return } diff --git a/internal/graph/generated/generated.go b/internal/graph/generated/generated.go index 3c8073d..a791e6c 100644 --- a/internal/graph/generated/generated.go +++ b/internal/graph/generated/generated.go @@ -3420,9 +3420,9 @@ func (ec *executionContext) _Job_walltime(ctx context.Context, field graphql.Col } return graphql.Null } - res := resTmp.(*int64) + res := resTmp.(int64) fc.Result = res - return ec.marshalNInt2ᚖint64(ctx, field.Selections, res) + return ec.marshalNInt2int64(ctx, field.Selections, res) } func (ec *executionContext) fieldContext_Job_walltime(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { @@ -3508,9 +3508,9 @@ func (ec *executionContext) _Job_numHWThreads(ctx context.Context, field graphql } return graphql.Null } - res := resTmp.(*int32) + res := resTmp.(int32) fc.Result = res - return ec.marshalNInt2ᚖint32(ctx, field.Selections, res) + return ec.marshalNInt2int32(ctx, field.Selections, res) } func (ec *executionContext) fieldContext_Job_numHWThreads(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { @@ -3552,9 +3552,9 @@ func (ec *executionContext) _Job_numAcc(ctx context.Context, field graphql.Colle } return graphql.Null } - res := resTmp.(*int32) + res := resTmp.(int32) fc.Result = res - return ec.marshalNInt2ᚖint32(ctx, field.Selections, res) + return ec.marshalNInt2int32(ctx, field.Selections, res) } func (ec *executionContext) fieldContext_Job_numAcc(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { @@ -3596,9 +3596,9 @@ func (ec *executionContext) _Job_SMT(ctx context.Context, field graphql.Collecte } return graphql.Null } - res := resTmp.(*int32) + res := resTmp.(int32) fc.Result = res - return ec.marshalNInt2ᚖint32(ctx, field.Selections, res) + return ec.marshalNInt2int32(ctx, field.Selections, res) } func (ec *executionContext) fieldContext_Job_SMT(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { @@ -3684,9 +3684,9 @@ func (ec *executionContext) _Job_partition(ctx context.Context, field graphql.Co } return graphql.Null } - res := resTmp.(*string) + res := resTmp.(string) fc.Result = res - return ec.marshalNString2ᚖstring(ctx, field.Selections, res) + return ec.marshalNString2string(ctx, field.Selections, res) } func (ec *executionContext) fieldContext_Job_partition(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { @@ -3728,9 +3728,9 @@ func (ec *executionContext) _Job_arrayJobId(ctx context.Context, field graphql.C } return graphql.Null } - res := resTmp.(*int64) + res := resTmp.(int64) fc.Result = res - return ec.marshalNInt2ᚖint64(ctx, field.Selections, res) + return ec.marshalNInt2int64(ctx, field.Selections, res) } func (ec *executionContext) fieldContext_Job_arrayJobId(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { @@ -9034,9 +9034,9 @@ func (ec *executionContext) _Unit_prefix(ctx context.Context, field graphql.Coll if resTmp == nil { return graphql.Null } - res := resTmp.(*string) + res := resTmp.(string) fc.Result = res - return ec.marshalOString2ᚖstring(ctx, field.Selections, res) + return ec.marshalOString2string(ctx, field.Selections, res) } func (ec *executionContext) fieldContext_Unit_prefix(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { @@ -14020,48 +14020,6 @@ func (ec *executionContext) marshalNInt2ᚖint(ctx context.Context, sel ast.Sele return res } -func (ec *executionContext) unmarshalNInt2ᚖint32(ctx context.Context, v interface{}) (*int32, error) { - res, err := graphql.UnmarshalInt32(v) - return &res, graphql.ErrorOnPath(ctx, err) -} - -func (ec *executionContext) marshalNInt2ᚖint32(ctx context.Context, sel ast.SelectionSet, v *int32) graphql.Marshaler { - if v == nil { - if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { - ec.Errorf(ctx, "the requested element is null which the schema does not allow") - } - return graphql.Null - } - res := graphql.MarshalInt32(*v) - if res == graphql.Null { - if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { - ec.Errorf(ctx, "the requested element is null which the schema does not allow") - } - } - return res -} - -func (ec *executionContext) unmarshalNInt2ᚖint64(ctx context.Context, v interface{}) (*int64, error) { - res, err := graphql.UnmarshalInt64(v) - return &res, graphql.ErrorOnPath(ctx, err) -} - -func (ec *executionContext) marshalNInt2ᚖint64(ctx context.Context, sel ast.SelectionSet, v *int64) graphql.Marshaler { - if v == nil { - if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { - ec.Errorf(ctx, "the requested element is null which the schema does not allow") - } - return graphql.Null - } - res := graphql.MarshalInt64(*v) - if res == graphql.Null { - if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { - ec.Errorf(ctx, "the requested element is null which the schema does not allow") - } - } - return res -} - func (ec *executionContext) marshalNJob2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋpkgᚋschemaᚐJobᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.Job) graphql.Marshaler { ret := make(graphql.Array, len(v)) var wg sync.WaitGroup @@ -14684,27 +14642,6 @@ func (ec *executionContext) marshalNString2ᚕstringᚄ(ctx context.Context, sel return ret } -func (ec *executionContext) unmarshalNString2ᚖstring(ctx context.Context, v interface{}) (*string, error) { - res, err := graphql.UnmarshalString(v) - return &res, graphql.ErrorOnPath(ctx, err) -} - -func (ec *executionContext) marshalNString2ᚖstring(ctx context.Context, sel ast.SelectionSet, v *string) graphql.Marshaler { - if v == nil { - if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { - ec.Errorf(ctx, "the requested element is null which the schema does not allow") - } - return graphql.Null - } - res := graphql.MarshalString(*v) - if res == graphql.Null { - if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { - ec.Errorf(ctx, "the requested element is null which the schema does not allow") - } - } - return res -} - func (ec *executionContext) marshalNSubCluster2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋpkgᚋschemaᚐSubClusterᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.SubCluster) graphql.Marshaler { ret := make(graphql.Array, len(v)) var wg sync.WaitGroup diff --git a/internal/importer/handleImport.go b/internal/importer/handleImport.go new file mode 100644 index 0000000..5679d53 --- /dev/null +++ b/internal/importer/handleImport.go @@ -0,0 +1,131 @@ +// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +package importer + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "strings" + "time" + + "github.com/ClusterCockpit/cc-backend/internal/config" + "github.com/ClusterCockpit/cc-backend/internal/repository" + "github.com/ClusterCockpit/cc-backend/pkg/archive" + "github.com/ClusterCockpit/cc-backend/pkg/log" + "github.com/ClusterCockpit/cc-backend/pkg/schema" +) + +// Import all jobs specified as `:,...` +func HandleImportFlag(flag string) error { + r := repository.GetJobRepository() + + for _, pair := range strings.Split(flag, ",") { + files := strings.Split(pair, ":") + if len(files) != 2 { + return fmt.Errorf("REPOSITORY/INIT > invalid import flag format") + } + + raw, err := os.ReadFile(files[0]) + if err != nil { + log.Warn("Error while reading metadata file for import") + return err + } + + if config.Keys.Validate { + if err = schema.Validate(schema.Meta, bytes.NewReader(raw)); err != nil { + return fmt.Errorf("REPOSITORY/INIT > validate job meta: %v", err) + } + } + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + jobMeta := schema.JobMeta{BaseJob: schema.JobDefaults} + if err = dec.Decode(&jobMeta); err != nil { + log.Warn("Error while decoding raw json metadata for import") + return err + } + + raw, err = os.ReadFile(files[1]) + if err != nil { + log.Warn("Error while reading jobdata file for import") + return err + } + + if config.Keys.Validate { + if err = schema.Validate(schema.Data, bytes.NewReader(raw)); err != nil { + return fmt.Errorf("REPOSITORY/INIT > validate job data: %v", err) + } + } + dec = json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + jobData := schema.JobData{} + if err = dec.Decode(&jobData); err != nil { + log.Warn("Error while decoding raw json jobdata for import") + return err + } + + // checkJobData(&jobData) + + jobMeta.MonitoringStatus = schema.MonitoringStatusArchivingSuccessful + + // if _, err = r.Find(&jobMeta.JobID, &jobMeta.Cluster, &jobMeta.StartTime); err != sql.ErrNoRows { + // if err != nil { + // log.Warn("Error while finding job in jobRepository") + // return err + // } + // + // return fmt.Errorf("REPOSITORY/INIT > a job with that jobId, cluster and startTime does already exist") + // } + // + job := schema.Job{ + BaseJob: jobMeta.BaseJob, + StartTime: time.Unix(jobMeta.StartTime, 0), + StartTimeUnix: jobMeta.StartTime, + } + + // TODO: Other metrics... + job.FlopsAnyAvg = loadJobStat(&jobMeta, "flops_any") + job.MemBwAvg = loadJobStat(&jobMeta, "mem_bw") + job.NetBwAvg = loadJobStat(&jobMeta, "net_bw") + job.FileBwAvg = loadJobStat(&jobMeta, "file_bw") + job.RawResources, err = json.Marshal(job.Resources) + if err != nil { + log.Warn("Error while marshaling job resources") + return err + } + job.RawMetaData, err = json.Marshal(job.MetaData) + if err != nil { + log.Warn("Error while marshaling job metadata") + return err + } + + if err = SanityChecks(&job.BaseJob); err != nil { + log.Warn("BaseJob SanityChecks failed") + return err + } + + if err = archive.GetHandle().ImportJob(&jobMeta, &jobData); err != nil { + log.Error("Error while importing job") + return err + } + + id, err := r.InsertJob(&job) + if err != nil { + log.Warn("Error while job db insert") + return err + } + + for _, tag := range job.Tags { + if _, err := r.AddTagOrCreate(id, tag.Type, tag.Name); err != nil { + log.Error("Error while adding or creating tag") + return err + } + } + + log.Infof("successfully imported a new job (jobId: %d, cluster: %s, dbid: %d)", job.JobID, job.Cluster, id) + } + return nil +} diff --git a/internal/importer/importer_test.go b/internal/importer/importer_test.go new file mode 100644 index 0000000..83ba5eb --- /dev/null +++ b/internal/importer/importer_test.go @@ -0,0 +1,172 @@ +// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +package importer_test + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/ClusterCockpit/cc-backend/internal/config" + "github.com/ClusterCockpit/cc-backend/internal/importer" + "github.com/ClusterCockpit/cc-backend/internal/repository" + "github.com/ClusterCockpit/cc-backend/pkg/archive" + "github.com/ClusterCockpit/cc-backend/pkg/log" +) + +func copyFile(s string, d string) error { + r, err := os.Open(s) + if err != nil { + return err + } + defer r.Close() + w, err := os.Create(d) + if err != nil { + return err + } + defer w.Close() + w.ReadFrom(r) + return nil +} + +func setup(t *testing.T) *repository.JobRepository { + const testconfig = `{ + "addr": "0.0.0.0:8080", + "validate": false, + "archive": { + "kind": "file", + "path": "./var/job-archive" + }, + "clusters": [ + { + "name": "testcluster", + "metricDataRepository": {"kind": "test", "url": "bla:8081"}, + "filterRanges": { + "numNodes": { "from": 1, "to": 64 }, + "duration": { "from": 0, "to": 86400 }, + "startTime": { "from": "2022-01-01T00:00:00Z", "to": null } + } + }, + { + "name": "fritz", + "metricDataRepository": {"kind": "test", "url": "bla:8081"}, + "filterRanges": { + "numNodes": { "from": 1, "to": 944 }, + "duration": { "from": 0, "to": 86400 }, + "startTime": { "from": "2022-01-01T00:00:00Z", "to": null } + } + }, + { + "name": "taurus", + "metricDataRepository": {"kind": "test", "url": "bla:8081"}, + "filterRanges": { + "numNodes": { "from": 1, "to": 4000 }, + "duration": { "from": 0, "to": 604800 }, + "startTime": { "from": "2010-01-01T00:00:00Z", "to": null } + } + } + ]}` + + log.Init("info", true) + tmpdir := t.TempDir() + + jobarchive := filepath.Join(tmpdir, "job-archive") + if err := os.Mkdir(jobarchive, 0777); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(jobarchive, "version.txt"), []byte(fmt.Sprintf("%d", 1)), 0666); err != nil { + t.Fatal(err) + } + fritzArchive := filepath.Join(tmpdir, "job-archive", "fritz") + if err := os.Mkdir(fritzArchive, 0777); err != nil { + t.Fatal(err) + } + if err := copyFile(filepath.Join("testdata", "cluster-fritz.json"), + filepath.Join(fritzArchive, "cluster.json")); err != nil { + t.Fatal(err) + } + + dbfilepath := filepath.Join(tmpdir, "test.db") + err := repository.MigrateDB("sqlite3", dbfilepath) + if err != nil { + t.Fatal(err) + } + + cfgFilePath := filepath.Join(tmpdir, "config.json") + if err := os.WriteFile(cfgFilePath, []byte(testconfig), 0666); err != nil { + t.Fatal(err) + } + + config.Init(cfgFilePath) + archiveCfg := fmt.Sprintf("{\"kind\": \"file\",\"path\": \"%s\"}", jobarchive) + + if err := archive.Init(json.RawMessage(archiveCfg), config.Keys.DisableArchive); err != nil { + t.Fatal(err) + } + + repository.Connect("sqlite3", dbfilepath) + return repository.GetJobRepository() +} + +type Result struct { + JobId int64 + Cluster string + StartTime int64 + Duration int32 +} + +func readResult(t *testing.T, testname string) Result { + var r Result + + content, err := os.ReadFile(filepath.Join("testdata", + fmt.Sprintf("%s-golden.json", testname))) + if err != nil { + t.Fatal("Error when opening file: ", err) + } + + err = json.Unmarshal(content, &r) + if err != nil { + t.Fatal("Error during Unmarshal(): ", err) + } + + return r +} + +func TestHandleImportFlag(t *testing.T) { + r := setup(t) + + tests, err := filepath.Glob(filepath.Join("testdata", "*.input")) + if err != nil { + t.Fatal(err) + } + + for _, path := range tests { + _, filename := filepath.Split(path) + str := strings.Split(strings.TrimSuffix(filename, ".input"), "-") + testname := str[1] + + t.Run(testname, func(t *testing.T) { + s := fmt.Sprintf("%s:%s", filepath.Join("testdata", + fmt.Sprintf("meta-%s.input", testname)), + filepath.Join("testdata", fmt.Sprintf("data-%s.json", testname))) + err := importer.HandleImportFlag(s) + if err != nil { + t.Fatal(err) + } + + result := readResult(t, testname) + job, err := r.Find(&result.JobId, &result.Cluster, &result.StartTime) + if err != nil { + t.Fatal(err) + } + if job.Duration != result.Duration { + t.Errorf("wrong duration for job\ngot: %d \nwant: %d", job.Duration, result.Duration) + } + }) + } +} diff --git a/internal/importer/initDB.go b/internal/importer/initDB.go new file mode 100644 index 0000000..cce9ebf --- /dev/null +++ b/internal/importer/initDB.go @@ -0,0 +1,197 @@ +// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +package importer + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/ClusterCockpit/cc-backend/internal/repository" + "github.com/ClusterCockpit/cc-backend/pkg/archive" + "github.com/ClusterCockpit/cc-backend/pkg/log" + "github.com/ClusterCockpit/cc-backend/pkg/schema" +) + +// Delete the tables "job", "tag" and "jobtag" from the database and +// repopulate them using the jobs found in `archive`. +func InitDB() error { + r := repository.GetJobRepository() + if err := r.Flush(); err != nil { + log.Errorf("repository initDB(): %v", err) + return err + } + starttime := time.Now() + log.Print("Building job table...") + + t, err := r.TransactionInit() + if err != nil { + log.Warn("Error while initializing SQL transactions") + return err + } + tags := make(map[string]int64) + + // Not using log.Print because we want the line to end with `\r` and + // this function is only ever called when a special command line flag + // is passed anyways. + fmt.Printf("%d jobs inserted...\r", 0) + + ar := archive.GetHandle() + i := 0 + errorOccured := 0 + + for jobContainer := range ar.Iter(false) { + + jobMeta := jobContainer.Meta + + // Bundle 100 inserts into one transaction for better performance + if i%100 == 0 { + r.TransactionCommit(t) + fmt.Printf("%d jobs inserted...\r", i) + } + + jobMeta.MonitoringStatus = schema.MonitoringStatusArchivingSuccessful + job := schema.Job{ + BaseJob: jobMeta.BaseJob, + StartTime: time.Unix(jobMeta.StartTime, 0), + StartTimeUnix: jobMeta.StartTime, + } + + // TODO: Other metrics... + job.FlopsAnyAvg = loadJobStat(jobMeta, "flops_any") + job.MemBwAvg = loadJobStat(jobMeta, "mem_bw") + job.NetBwAvg = loadJobStat(jobMeta, "net_bw") + job.FileBwAvg = loadJobStat(jobMeta, "file_bw") + + job.RawResources, err = json.Marshal(job.Resources) + if err != nil { + log.Errorf("repository initDB(): %v", err) + errorOccured++ + continue + } + + job.RawMetaData, err = json.Marshal(job.MetaData) + if err != nil { + log.Errorf("repository initDB(): %v", err) + errorOccured++ + continue + } + + if err := SanityChecks(&job.BaseJob); err != nil { + log.Errorf("repository initDB(): %v", err) + errorOccured++ + continue + } + + id, err := r.TransactionAdd(t, job) + if err != nil { + log.Errorf("repository initDB(): %v", err) + errorOccured++ + continue + } + + for _, tag := range job.Tags { + tagstr := tag.Name + ":" + tag.Type + tagId, ok := tags[tagstr] + if !ok { + tagId, err = r.TransactionAddTag(t, tag) + if err != nil { + log.Errorf("Error adding tag: %v", err) + errorOccured++ + continue + } + tags[tagstr] = tagId + } + + r.TransactionSetTag(t, id, tagId) + } + + if err == nil { + i += 1 + } + } + + if errorOccured > 0 { + log.Warnf("Error in import of %d jobs!", errorOccured) + } + + r.TransactionEnd(t) + log.Printf("A total of %d jobs have been registered in %.3f seconds.\n", i, time.Since(starttime).Seconds()) + return nil +} + +// This function also sets the subcluster if necessary! +func SanityChecks(job *schema.BaseJob) error { + if c := archive.GetCluster(job.Cluster); c == nil { + return fmt.Errorf("no such cluster: %v", job.Cluster) + } + if err := archive.AssignSubCluster(job); err != nil { + log.Warn("Error while assigning subcluster to job") + return err + } + if !job.State.Valid() { + return fmt.Errorf("not a valid job state: %v", job.State) + } + if len(job.Resources) == 0 || len(job.User) == 0 { + return fmt.Errorf("'resources' and 'user' should not be empty") + } + if job.NumAcc < 0 || job.NumHWThreads < 0 || job.NumNodes < 1 { + return fmt.Errorf("'numNodes', 'numAcc' or 'numHWThreads' invalid") + } + if len(job.Resources) != int(job.NumNodes) { + return fmt.Errorf("len(resources) does not equal numNodes (%d vs %d)", len(job.Resources), job.NumNodes) + } + + return nil +} + +func loadJobStat(job *schema.JobMeta, metric string) float64 { + if stats, ok := job.Statistics[metric]; ok { + return stats.Avg + } + + return 0.0 +} + +func checkJobData(d *schema.JobData) error { + for _, scopes := range *d { + // var newUnit schema.Unit + // TODO Add node scope if missing + for _, metric := range scopes { + if strings.Contains(metric.Unit.Base, "B/s") || + strings.Contains(metric.Unit.Base, "F/s") || + strings.Contains(metric.Unit.Base, "B") { + + // get overall avg + sum := 0.0 + for _, s := range metric.Series { + sum += s.Statistics.Avg + } + + avg := sum / float64(len(metric.Series)) + f, p := Normalize(avg, metric.Unit.Prefix) + + if p != metric.Unit.Prefix { + + fmt.Printf("Convert %e", f) + // for _, s := range metric.Series { + // fp := schema.ConvertFloatToFloat64(s.Data) + // + // for i := 0; i < len(fp); i++ { + // fp[i] *= f + // fp[i] = math.Ceil(fp[i]) + // } + // + // s.Data = schema.GetFloat64ToFloat(fp) + // } + + metric.Unit.Prefix = p + } + } + } + } + return nil +} diff --git a/internal/importer/normalize.go b/internal/importer/normalize.go new file mode 100644 index 0000000..a2efac3 --- /dev/null +++ b/internal/importer/normalize.go @@ -0,0 +1,58 @@ +// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +package importer + +import ( + "math" + + ccunits "github.com/ClusterCockpit/cc-units" +) + +func getNormalizationFactor(v float64) (float64, int) { + count := 0 + scale := -3 + + if v > 1000.0 { + for v > 1000.0 { + v *= 1e-3 + count++ + } + } else { + for v < 1.0 { + v *= 1e3 + count++ + } + scale = 3 + } + return math.Pow10(count * scale), count * scale +} + +func getExponent(p float64) int { + count := 0 + + for p > 1.0 { + p = p / 1000.0 + count++ + } + + return count * 3 +} + +func newPrefixFromFactor(op ccunits.Prefix, e int) ccunits.Prefix { + f := float64(op) + exp := math.Pow10(getExponent(f) - e) + return ccunits.Prefix(exp) +} + +func Normalize(avg float64, p string) (float64, string) { + f, e := getNormalizationFactor(avg) + + if e != 0 { + np := newPrefixFromFactor(ccunits.NewPrefix(p), e) + return f, np.Prefix() + } + + return f, p +} diff --git a/internal/importer/normalize_test.go b/internal/importer/normalize_test.go new file mode 100644 index 0000000..544000a --- /dev/null +++ b/internal/importer/normalize_test.go @@ -0,0 +1,64 @@ +// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +package importer + +import ( + "fmt" + "testing" + + ccunits "github.com/ClusterCockpit/cc-units" +) + +func TestNormalizeFactor(t *testing.T) { + // var us string + s := []float64{2890031237, 23998994567, 389734042344, 390349424345} + // r := []float64{3, 24, 390, 391} + + total := 0.0 + for _, number := range s { + total += number + } + avg := total / float64(len(s)) + + fmt.Printf("AVG: %e\n", avg) + f, e := getNormalizationFactor(avg) + + fmt.Printf("Factor %e Count %d\n", f, e) + + np := ccunits.NewPrefix("") + + fmt.Printf("Prefix %e Short %s\n", float64(np), np.Prefix()) + + p := newPrefixFromFactor(np, e) + + if p.Prefix() != "G" { + t.Errorf("Failed Prefix or unit: Want G, Got %s", p.Prefix()) + } +} + +func TestNormalizeKeep(t *testing.T) { + s := []float64{3.0, 24.0, 390.0, 391.0} + + total := 0.0 + for _, number := range s { + total += number + } + avg := total / float64(len(s)) + + fmt.Printf("AVG: %e\n", avg) + f, e := getNormalizationFactor(avg) + + fmt.Printf("Factor %e Count %d\n", f, e) + + np := ccunits.NewPrefix("G") + + fmt.Printf("Prefix %e Short %s\n", float64(np), np.Prefix()) + + p := newPrefixFromFactor(np, e) + + if p.Prefix() != "G" { + t.Errorf("Failed Prefix or unit: Want G, Got %s", p.Prefix()) + } +} diff --git a/internal/importer/testdata/cluster-fritz.json b/internal/importer/testdata/cluster-fritz.json new file mode 100644 index 0000000..23a8343 --- /dev/null +++ b/internal/importer/testdata/cluster-fritz.json @@ -0,0 +1,746 @@ +{ + "name": "fritz", + "metricConfig": [ + { + "name": "cpu_load", + "unit": { + "base": "" + }, + "scope": "node", + "aggregation": "avg", + "timestep": 60, + "peak": 72, + "normal": 72, + "caution": 36, + "alert": 20 + }, + { + "name": "cpu_user", + "unit": { + "base": "" + }, + "scope": "hwthread", + "aggregation": "avg", + "timestep": 60, + "peak": 100, + "normal": 50, + "caution": 20, + "alert": 10 + }, + { + "name": "mem_used", + "unit": { + "base": "B", + "prefix": "G" + }, + "scope": "node", + "aggregation": "sum", + "timestep": 60, + "peak": 256, + "normal": 128, + "caution": 200, + "alert": 240 + }, + { + "name": "flops_any", + "unit": { + "base": "F/s", + "prefix": "G" + }, + "scope": "hwthread", + "aggregation": "sum", + "timestep": 60, + "peak": 5600, + "normal": 1000, + "caution": 200, + "alert": 50 + }, + { + "name": "flops_sp", + "unit": { + "base": "F/s", + "prefix": "G" + }, + "scope": "hwthread", + "aggregation": "sum", + "timestep": 60, + "peak": 5600, + "normal": 1000, + "caution": 200, + "alert": 50 + }, + { + "name": "flops_dp", + "unit": { + "base": "F/s", + "prefix": "G" + }, + "scope": "hwthread", + "aggregation": "sum", + "timestep": 60, + "peak": 2300, + "normal": 500, + "caution": 100, + "alert": 50 + }, + { + "name": "mem_bw", + "unit": { + "base": "B/s", + "prefix": "G" + }, + "scope": "socket", + "aggregation": "sum", + "timestep": 60, + "peak": 350, + "normal": 100, + "caution": 50, + "alert": 10 + }, + { + "name": "clock", + "unit": { + "base": "Hz", + "prefix": "M" + }, + "scope": "hwthread", + "aggregation": "avg", + "timestep": 60, + "peak": 3000, + "normal": 2400, + "caution": 1800, + "alert": 1200 + }, + { + "name": "cpu_power", + "unit": { + "base": "W" + }, + "scope": "socket", + "aggregation": "sum", + "timestep": 60, + "peak": 500, + "normal": 250, + "caution": 100, + "alert": 50 + }, + { + "name": "mem_power", + "unit": { + "base": "W" + }, + "scope": "socket", + "aggregation": "sum", + "timestep": 60, + "peak": 100, + "normal": 50, + "caution": 20, + "alert": 10 + }, + { + "name": "ipc", + "unit": { + "base": "IPC" + }, + "scope": "hwthread", + "aggregation": "avg", + "timestep": 60, + "peak": 4, + "normal": 2, + "caution": 1, + "alert": 0.5 + }, + { + "name": "vectorization_ratio", + "unit": { + "base": "" + }, + "scope": "hwthread", + "aggregation": "avg", + "timestep": 60, + "peak": 100, + "normal": 60, + "caution": 40, + "alert": 10 + }, + { + "name": "ib_recv", + "unit": { + "base": "B/s" + }, + "scope": "node", + "aggregation": "sum", + "timestep": 60, + "peak": 1250000, + "normal": 6000000, + "caution": 200, + "alert": 1 + }, + { + "name": "ib_xmit", + "unit": { + "base": "B/s" + }, + "scope": "node", + "aggregation": "sum", + "timestep": 60, + "peak": 1250000, + "normal": 6000000, + "caution": 200, + "alert": 1 + }, + { + "name": "ib_recv_pkts", + "unit": { + "base": "" + }, + "scope": "node", + "aggregation": "sum", + "timestep": 60, + "peak": 6, + "normal": 4, + "caution": 2, + "alert": 1 + }, + { + "name": "ib_xmit_pkts", + "unit": { + "base": "" + }, + "scope": "node", + "aggregation": "sum", + "timestep": 60, + "peak": 6, + "normal": 4, + "caution": 2, + "alert": 1 + }, + { + "name": "nfs4_read", + "unit": { + "base": "B/s", + "prefix": "M" + }, + "scope": "node", + "aggregation": "sum", + "timestep": 60, + "peak": 6, + "normal": 4, + "caution": 2, + "alert": 1 + }, + { + "name": "nfs4_write", + "unit": { + "base": "B/s", + "prefix": "M" + }, + "scope": "node", + "aggregation": "sum", + "timestep": 60, + "peak": 6, + "normal": 4, + "caution": 2, + "alert": 1 + }, + { + "name": "nfs4_total", + "unit": { + "base": "B/s", + "prefix": "M" + }, + "scope": "node", + "aggregation": "sum", + "timestep": 60, + "peak": 6, + "normal": 4, + "caution": 2, + "alert": 1 + } + ], + "subClusters": [ + { + "name": "main", + "nodes": "f01[01-88],f02[01-88],f03[01-88],f03[01-88],f04[01-88],f05[01-88],f06[01-88],f07[01-88],f08[01-88],f09[01-88],f10[01-88],f11[01-56],f12[01-56]", + "processorType": "Intel Icelake", + "socketsPerNode": 2, + "coresPerSocket": 36, + "threadsPerCore": 1, + "flopRateScalar": { + "unit": { + "base": "F/s", + "prefix": "G" + }, + "value": 432 + }, + "flopRateSimd": { + "unit": { + "base": "F/s", + "prefix": "G" + }, + "value": 9216 + }, + "memoryBandwidth": { + "unit": { + "base": "B/s", + "prefix": "G" + }, + "value": 350 + }, + "topology": { + "node": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71 + ], + "socket": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35 + ], + [ + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71 + ] + ], + "memoryDomain": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17 + ], + [ + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35 + ], + [ + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53 + ], + [ + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71 + ] + ], + "core": [ + [ + 0 + ], + [ + 1 + ], + [ + 2 + ], + [ + 3 + ], + [ + 4 + ], + [ + 5 + ], + [ + 6 + ], + [ + 7 + ], + [ + 8 + ], + [ + 9 + ], + [ + 10 + ], + [ + 11 + ], + [ + 12 + ], + [ + 13 + ], + [ + 14 + ], + [ + 15 + ], + [ + 16 + ], + [ + 17 + ], + [ + 18 + ], + [ + 19 + ], + [ + 20 + ], + [ + 21 + ], + [ + 22 + ], + [ + 23 + ], + [ + 24 + ], + [ + 25 + ], + [ + 26 + ], + [ + 27 + ], + [ + 28 + ], + [ + 29 + ], + [ + 30 + ], + [ + 31 + ], + [ + 32 + ], + [ + 33 + ], + [ + 34 + ], + [ + 35 + ], + [ + 36 + ], + [ + 37 + ], + [ + 38 + ], + [ + 39 + ], + [ + 40 + ], + [ + 41 + ], + [ + 42 + ], + [ + 43 + ], + [ + 44 + ], + [ + 45 + ], + [ + 46 + ], + [ + 47 + ], + [ + 48 + ], + [ + 49 + ], + [ + 50 + ], + [ + 51 + ], + [ + 52 + ], + [ + 53 + ], + [ + 54 + ], + [ + 55 + ], + [ + 56 + ], + [ + 57 + ], + [ + 58 + ], + [ + 59 + ], + [ + 60 + ], + [ + 61 + ], + [ + 62 + ], + [ + 63 + ], + [ + 64 + ], + [ + 65 + ], + [ + 66 + ], + [ + 67 + ], + [ + 68 + ], + [ + 69 + ], + [ + 70 + ], + [ + 71 + ] + ] + } + } + ] +} diff --git a/internal/importer/testdata/data-fritzError.json b/internal/importer/testdata/data-fritzError.json new file mode 100644 index 0000000..b8847d6 --- /dev/null +++ b/internal/importer/testdata/data-fritzError.json @@ -0,0 +1 @@ +{"clock":{"core":{"unit":{"base":"Hz","prefix":"M"},"timestep":60,"series":[{"hostname":"f0720","id":"38","statistics":{"min":799.20,"avg":2340.09,"max":2748.63},"data":[2748.63,799.20,2722.40,2716.55,2713.69]},{"hostname":"f0720","id":"45","statistics":{"min":799.69,"avg":2330.70,"max":2723.98},"data":[2714.38,799.69,2723.98,2710.05,2705.39]},{"hostname":"f0720","id":"22","statistics":{"min":804.52,"avg":2340.53,"max":2754.76},"data":[2754.76,804.52,2724.64,2710.93,2707.83]},{"hostname":"f0720","id":"44","statistics":{"min":793.83,"avg":2336.67,"max":2734.08},"data":[2734.08,793.83,2723.55,2717.44,2714.44]},{"hostname":"f0720","id":"47","statistics":{"min":800.14,"avg":2336.52,"max":2726.99},"data":[2726.99,800.15,2724.53,2716.90,2714.04]},{"hostname":"f0720","id":"57","statistics":{"min":800.15,"avg":2336.06,"max":2737.29},"data":[2737.29,800.15,2724.07,2710.76,2708.03]},{"hostname":"f0720","id":"33","statistics":{"min":795.71,"avg":2341.69,"max":2754.59},"data":[2754.59,795.71,2727.41,2717.30,2713.44]},{"hostname":"f0720","id":"23","statistics":{"min":789.42,"avg":2339.54,"max":2749.11},"data":[2749.11,789.42,2727.11,2717.67,2714.39]},{"hostname":"f0720","id":"25","statistics":{"min":790.76,"avg":2338.34,"max":2753.97},"data":[2753.97,790.76,2726.46,2712.44,2708.05]},{"hostname":"f0720","id":"4","statistics":{"min":799.97,"avg":2334.92,"max":2736.79},"data":[2736.79,799.97,2723.72,2708.67,2705.45]},{"hostname":"f0720","id":"41","statistics":{"min":796.66,"avg":2335.72,"max":2728.99},"data":[2728.99,796.66,2724.33,2716.17,2712.43]},{"hostname":"f0720","id":"49","statistics":{"min":798.45,"avg":2331.55,"max":2724.00},"data":[2717.34,798.45,2724.00,2710.38,2707.60]},{"hostname":"f0720","id":"69","statistics":{"min":788.80,"avg":2335.50,"max":2734.92},"data":[2734.92,788.80,2727.79,2715.84,2710.16]},{"hostname":"f0720","id":"39","statistics":{"min":788.86,"avg":2306.22,"max":2695.45},"data":[2685.16,788.86,2695.46,2682.08,2679.57]},{"hostname":"f0720","id":"5","statistics":{"min":803.70,"avg":2341.47,"max":2748.27},"data":[2748.27,803.70,2725.55,2716.94,2712.89]},{"hostname":"f0720","id":"14","statistics":{"min":796.79,"avg":2337.13,"max":2737.92},"data":[2737.92,796.79,2725.38,2714.77,2710.81]},{"hostname":"f0720","id":"18","statistics":{"min":805.57,"avg":2337.53,"max":2738.85},"data":[2738.85,805.57,2724.57,2710.73,2707.93]},{"hostname":"f0720","id":"21","statistics":{"min":793.95,"avg":2335.22,"max":2737.20},"data":[2737.20,793.95,2725.16,2712.23,2707.55]},{"hostname":"f0720","id":"28","statistics":{"min":794.36,"avg":2336.76,"max":2739.06},"data":[2739.06,794.35,2727.69,2712.66,2710.04]},{"hostname":"f0720","id":"54","statistics":{"min":799.11,"avg":2335.85,"max":2738.74},"data":[2738.74,799.11,2723.30,2710.26,2707.81]},{"hostname":"f0720","id":"1","statistics":{"min":803.76,"avg":2334.31,"max":2742.36},"data":[2742.36,803.76,2718.99,2705.14,2701.28]},{"hostname":"f0720","id":"12","statistics":{"min":803.93,"avg":2335.87,"max":2733.22},"data":[2733.22,803.93,2724.47,2710.55,2707.18]},{"hostname":"f0720","id":"43","statistics":{"min":796.45,"avg":2333.10,"max":2730.24},"data":[2730.24,796.45,2722.94,2710.02,2705.84]},{"hostname":"f0720","id":"55","statistics":{"min":801.45,"avg":2336.81,"max":2739.03},"data":[2739.03,801.45,2724.08,2711.30,2708.19]},{"hostname":"f0720","id":"58","statistics":{"min":808.23,"avg":2336.77,"max":2732.86},"data":[2732.85,808.23,2723.59,2710.87,2708.28]},{"hostname":"f0720","id":"65","statistics":{"min":805.45,"avg":2340.68,"max":2732.15},"data":[2732.15,805.45,2727.96,2720.11,2717.71]},{"hostname":"f0720","id":"67","statistics":{"min":800.23,"avg":2336.62,"max":2730.16},"data":[2730.16,800.23,2726.00,2715.75,2710.95]},{"hostname":"f0720","id":"0","statistics":{"min":863.52,"avg":2285.04,"max":2679.67},"data":[2679.67,863.52,2630.73,2624.93,2626.32]},{"hostname":"f0720","id":"11","statistics":{"min":798.30,"avg":2337.16,"max":2732.36},"data":[2732.36,798.30,2725.50,2717.15,2712.48]},{"hostname":"f0720","id":"17","statistics":{"min":793.25,"avg":2335.67,"max":2730.72},"data":[2730.72,793.25,2724.39,2716.56,2713.43]},{"hostname":"f0720","id":"24","statistics":{"min":800.96,"avg":2340.06,"max":2754.81},"data":[2754.81,800.96,2724.72,2711.53,2708.27]},{"hostname":"f0720","id":"26","statistics":{"min":804.72,"avg":2342.17,"max":2748.87},"data":[2748.87,804.72,2725.35,2718.06,2713.85]},{"hostname":"f0720","id":"27","statistics":{"min":794.13,"avg":2335.96,"max":2739.86},"data":[2739.86,794.13,2724.18,2712.88,2708.78]},{"hostname":"f0720","id":"31","statistics":{"min":796.76,"avg":2341.37,"max":2754.09},"data":[2754.09,796.76,2727.82,2716.15,2712.05]},{"hostname":"f0720","id":"51","statistics":{"min":801.01,"avg":2334.13,"max":2727.16},"data":[2727.17,801.01,2724.06,2710.79,2707.61]},{"hostname":"f0720","id":"8","statistics":{"min":800.08,"avg":2338.38,"max":2734.09},"data":[2734.08,800.08,2727.38,2716.96,2713.41]},{"hostname":"f0720","id":"68","statistics":{"min":813.39,"avg":2346.74,"max":2749.81},"data":[2749.81,813.39,2729.49,2723.12,2717.88]},{"hostname":"f0720","id":"10","statistics":{"min":800.43,"avg":2333.67,"max":2728.79},"data":[2728.79,800.43,2723.10,2709.51,2706.54]},{"hostname":"f0720","id":"19","statistics":{"min":799.99,"avg":2340.08,"max":2756.32},"data":[2756.32,799.99,2724.95,2711.64,2707.50]},{"hostname":"f0720","id":"20","statistics":{"min":801.47,"avg":2338.55,"max":2732.72},"data":[2732.72,801.47,2727.46,2716.42,2714.68]},{"hostname":"f0720","id":"2","statistics":{"min":799.34,"avg":2335.61,"max":2727.37},"data":[2727.37,799.34,2725.40,2713.59,2712.33]},{"hostname":"f0720","id":"70","statistics":{"min":811.56,"avg":2344.07,"max":2752.15},"data":[2752.15,811.56,2728.53,2716.70,2711.40]},{"hostname":"f0720","id":"61","statistics":{"min":789.94,"avg":2333.18,"max":2733.32},"data":[2733.32,789.94,2724.53,2710.68,2707.42]},{"hostname":"f0720","id":"29","statistics":{"min":806.70,"avg":2342.27,"max":2740.66},"data":[2740.66,806.70,2727.01,2720.79,2716.17]},{"hostname":"f0720","id":"62","statistics":{"min":799.36,"avg":2340.48,"max":2748.63},"data":[2748.63,799.36,2722.66,2716.66,2715.09]},{"hostname":"f0720","id":"16","statistics":{"min":800.17,"avg":2335.00,"max":2729.38},"data":[2729.38,800.17,2726.58,2711.04,2707.85]},{"hostname":"f0720","id":"9","statistics":{"min":800.04,"avg":2329.32,"max":2721.82},"data":[2712.87,800.04,2721.82,2707.45,2704.44]},{"hostname":"f0720","id":"13","statistics":{"min":801.10,"avg":2336.02,"max":2736.38},"data":[2736.38,801.10,2724.50,2710.73,2707.39]},{"hostname":"f0720","id":"30","statistics":{"min":787.64,"avg":2339.48,"max":2754.86},"data":[2754.86,787.64,2727.81,2714.86,2712.21]},{"hostname":"f0720","id":"32","statistics":{"min":790.66,"avg":2338.73,"max":2730.93},"data":[2730.93,790.66,2729.92,2722.86,2719.27]},{"hostname":"f0720","id":"34","statistics":{"min":792.10,"avg":2343.78,"max":2755.01},"data":[2755.01,792.10,2734.16,2723.82,2713.83]},{"hostname":"f0720","id":"40","statistics":{"min":798.41,"avg":2336.44,"max":2747.66},"data":[2747.66,798.41,2721.69,2709.22,2705.21]},{"hostname":"f0720","id":"42","statistics":{"min":803.59,"avg":2322.74,"max":2715.31},"data":[2715.31,803.59,2706.89,2694.84,2693.08]},{"hostname":"f0720","id":"6","statistics":{"min":799.88,"avg":2319.54,"max":2709.17},"data":[2701.62,799.88,2709.17,2694.70,2692.32]},{"hostname":"f0720","id":"63","statistics":{"min":810.78,"avg":2337.41,"max":2732.38},"data":[2732.38,810.78,2725.14,2710.95,2707.82]},{"hostname":"f0720","id":"46","statistics":{"min":799.54,"avg":2333.76,"max":2730.67},"data":[2730.67,799.54,2721.81,2709.88,2706.88]},{"hostname":"f0720","id":"15","statistics":{"min":798.73,"avg":2333.24,"max":2724.58},"data":[2724.30,798.73,2724.58,2710.94,2707.66]},{"hostname":"f0720","id":"35","statistics":{"min":808.86,"avg":2345.06,"max":2732.46},"data":[2732.46,808.86,2730.81,2727.86,2725.30]},{"hostname":"f0720","id":"48","statistics":{"min":788.74,"avg":2328.09,"max":2723.19},"data":[2711.70,788.73,2723.19,2709.21,2707.61]},{"hostname":"f0720","id":"56","statistics":{"min":791.56,"avg":2338.93,"max":2750.20},"data":[2750.21,791.56,2723.64,2714.87,2714.40]},{"hostname":"f0720","id":"7","statistics":{"min":800.34,"avg":2321.34,"max":2713.23},"data":[2702.26,800.34,2713.23,2696.90,2693.95]},{"hostname":"f0720","id":"50","statistics":{"min":797.82,"avg":2339.07,"max":2747.67},"data":[2747.67,797.82,2722.81,2715.25,2711.80]},{"hostname":"f0720","id":"53","statistics":{"min":807.10,"avg":2337.70,"max":2730.33},"data":[2730.33,807.10,2723.69,2714.65,2712.75]},{"hostname":"f0720","id":"36","statistics":{"min":800.04,"avg":2334.72,"max":2740.52},"data":[2740.52,800.04,2721.72,2707.69,2703.61]},{"hostname":"f0720","id":"59","statistics":{"min":804.25,"avg":2338.60,"max":2729.02},"data":[2729.02,804.24,2725.92,2718.45,2715.36]},{"hostname":"f0720","id":"60","statistics":{"min":799.84,"avg":2339.57,"max":2756.62},"data":[2756.62,799.84,2722.77,2710.53,2708.11]},{"hostname":"f0720","id":"66","statistics":{"min":803.96,"avg":2342.14,"max":2753.19},"data":[2753.19,803.96,2727.72,2714.43,2711.42]},{"hostname":"f0720","id":"71","statistics":{"min":799.34,"avg":2341.61,"max":2733.53},"data":[2733.53,799.34,2729.61,2725.67,2719.89]},{"hostname":"f0720","id":"52","statistics":{"min":799.18,"avg":2337.76,"max":2748.03},"data":[2748.03,799.18,2723.53,2710.51,2707.57]},{"hostname":"f0720","id":"37","statistics":{"min":802.83,"avg":2332.14,"max":2723.60},"data":[2723.60,802.83,2721.87,2708.47,2703.91]},{"hostname":"f0720","id":"64","statistics":{"min":802.43,"avg":2340.87,"max":2753.80},"data":[2753.80,802.43,2725.26,2712.31,2710.57]},{"hostname":"f0720","id":"3","statistics":{"min":807.12,"avg":2308.45,"max":2701.63},"data":[2701.63,807.12,2687.20,2674.66,2671.64]}]},"node":{"unit":{"base":"Hz","prefix":"M"},"timestep":60,"series":[{"hostname":"f0720","id":"0","statistics":{"min":800.42,"avg":2335.25,"max":2734.92},"data":[2734.92,800.42,2722.27,2710.99,2707.67]}]}},"cpu_load":{"node":{"unit":{"base":""},"timestep":60,"series":[{"hostname":"f0720","statistics":{"min":34.46,"avg":52.72,"max":71.91},"data":[71.91,34.46,37.67,59.81,59.75]}]}},"cpu_power":{"node":{"unit":{"base":"W"},"timestep":60,"series":[{"hostname":"f0720","id":"0","statistics":{"min":93.93,"avg":407.77,"max":497.64},"data":[486.14,93.93,482.76,478.37,497.64]}]},"socket":{"unit":{"base":"W"},"timestep":60,"series":[{"hostname":"f0720","id":"0","statistics":{"min":48.24,"avg":205.10,"max":251.37},"data":[243.26,48.24,242.91,239.73,251.37]},{"hostname":"f0720","id":"0","statistics":{"min":45.69,"avg":202.66,"max":246.27},"data":[242.87,45.69,239.85,238.64,246.27]}]}},"cpu_user":{"core":{"unit":{"base":""},"timestep":60,"series":[{"hostname":"f0720","id":"1","statistics":{"min":19.70,"avg":63.80,"max":96.68},"data":[96.68,19.70,38.28,82.15,82.17]},{"hostname":"f0720","id":"20","statistics":{"min":19.83,"avg":63.63,"max":96.68},"data":[96.68,19.83,38.06,81.68,81.88]},{"hostname":"f0720","id":"35","statistics":{"min":19.60,"avg":63.49,"max":96.53},"data":[96.53,19.60,38.06,81.57,81.67]},{"hostname":"f0720","id":"8","statistics":{"min":19.80,"avg":63.69,"max":96.53},"data":[96.53,19.80,38.13,81.95,82.05]},{"hostname":"f0720","id":"9","statistics":{"min":19.80,"avg":63.72,"max":96.73},"data":[96.73,19.80,38.12,81.93,82.03]},{"hostname":"f0720","id":"12","statistics":{"min":19.85,"avg":63.78,"max":96.67},"data":[96.67,19.85,38.18,82.00,82.22]},{"hostname":"f0720","id":"33","statistics":{"min":19.83,"avg":63.53,"max":96.43},"data":[96.43,19.83,37.87,81.62,81.88]},{"hostname":"f0720","id":"54","statistics":{"min":19.92,"avg":63.66,"max":96.60},"data":[96.60,19.92,38.08,81.77,81.93]},{"hostname":"f0720","id":"67","statistics":{"min":19.60,"avg":63.60,"max":96.65},"data":[96.65,19.60,37.98,81.80,81.95]},{"hostname":"f0720","id":"69","statistics":{"min":19.82,"avg":63.60,"max":96.68},"data":[96.68,19.82,37.98,81.68,81.82]},{"hostname":"f0720","id":"6","statistics":{"min":19.87,"avg":63.71,"max":96.68},"data":[96.68,19.87,38.08,81.92,82.02]},{"hostname":"f0720","id":"2","statistics":{"min":20.40,"avg":63.97,"max":96.88},"data":[96.88,20.40,38.10,82.17,82.28]},{"hostname":"f0720","id":"15","statistics":{"min":19.68,"avg":63.53,"max":96.55},"data":[96.55,19.68,37.87,81.63,81.90]},{"hostname":"f0720","id":"26","statistics":{"min":19.85,"avg":63.61,"max":96.90},"data":[96.90,19.85,37.93,81.68,81.67]},{"hostname":"f0720","id":"50","statistics":{"min":19.70,"avg":63.62,"max":96.53},"data":[96.53,19.70,38.33,81.78,81.75]},{"hostname":"f0720","id":"53","statistics":{"min":20.22,"avg":63.94,"max":96.72},"data":[96.72,20.22,38.13,81.95,82.68]},{"hostname":"f0720","id":"55","statistics":{"min":20.03,"avg":63.76,"max":96.75},"data":[96.75,20.03,38.00,82.12,81.90]},{"hostname":"f0720","id":"56","statistics":{"min":20.33,"avg":63.69,"max":96.47},"data":[96.47,20.33,38.06,81.67,81.93]},{"hostname":"f0720","id":"68","statistics":{"min":19.78,"avg":63.58,"max":96.62},"data":[96.62,19.78,38.01,81.67,81.83]},{"hostname":"f0720","id":"25","statistics":{"min":19.80,"avg":63.43,"max":96.53},"data":[96.53,19.80,37.65,81.30,81.88]},{"hostname":"f0720","id":"14","statistics":{"min":19.83,"avg":63.74,"max":96.65},"data":[96.65,19.83,38.06,81.95,82.18]},{"hostname":"f0720","id":"16","statistics":{"min":19.65,"avg":63.59,"max":96.58},"data":[96.58,19.65,37.97,81.65,82.10]},{"hostname":"f0720","id":"22","statistics":{"min":19.78,"avg":63.64,"max":96.68},"data":[96.68,19.78,37.92,82.00,81.80]},{"hostname":"f0720","id":"47","statistics":{"min":19.97,"avg":63.93,"max":96.60},"data":[96.60,19.97,38.22,82.72,82.15]},{"hostname":"f0720","id":"57","statistics":{"min":19.98,"avg":63.71,"max":96.63},"data":[96.63,19.98,38.10,81.90,81.95]},{"hostname":"f0720","id":"58","statistics":{"min":19.78,"avg":63.61,"max":96.60},"data":[96.60,19.78,38.20,81.63,81.85]},{"hostname":"f0720","id":"5","statistics":{"min":20.40,"avg":63.87,"max":96.48},"data":[96.48,20.40,38.15,82.08,82.23]},{"hostname":"f0720","id":"30","statistics":{"min":19.78,"avg":63.58,"max":96.65},"data":[96.65,19.78,38.00,81.62,81.87]},{"hostname":"f0720","id":"37","statistics":{"min":20.20,"avg":63.86,"max":96.65},"data":[96.65,20.20,37.97,82.20,82.27]},{"hostname":"f0720","id":"46","statistics":{"min":19.95,"avg":63.66,"max":96.52},"data":[96.52,19.95,37.83,82.03,81.97]},{"hostname":"f0720","id":"59","statistics":{"min":19.73,"avg":63.65,"max":96.63},"data":[96.63,19.73,38.12,81.82,81.95]},{"hostname":"f0720","id":"60","statistics":{"min":19.98,"avg":63.67,"max":96.75},"data":[96.75,19.98,38.00,81.72,81.90]},{"hostname":"f0720","id":"61","statistics":{"min":19.83,"avg":63.62,"max":96.67},"data":[96.67,19.83,38.08,81.73,81.80]},{"hostname":"f0720","id":"7","statistics":{"min":19.83,"avg":63.73,"max":96.63},"data":[96.63,19.83,38.10,81.95,82.15]},{"hostname":"f0720","id":"21","statistics":{"min":19.80,"avg":63.53,"max":96.50},"data":[96.50,19.80,37.98,81.52,81.87]},{"hostname":"f0720","id":"62","statistics":{"min":19.90,"avg":63.61,"max":96.60},"data":[96.60,19.90,37.95,81.77,81.83]},{"hostname":"f0720","id":"70","statistics":{"min":19.85,"avg":63.55,"max":96.52},"data":[96.52,19.85,37.87,81.63,81.87]},{"hostname":"f0720","id":"13","statistics":{"min":19.60,"avg":63.64,"max":96.67},"data":[96.67,19.60,38.10,81.80,82.03]},{"hostname":"f0720","id":"34","statistics":{"min":19.78,"avg":63.51,"max":96.62},"data":[96.62,19.78,37.80,81.53,81.82]},{"hostname":"f0720","id":"38","statistics":{"min":19.82,"avg":63.79,"max":96.80},"data":[96.80,19.82,37.83,82.28,82.20]},{"hostname":"f0720","id":"44","statistics":{"min":19.82,"avg":63.73,"max":96.67},"data":[96.67,19.82,38.13,82.13,81.90]},{"hostname":"f0720","id":"24","statistics":{"min":19.87,"avg":63.64,"max":96.90},"data":[96.90,19.87,38.12,81.68,81.63]},{"hostname":"f0720","id":"45","statistics":{"min":19.93,"avg":63.96,"max":96.67},"data":[96.67,19.93,38.58,81.83,82.80]},{"hostname":"f0720","id":"51","statistics":{"min":20.28,"avg":63.79,"max":96.60},"data":[96.60,20.28,38.23,81.83,82.00]},{"hostname":"f0720","id":"43","statistics":{"min":20.13,"avg":63.73,"max":96.73},"data":[96.73,20.13,38.05,81.70,82.05]},{"hostname":"f0720","id":"52","statistics":{"min":20.05,"avg":63.76,"max":96.55},"data":[96.55,20.05,38.22,82.15,81.82]},{"hostname":"f0720","id":"4","statistics":{"min":19.83,"avg":63.83,"max":96.58},"data":[96.58,19.83,38.43,82.10,82.22]},{"hostname":"f0720","id":"17","statistics":{"min":19.63,"avg":63.54,"max":96.60},"data":[96.60,19.63,37.85,81.87,81.75]},{"hostname":"f0720","id":"18","statistics":{"min":19.62,"avg":63.65,"max":96.65},"data":[96.65,19.62,37.98,81.95,82.07]},{"hostname":"f0720","id":"29","statistics":{"min":19.63,"avg":63.44,"max":96.45},"data":[96.45,19.63,37.85,81.43,81.82]},{"hostname":"f0720","id":"31","statistics":{"min":19.77,"avg":63.54,"max":96.50},"data":[96.50,19.77,38.00,81.67,81.77]},{"hostname":"f0720","id":"48","statistics":{"min":19.88,"avg":63.82,"max":96.57},"data":[96.57,19.88,38.17,81.98,82.52]},{"hostname":"f0720","id":"63","statistics":{"min":19.77,"avg":63.69,"max":96.68},"data":[96.68,19.77,38.25,81.78,81.97]},{"hostname":"f0720","id":"19","statistics":{"min":19.55,"avg":63.70,"max":96.72},"data":[96.72,19.55,38.55,81.93,81.73]},{"hostname":"f0720","id":"23","statistics":{"min":19.65,"avg":63.61,"max":96.48},"data":[96.48,19.65,38.05,81.93,81.92]},{"hostname":"f0720","id":"27","statistics":{"min":19.80,"avg":63.51,"max":96.58},"data":[96.58,19.80,37.88,81.48,81.82]},{"hostname":"f0720","id":"28","statistics":{"min":19.80,"avg":63.61,"max":96.77},"data":[96.77,19.80,38.00,81.65,81.82]},{"hostname":"f0720","id":"39","statistics":{"min":19.75,"avg":63.75,"max":96.70},"data":[96.70,19.75,38.20,81.92,82.17]},{"hostname":"f0720","id":"40","statistics":{"min":19.90,"avg":63.77,"max":96.63},"data":[96.63,19.90,38.03,82.15,82.12]},{"hostname":"f0720","id":"41","statistics":{"min":20.35,"avg":63.84,"max":96.48},"data":[96.48,20.35,38.05,82.20,82.10]},{"hostname":"f0720","id":"42","statistics":{"min":20.48,"avg":63.89,"max":96.65},"data":[96.65,20.48,38.25,81.95,82.10]},{"hostname":"f0720","id":"65","statistics":{"min":19.75,"avg":63.63,"max":96.73},"data":[96.73,19.75,37.93,81.70,82.02]},{"hostname":"f0720","id":"10","statistics":{"min":19.70,"avg":63.70,"max":96.73},"data":[96.73,19.70,38.03,81.88,82.17]},{"hostname":"f0720","id":"11","statistics":{"min":19.83,"avg":63.67,"max":96.67},"data":[96.67,19.83,37.97,81.67,82.22]},{"hostname":"f0720","id":"32","statistics":{"min":19.78,"avg":63.51,"max":96.68},"data":[96.68,19.78,37.95,81.48,81.65]},{"hostname":"f0720","id":"49","statistics":{"min":20.37,"avg":63.93,"max":96.63},"data":[96.63,20.37,38.47,81.85,82.32]},{"hostname":"f0720","id":"0","statistics":{"min":19.48,"avg":63.38,"max":96.35},"data":[96.35,19.48,38.43,81.60,81.02]},{"hostname":"f0720","id":"3","statistics":{"min":19.82,"avg":63.76,"max":96.70},"data":[96.70,19.82,38.08,82.07,82.13]},{"hostname":"f0720","id":"36","statistics":{"min":20.02,"avg":63.85,"max":96.62},"data":[96.62,20.02,38.15,82.35,82.12]},{"hostname":"f0720","id":"64","statistics":{"min":19.78,"avg":63.57,"max":96.70},"data":[96.70,19.78,37.83,81.73,81.80]},{"hostname":"f0720","id":"66","statistics":{"min":19.83,"avg":63.60,"max":96.73},"data":[96.73,19.83,38.05,81.65,81.72]},{"hostname":"f0720","id":"71","statistics":{"min":19.97,"avg":63.63,"max":96.57},"data":[96.57,19.97,38.08,81.58,81.97]}]},"node":{"unit":{"base":""},"timestep":60,"series":[{"hostname":"f0720","id":"0","statistics":{"min":19.87,"avg":63.68,"max":96.63},"data":[96.63,19.87,38.07,81.84,81.98]}]}},"flops_any":{"core":{"unit":{"base":"F/s","prefix":"G"},"timestep":60,"series":[{"hostname":"f0720","id":"3","statistics":{"min":0.00,"avg":7.31,"max":10.66},"data":[10.66,0.00,8.41,8.52,8.98]},{"hostname":"f0720","id":"15","statistics":{"min":0.00,"avg":9.06,"max":19.60},"data":[19.60,0.00,8.34,8.46,8.90]},{"hostname":"f0720","id":"17","statistics":{"min":0.00,"avg":9.06,"max":19.61},"data":[19.61,0.00,8.34,8.46,8.90]},{"hostname":"f0720","id":"29","statistics":{"min":0.00,"avg":8.90,"max":19.46},"data":[19.46,0.00,8.13,8.27,8.64]},{"hostname":"f0720","id":"65","statistics":{"min":0.00,"avg":8.80,"max":19.58},"data":[19.58,0.00,7.93,8.18,8.30]},{"hostname":"f0720","id":"26","statistics":{"min":0.00,"avg":9.14,"max":19.43},"data":[19.43,0.00,8.35,8.80,9.13]},{"hostname":"f0720","id":"41","statistics":{"min":0.00,"avg":9.12,"max":20.27},"data":[20.27,0.00,8.24,8.39,8.68]},{"hostname":"f0720","id":"49","statistics":{"min":0.00,"avg":8.94,"max":19.73},"data":[19.73,0.00,8.16,8.36,8.46]},{"hostname":"f0720","id":"58","statistics":{"min":0.00,"avg":9.03,"max":19.45},"data":[19.45,0.00,8.16,8.74,8.81]},{"hostname":"f0720","id":"70","statistics":{"min":0.00,"avg":8.42,"max":18.96},"data":[18.96,0.00,7.16,8.02,7.97]},{"hostname":"f0720","id":"1","statistics":{"min":0.00,"avg":7.28,"max":10.61},"data":[10.61,0.00,8.37,8.48,8.94]},{"hostname":"f0720","id":"21","statistics":{"min":0.00,"avg":9.02,"max":19.52},"data":[19.52,0.00,8.29,8.41,8.85]},{"hostname":"f0720","id":"23","statistics":{"min":0.00,"avg":8.95,"max":19.43},"data":[19.43,0.00,8.23,8.35,8.73]},{"hostname":"f0720","id":"45","statistics":{"min":0.00,"avg":9.09,"max":20.40},"data":[20.40,0.00,8.18,8.38,8.49]},{"hostname":"f0720","id":"50","statistics":{"min":0.00,"avg":9.07,"max":19.60},"data":[19.60,0.00,8.18,8.75,8.82]},{"hostname":"f0720","id":"54","statistics":{"min":0.00,"avg":9.07,"max":19.61},"data":[19.60,0.00,8.17,8.75,8.82]},{"hostname":"f0720","id":"7","statistics":{"min":0.00,"avg":7.29,"max":10.70},"data":[10.70,0.00,8.36,8.48,8.92]},{"hostname":"f0720","id":"9","statistics":{"min":0.00,"avg":7.48,"max":11.63},"data":[11.63,0.00,8.36,8.48,8.93]},{"hostname":"f0720","id":"27","statistics":{"min":0.00,"avg":8.93,"max":19.43},"data":[19.43,0.00,8.21,8.34,8.70]},{"hostname":"f0720","id":"33","statistics":{"min":0.00,"avg":8.84,"max":19.44},"data":[19.44,0.00,8.05,8.19,8.51]},{"hostname":"f0720","id":"43","statistics":{"min":0.00,"avg":9.10,"max":20.39},"data":[20.39,0.00,8.24,8.38,8.50]},{"hostname":"f0720","id":"55","statistics":{"min":0.00,"avg":8.93,"max":19.71},"data":[19.71,0.00,8.11,8.35,8.45]},{"hostname":"f0720","id":"68","statistics":{"min":0.00,"avg":8.88,"max":19.45},"data":[19.45,0.00,7.91,8.47,8.58]},{"hostname":"f0720","id":"6","statistics":{"min":0.00,"avg":7.46,"max":10.74},"data":[10.74,0.00,8.42,8.87,9.29]},{"hostname":"f0720","id":"34","statistics":{"min":0.00,"avg":8.39,"max":18.96},"data":[18.96,0.00,7.10,7.52,8.39]},{"hostname":"f0720","id":"35","statistics":{"min":0.00,"avg":8.23,"max":18.91},"data":[18.91,0.00,7.33,7.35,7.57]},{"hostname":"f0720","id":"64","statistics":{"min":0.00,"avg":8.94,"max":19.48},"data":[19.48,0.00,7.99,8.57,8.66]},{"hostname":"f0720","id":"13","statistics":{"min":0.00,"avg":9.06,"max":19.62},"data":[19.62,0.00,8.35,8.46,8.90]},{"hostname":"f0720","id":"24","statistics":{"min":0.00,"avg":9.15,"max":19.41},"data":[19.41,0.00,8.36,8.82,9.15]},{"hostname":"f0720","id":"44","statistics":{"min":0.00,"avg":9.23,"max":20.26},"data":[20.26,0.00,8.29,8.77,8.85]},{"hostname":"f0720","id":"59","statistics":{"min":0.00,"avg":8.83,"max":19.55},"data":[19.55,0.00,7.99,8.25,8.35]},{"hostname":"f0720","id":"60","statistics":{"min":0.00,"avg":9.04,"max":19.49},"data":[19.49,0.00,8.16,8.74,8.81]},{"hostname":"f0720","id":"16","statistics":{"min":0.00,"avg":9.21,"max":19.63},"data":[19.63,0.00,8.38,8.83,9.23]},{"hostname":"f0720","id":"42","statistics":{"min":0.00,"avg":9.29,"max":20.32},"data":[20.32,0.00,8.34,8.80,9.01]},{"hostname":"f0720","id":"67","statistics":{"min":0.00,"avg":8.75,"max":19.48},"data":[19.48,0.00,7.88,8.12,8.26]},{"hostname":"f0720","id":"11","statistics":{"min":0.00,"avg":8.71,"max":17.80},"data":[17.80,0.00,8.35,8.47,8.91]},{"hostname":"f0720","id":"14","statistics":{"min":0.00,"avg":9.21,"max":19.60},"data":[19.60,0.00,8.38,8.83,9.24]},{"hostname":"f0720","id":"19","statistics":{"min":0.00,"avg":9.06,"max":19.60},"data":[19.60,0.00,8.34,8.46,8.89]},{"hostname":"f0720","id":"22","statistics":{"min":0.00,"avg":9.16,"max":19.44},"data":[19.44,0.00,8.37,8.82,9.16]},{"hostname":"f0720","id":"25","statistics":{"min":0.00,"avg":8.95,"max":19.47},"data":[19.47,0.00,8.22,8.35,8.72]},{"hostname":"f0720","id":"48","statistics":{"min":0.00,"avg":9.09,"max":19.64},"data":[19.64,0.00,8.23,8.76,8.83]},{"hostname":"f0720","id":"51","statistics":{"min":0.00,"avg":8.93,"max":19.72},"data":[19.72,0.00,8.11,8.36,8.46]},{"hostname":"f0720","id":"2","statistics":{"min":0.00,"avg":7.42,"max":10.60},"data":[10.60,0.00,8.39,8.85,9.27]},{"hostname":"f0720","id":"20","statistics":{"min":0.00,"avg":9.21,"max":19.61},"data":[19.61,0.00,8.37,8.83,9.23]},{"hostname":"f0720","id":"28","statistics":{"min":0.00,"avg":9.07,"max":19.55},"data":[19.55,0.00,8.17,8.65,8.97]},{"hostname":"f0720","id":"46","statistics":{"min":0.00,"avg":9.09,"max":19.58},"data":[19.58,0.00,8.23,8.77,8.84]},{"hostname":"f0720","id":"69","statistics":{"min":0.00,"avg":8.75,"max":19.63},"data":[19.63,0.00,7.85,8.09,8.19]},{"hostname":"f0720","id":"71","statistics":{"min":0.00,"avg":8.27,"max":18.84},"data":[18.84,0.00,7.52,7.27,7.72]},{"hostname":"f0720","id":"0","statistics":{"min":0.00,"avg":10.60,"max":13.89},"data":[12.32,0.00,13.49,13.27,13.89]},{"hostname":"f0720","id":"10","statistics":{"min":0.00,"avg":8.36,"max":15.29},"data":[15.29,0.00,8.39,8.85,9.26]},{"hostname":"f0720","id":"18","statistics":{"min":0.00,"avg":9.21,"max":19.61},"data":[19.61,0.00,8.38,8.83,9.23]},{"hostname":"f0720","id":"31","statistics":{"min":0.00,"avg":8.84,"max":19.34},"data":[19.34,0.00,8.08,8.22,8.54]},{"hostname":"f0720","id":"36","statistics":{"min":0.00,"avg":9.90,"max":23.22},"data":[23.22,0.00,8.33,8.81,9.13]},{"hostname":"f0720","id":"53","statistics":{"min":0.00,"avg":8.93,"max":19.71},"data":[19.71,0.00,8.11,8.35,8.45]},{"hostname":"f0720","id":"57","statistics":{"min":0.00,"avg":8.88,"max":19.63},"data":[19.63,0.00,8.06,8.31,8.41]},{"hostname":"f0720","id":"61","statistics":{"min":0.00,"avg":8.83,"max":19.59},"data":[19.59,0.00,7.99,8.24,8.35]},{"hostname":"f0720","id":"5","statistics":{"min":0.00,"avg":7.26,"max":10.57},"data":[10.57,0.00,8.35,8.47,8.93]},{"hostname":"f0720","id":"8","statistics":{"min":0.00,"avg":7.43,"max":10.69},"data":[10.69,0.00,8.38,8.84,9.25]},{"hostname":"f0720","id":"52","statistics":{"min":0.00,"avg":9.07,"max":19.62},"data":[19.62,0.00,8.17,8.75,8.82]},{"hostname":"f0720","id":"63","statistics":{"min":0.00,"avg":8.82,"max":19.55},"data":[19.55,0.00,7.97,8.23,8.34]},{"hostname":"f0720","id":"12","statistics":{"min":0.00,"avg":9.22,"max":19.64},"data":[19.65,0.00,8.38,8.83,9.24]},{"hostname":"f0720","id":"32","statistics":{"min":0.00,"avg":8.97,"max":19.37},"data":[19.37,0.00,8.08,8.55,8.83]},{"hostname":"f0720","id":"47","statistics":{"min":0.00,"avg":8.94,"max":19.67},"data":[19.67,0.00,8.17,8.37,8.47]},{"hostname":"f0720","id":"66","statistics":{"min":0.00,"avg":8.89,"max":19.36},"data":[19.36,0.00,7.94,8.51,8.62]},{"hostname":"f0720","id":"30","statistics":{"min":0.00,"avg":8.98,"max":19.36},"data":[19.35,0.00,8.12,8.58,8.87]},{"hostname":"f0720","id":"38","statistics":{"min":0.00,"avg":9.99,"max":23.78},"data":[23.78,0.00,8.31,8.79,9.06]},{"hostname":"f0720","id":"56","statistics":{"min":0.00,"avg":9.05,"max":19.53},"data":[19.53,0.00,8.17,8.74,8.82]},{"hostname":"f0720","id":"62","statistics":{"min":0.00,"avg":9.02,"max":19.44},"data":[19.44,0.00,8.14,8.72,8.80]},{"hostname":"f0720","id":"39","statistics":{"min":0.00,"avg":9.86,"max":23.91},"data":[23.91,0.00,8.28,8.42,8.71]},{"hostname":"f0720","id":"40","statistics":{"min":0.00,"avg":9.27,"max":20.18},"data":[20.18,0.00,8.31,8.79,9.06]},{"hostname":"f0720","id":"4","statistics":{"min":0.00,"avg":7.42,"max":10.58},"data":[10.58,0.00,8.39,8.85,9.27]},{"hostname":"f0720","id":"37","statistics":{"min":0.00,"avg":9.73,"max":23.31},"data":[23.31,0.00,8.25,8.40,8.68]}]},"node":{"unit":{"base":"F/s","prefix":"G"},"timestep":60,"series":[{"hostname":"f0720","id":"0","statistics":{"min":0.00,"avg":635.67,"max":1332.87},"data":[1332.87,0.00,593.24,616.33,635.92]}]}},"flops_dp":{"core":{"unit":{"base":"F/s","prefix":"G"},"timestep":60,"series":[{"hostname":"f0720","id":"4","statistics":{"min":0.00,"avg":3.71,"max":5.29},"data":[5.29,0.00,4.20,4.43,4.64]},{"hostname":"f0720","id":"25","statistics":{"min":0.00,"avg":3.58,"max":5.27},"data":[5.27,0.00,4.11,4.18,4.36]},{"hostname":"f0720","id":"41","statistics":{"min":0.00,"avg":3.60,"max":5.35},"data":[5.35,0.00,4.12,4.20,4.34]},{"hostname":"f0720","id":"42","statistics":{"min":0.00,"avg":3.69,"max":5.37},"data":[5.37,0.00,4.17,4.40,4.51]},{"hostname":"f0720","id":"46","statistics":{"min":0.00,"avg":3.64,"max":5.29},"data":[5.28,0.00,4.12,4.38,4.42]},{"hostname":"f0720","id":"63","statistics":{"min":0.00,"avg":3.52,"max":5.30},"data":[5.30,0.00,3.99,4.11,4.17]},{"hostname":"f0720","id":"5","statistics":{"min":0.00,"avg":3.63,"max":5.28},"data":[5.28,0.00,4.18,4.24,4.46]},{"hostname":"f0720","id":"8","statistics":{"min":0.00,"avg":3.72,"max":5.34},"data":[5.34,0.00,4.19,4.42,4.63]},{"hostname":"f0720","id":"14","statistics":{"min":0.00,"avg":3.70,"max":5.29},"data":[5.30,0.00,4.19,4.42,4.62]},{"hostname":"f0720","id":"16","statistics":{"min":0.00,"avg":3.71,"max":5.34},"data":[5.34,0.00,4.19,4.42,4.62]},{"hostname":"f0720","id":"21","statistics":{"min":0.00,"avg":3.62,"max":5.33},"data":[5.33,0.00,4.15,4.21,4.42]},{"hostname":"f0720","id":"39","statistics":{"min":0.00,"avg":3.62,"max":5.38},"data":[5.38,0.00,4.14,4.21,4.35]},{"hostname":"f0720","id":"13","statistics":{"min":0.00,"avg":3.63,"max":5.30},"data":[5.30,0.00,4.17,4.23,4.45]},{"hostname":"f0720","id":"33","statistics":{"min":0.00,"avg":3.52,"max":5.25},"data":[5.25,0.00,4.03,4.09,4.26]},{"hostname":"f0720","id":"34","statistics":{"min":0.00,"avg":3.30,"max":5.01},"data":[5.01,0.00,3.55,3.76,4.20]},{"hostname":"f0720","id":"64","statistics":{"min":0.00,"avg":3.58,"max":5.31},"data":[5.31,0.00,4.00,4.28,4.33]},{"hostname":"f0720","id":"18","statistics":{"min":0.00,"avg":3.71,"max":5.34},"data":[5.34,0.00,4.19,4.41,4.62]},{"hostname":"f0720","id":"27","statistics":{"min":0.00,"avg":3.57,"max":5.25},"data":[5.25,0.00,4.10,4.17,4.35]},{"hostname":"f0720","id":"58","statistics":{"min":0.00,"avg":3.63,"max":5.29},"data":[5.29,0.00,4.08,4.37,4.41]},{"hostname":"f0720","id":"59","statistics":{"min":0.00,"avg":3.53,"max":5.34},"data":[5.34,0.00,4.00,4.12,4.17]},{"hostname":"f0720","id":"66","statistics":{"min":0.00,"avg":3.56,"max":5.25},"data":[5.25,0.00,3.97,4.26,4.31]},{"hostname":"f0720","id":"1","statistics":{"min":0.00,"avg":3.64,"max":5.30},"data":[5.30,0.00,4.19,4.24,4.47]},{"hostname":"f0720","id":"15","statistics":{"min":0.00,"avg":3.63,"max":5.29},"data":[5.30,0.00,4.17,4.23,4.45]},{"hostname":"f0720","id":"35","statistics":{"min":0.00,"avg":3.22,"max":4.99},"data":[4.99,0.00,3.66,3.67,3.78]},{"hostname":"f0720","id":"45","statistics":{"min":0.00,"avg":3.59,"max":5.41},"data":[5.41,0.00,4.09,4.19,4.24]},{"hostname":"f0720","id":"50","statistics":{"min":0.00,"avg":3.63,"max":5.29},"data":[5.29,0.00,4.09,4.38,4.41]},{"hostname":"f0720","id":"6","statistics":{"min":0.00,"avg":3.73,"max":5.37},"data":[5.37,0.00,4.21,4.43,4.64]},{"hostname":"f0720","id":"19","statistics":{"min":0.00,"avg":3.63,"max":5.33},"data":[5.33,0.00,4.17,4.23,4.45]},{"hostname":"f0720","id":"53","statistics":{"min":0.00,"avg":3.57,"max":5.39},"data":[5.39,0.00,4.05,4.18,4.23]},{"hostname":"f0720","id":"12","statistics":{"min":0.00,"avg":3.71,"max":5.31},"data":[5.31,0.00,4.19,4.42,4.62]},{"hostname":"f0720","id":"49","statistics":{"min":0.00,"avg":3.57,"max":5.36},"data":[5.36,0.00,4.08,4.18,4.23]},{"hostname":"f0720","id":"54","statistics":{"min":0.00,"avg":3.64,"max":5.33},"data":[5.33,0.00,4.09,4.37,4.41]},{"hostname":"f0720","id":"62","statistics":{"min":0.00,"avg":3.62,"max":5.25},"data":[5.26,0.00,4.07,4.36,4.40]},{"hostname":"f0720","id":"69","statistics":{"min":0.00,"avg":3.48,"max":5.35},"data":[5.35,0.00,3.92,4.05,4.09]},{"hostname":"f0720","id":"3","statistics":{"min":0.00,"avg":3.66,"max":5.33},"data":[5.33,0.00,4.20,4.26,4.49]},{"hostname":"f0720","id":"7","statistics":{"min":0.00,"avg":3.65,"max":5.35},"data":[5.35,0.00,4.18,4.24,4.46]},{"hostname":"f0720","id":"10","statistics":{"min":0.00,"avg":3.71,"max":5.29},"data":[5.29,0.00,4.19,4.42,4.63]},{"hostname":"f0720","id":"23","statistics":{"min":0.00,"avg":3.59,"max":5.29},"data":[5.28,0.00,4.11,4.18,4.36]},{"hostname":"f0720","id":"47","statistics":{"min":0.00,"avg":3.57,"max":5.33},"data":[5.33,0.00,4.08,4.18,4.24]},{"hostname":"f0720","id":"55","statistics":{"min":0.00,"avg":3.57,"max":5.38},"data":[5.39,0.00,4.05,4.18,4.23]},{"hostname":"f0720","id":"17","statistics":{"min":0.00,"avg":3.64,"max":5.34},"data":[5.34,0.00,4.17,4.23,4.45]},{"hostname":"f0720","id":"37","statistics":{"min":0.00,"avg":3.61,"max":5.37},"data":[5.36,0.00,4.13,4.20,4.34]},{"hostname":"f0720","id":"56","statistics":{"min":0.00,"avg":3.64,"max":5.33},"data":[5.33,0.00,4.08,4.37,4.41]},{"hostname":"f0720","id":"61","statistics":{"min":0.00,"avg":3.52,"max":5.33},"data":[5.32,0.00,3.99,4.12,4.17]},{"hostname":"f0720","id":"67","statistics":{"min":0.00,"avg":3.49,"max":5.31},"data":[5.31,0.00,3.94,4.06,4.13]},{"hostname":"f0720","id":"28","statistics":{"min":0.00,"avg":3.64,"max":5.31},"data":[5.31,0.00,4.08,4.33,4.49]},{"hostname":"f0720","id":"43","statistics":{"min":0.00,"avg":3.59,"max":5.40},"data":[5.40,0.00,4.12,4.19,4.25]},{"hostname":"f0720","id":"68","statistics":{"min":0.00,"avg":3.55,"max":5.26},"data":[5.26,0.00,3.96,4.24,4.29]},{"hostname":"f0720","id":"40","statistics":{"min":0.00,"avg":3.68,"max":5.30},"data":[5.30,0.00,4.15,4.40,4.53]},{"hostname":"f0720","id":"60","statistics":{"min":0.00,"avg":3.62,"max":5.27},"data":[5.27,0.00,4.08,4.37,4.40]},{"hostname":"f0720","id":"0","statistics":{"min":0.00,"avg":5.30,"max":6.95},"data":[6.16,0.00,6.74,6.64,6.95]},{"hostname":"f0720","id":"2","statistics":{"min":0.00,"avg":3.71,"max":5.30},"data":[5.30,0.00,4.20,4.42,4.63]},{"hostname":"f0720","id":"9","statistics":{"min":0.00,"avg":3.65,"max":5.36},"data":[5.36,0.00,4.18,4.24,4.46]},{"hostname":"f0720","id":"22","statistics":{"min":0.00,"avg":3.69,"max":5.29},"data":[5.29,0.00,4.18,4.41,4.58]},{"hostname":"f0720","id":"26","statistics":{"min":0.00,"avg":3.68,"max":5.25},"data":[5.25,0.00,4.17,4.40,4.56]},{"hostname":"f0720","id":"38","statistics":{"min":0.00,"avg":3.68,"max":5.31},"data":[5.31,0.00,4.16,4.40,4.53]},{"hostname":"f0720","id":"20","statistics":{"min":0.00,"avg":3.71,"max":5.33},"data":[5.34,0.00,4.19,4.41,4.61]},{"hostname":"f0720","id":"30","statistics":{"min":0.00,"avg":3.61,"max":5.25},"data":[5.25,0.00,4.06,4.29,4.43]},{"hostname":"f0720","id":"36","statistics":{"min":0.00,"avg":3.69,"max":5.32},"data":[5.32,0.00,4.16,4.40,4.56]},{"hostname":"f0720","id":"65","statistics":{"min":0.00,"avg":3.51,"max":5.36},"data":[5.36,0.00,3.96,4.09,4.15]},{"hostname":"f0720","id":"29","statistics":{"min":0.00,"avg":3.56,"max":5.30},"data":[5.30,0.00,4.07,4.14,4.32]},{"hostname":"f0720","id":"70","statistics":{"min":0.00,"avg":3.32,"max":5.01},"data":[5.01,0.00,3.58,4.01,3.98]},{"hostname":"f0720","id":"24","statistics":{"min":0.00,"avg":3.69,"max":5.27},"data":[5.27,0.00,4.18,4.41,4.58]},{"hostname":"f0720","id":"32","statistics":{"min":0.00,"avg":3.60,"max":5.25},"data":[5.25,0.00,4.04,4.27,4.42]},{"hostname":"f0720","id":"52","statistics":{"min":0.00,"avg":3.64,"max":5.34},"data":[5.34,0.00,4.09,4.38,4.41]},{"hostname":"f0720","id":"71","statistics":{"min":0.00,"avg":3.24,"max":4.95},"data":[4.95,0.00,3.76,3.63,3.86]},{"hostname":"f0720","id":"11","statistics":{"min":0.00,"avg":3.63,"max":5.27},"data":[5.27,0.00,4.17,4.23,4.46]},{"hostname":"f0720","id":"31","statistics":{"min":0.00,"avg":3.53,"max":5.24},"data":[5.24,0.00,4.04,4.11,4.27]},{"hostname":"f0720","id":"44","statistics":{"min":0.00,"avg":3.66,"max":5.34},"data":[5.34,0.00,4.14,4.38,4.43]},{"hostname":"f0720","id":"48","statistics":{"min":0.00,"avg":3.64,"max":5.31},"data":[5.31,0.00,4.11,4.38,4.41]},{"hostname":"f0720","id":"51","statistics":{"min":0.00,"avg":3.56,"max":5.35},"data":[5.35,0.00,4.06,4.18,4.23]},{"hostname":"f0720","id":"57","statistics":{"min":0.00,"avg":3.55,"max":5.38},"data":[5.38,0.00,4.03,4.15,4.20]}]},"node":{"unit":{"base":"F/s","prefix":"G"},"timestep":60,"series":[{"hostname":"f0720","id":"0","statistics":{"min":0.00,"avg":261.01,"max":382.29},"data":[382.29,0.00,296.62,308.16,317.96]}]}},"flops_sp":{"core":{"unit":{"base":"F/s","prefix":"G"},"timestep":60,"series":[{"hostname":"f0720","id":"1","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"4","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"5","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"19","statistics":{"min":0.00,"avg":1.79,"max":8.94},"data":[8.94,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"34","statistics":{"min":0.00,"avg":1.79,"max":8.94},"data":[8.94,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"35","statistics":{"min":0.00,"avg":1.79,"max":8.94},"data":[8.94,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"7","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"33","statistics":{"min":0.00,"avg":1.79,"max":8.94},"data":[8.94,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"42","statistics":{"min":0.00,"avg":1.92,"max":9.58},"data":[9.58,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"52","statistics":{"min":0.00,"avg":1.79,"max":8.94},"data":[8.94,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"13","statistics":{"min":0.00,"avg":1.80,"max":9.01},"data":[9.01,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"14","statistics":{"min":0.00,"avg":1.80,"max":9.01},"data":[9.01,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"68","statistics":{"min":0.00,"avg":1.79,"max":8.94},"data":[8.94,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"6","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"20","statistics":{"min":0.00,"avg":1.79,"max":8.94},"data":[8.94,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"22","statistics":{"min":0.00,"avg":1.77,"max":8.86},"data":[8.86,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"36","statistics":{"min":0.00,"avg":2.52,"max":12.59},"data":[12.59,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"45","statistics":{"min":0.00,"avg":1.92,"max":9.58},"data":[9.58,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"57","statistics":{"min":0.00,"avg":1.77,"max":8.86},"data":[8.86,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"11","statistics":{"min":0.00,"avg":1.45,"max":7.26},"data":[7.26,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"26","statistics":{"min":0.00,"avg":1.79,"max":8.94},"data":[8.94,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"69","statistics":{"min":0.00,"avg":1.79,"max":8.94},"data":[8.94,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"16","statistics":{"min":0.00,"avg":1.79,"max":8.94},"data":[8.94,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"21","statistics":{"min":0.00,"avg":1.77,"max":8.86},"data":[8.86,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"25","statistics":{"min":0.00,"avg":1.79,"max":8.94},"data":[8.94,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"28","statistics":{"min":0.00,"avg":1.79,"max":8.94},"data":[8.94,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"32","statistics":{"min":0.00,"avg":1.77,"max":8.86},"data":[8.86,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"40","statistics":{"min":0.00,"avg":1.92,"max":9.59},"data":[9.59,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"56","statistics":{"min":0.00,"avg":1.77,"max":8.87},"data":[8.86,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"15","statistics":{"min":0.00,"avg":1.80,"max":9.01},"data":[9.01,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"37","statistics":{"min":0.00,"avg":2.52,"max":12.58},"data":[12.58,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"55","statistics":{"min":0.00,"avg":1.79,"max":8.94},"data":[8.94,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"64","statistics":{"min":0.00,"avg":1.77,"max":8.87},"data":[8.86,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"30","statistics":{"min":0.00,"avg":1.77,"max":8.86},"data":[8.86,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"38","statistics":{"min":0.00,"avg":2.63,"max":13.16},"data":[13.16,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"43","statistics":{"min":0.00,"avg":1.92,"max":9.59},"data":[9.59,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"2","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"3","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"18","statistics":{"min":0.00,"avg":1.79,"max":8.94},"data":[8.94,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"31","statistics":{"min":0.00,"avg":1.77,"max":8.86},"data":[8.86,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"46","statistics":{"min":0.00,"avg":1.80,"max":9.01},"data":[9.01,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"17","statistics":{"min":0.00,"avg":1.79,"max":8.94},"data":[8.94,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"60","statistics":{"min":0.00,"avg":1.79,"max":8.94},"data":[8.94,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"9","statistics":{"min":0.00,"avg":0.18,"max":0.91},"data":[0.90,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"12","statistics":{"min":0.00,"avg":1.80,"max":9.02},"data":[9.02,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"50","statistics":{"min":0.00,"avg":1.80,"max":9.01},"data":[9.01,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"51","statistics":{"min":0.00,"avg":1.80,"max":9.02},"data":[9.02,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"53","statistics":{"min":0.00,"avg":1.79,"max":8.94},"data":[8.94,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"65","statistics":{"min":0.00,"avg":1.77,"max":8.86},"data":[8.86,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"8","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"61","statistics":{"min":0.00,"avg":1.79,"max":8.94},"data":[8.94,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"71","statistics":{"min":0.00,"avg":1.79,"max":8.94},"data":[8.94,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"10","statistics":{"min":0.00,"avg":0.94,"max":4.71},"data":[4.71,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"27","statistics":{"min":0.00,"avg":1.79,"max":8.94},"data":[8.94,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"48","statistics":{"min":0.00,"avg":1.80,"max":9.02},"data":[9.02,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"49","statistics":{"min":0.00,"avg":1.80,"max":9.01},"data":[9.01,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"54","statistics":{"min":0.00,"avg":1.79,"max":8.94},"data":[8.94,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"58","statistics":{"min":0.00,"avg":1.77,"max":8.86},"data":[8.86,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"0","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"24","statistics":{"min":0.00,"avg":1.77,"max":8.86},"data":[8.86,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"44","statistics":{"min":0.00,"avg":1.92,"max":9.59},"data":[9.59,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"59","statistics":{"min":0.00,"avg":1.77,"max":8.86},"data":[8.86,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"67","statistics":{"min":0.00,"avg":1.77,"max":8.86},"data":[8.86,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"70","statistics":{"min":0.00,"avg":1.79,"max":8.94},"data":[8.94,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"23","statistics":{"min":0.00,"avg":1.77,"max":8.86},"data":[8.86,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"29","statistics":{"min":0.00,"avg":1.77,"max":8.86},"data":[8.86,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"41","statistics":{"min":0.00,"avg":1.92,"max":9.58},"data":[9.58,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"62","statistics":{"min":0.00,"avg":1.79,"max":8.94},"data":[8.94,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"63","statistics":{"min":0.00,"avg":1.79,"max":8.94},"data":[8.94,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"66","statistics":{"min":0.00,"avg":1.77,"max":8.86},"data":[8.86,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"39","statistics":{"min":0.00,"avg":2.63,"max":13.16},"data":[13.16,0.00,0.00,0.00,0.00]},{"hostname":"f0720","id":"47","statistics":{"min":0.00,"avg":1.80,"max":9.02},"data":[9.02,0.00,0.00,0.00,0.00]}]},"node":{"unit":{"base":"F/s","prefix":"G"},"timestep":60,"series":[{"hostname":"f0720","id":"0","statistics":{"min":0.00,"avg":113.66,"max":568.29},"data":[568.29,0.00,0.00,0.00,0.00]}]}},"ib_recv":{"node":{"unit":{"base":"B/s"},"timestep":60,"series":[{"hostname":"f0720","statistics":{"min":69.40,"avg":27981.11,"max":48084.59},"data":[69.40,682.30,42991.74,48077.52,48084.59]}]}},"ib_recv_pkts":{"node":{"unit":{"base":"packets/s"},"timestep":60,"series":[{"hostname":"f0720","statistics":{"min":0.50,"avg":398.94,"max":693.82},"data":[0.50,2.93,605.84,693.82,691.60]}]}},"ib_xmit":{"node":{"unit":{"base":"B/s"},"timestep":60,"series":[{"hostname":"f0720","statistics":{"min":39.60,"avg":188.51,"max":724.57},"data":[69.40,724.57,59.40,49.60,39.60]}]}},"ib_xmit_pkts":{"node":{"unit":{"base":"packets/s"},"timestep":60,"series":[{"hostname":"f0720","statistics":{"min":0.20,"avg":0.87,"max":2.93},"data":[0.50,2.93,0.30,0.40,0.20]}]}},"ipc":{"core":{"unit":{"base":"IPC"},"timestep":60,"series":[{"hostname":"f0720","id":"18","statistics":{"min":0.56,"avg":0.91,"max":1.25},"data":[1.25,0.56,0.93,0.91,0.91]},{"hostname":"f0720","id":"38","statistics":{"min":0.31,"avg":0.87,"max":1.28},"data":[1.28,0.31,0.92,0.94,0.90]},{"hostname":"f0720","id":"44","statistics":{"min":0.57,"avg":0.92,"max":1.25},"data":[1.25,0.57,0.93,0.95,0.90]},{"hostname":"f0720","id":"5","statistics":{"min":0.55,"avg":0.97,"max":1.31},"data":[1.31,0.55,0.99,1.02,0.99]},{"hostname":"f0720","id":"15","statistics":{"min":0.57,"avg":0.92,"max":1.27},"data":[1.27,0.57,0.92,0.91,0.93]},{"hostname":"f0720","id":"20","statistics":{"min":0.23,"avg":0.85,"max":1.26},"data":[1.26,0.23,0.94,0.89,0.92]},{"hostname":"f0720","id":"34","statistics":{"min":0.58,"avg":0.95,"max":1.25},"data":[1.25,0.58,1.00,0.96,0.95]},{"hostname":"f0720","id":"39","statistics":{"min":0.27,"avg":0.91,"max":1.34},"data":[1.34,0.26,0.95,1.00,0.98]},{"hostname":"f0720","id":"46","statistics":{"min":0.54,"avg":0.91,"max":1.27},"data":[1.27,0.54,0.93,0.93,0.90]},{"hostname":"f0720","id":"48","statistics":{"min":0.56,"avg":0.93,"max":1.29},"data":[1.29,0.56,0.94,0.95,0.91]},{"hostname":"f0720","id":"64","statistics":{"min":0.58,"avg":0.94,"max":1.27},"data":[1.27,0.58,0.94,0.98,0.95]},{"hostname":"f0720","id":"6","statistics":{"min":0.74,"avg":1.03,"max":1.36},"data":[1.37,0.75,1.04,0.99,1.02]},{"hostname":"f0720","id":"14","statistics":{"min":0.57,"avg":0.92,"max":1.25},"data":[1.25,0.57,0.93,0.91,0.93]},{"hostname":"f0720","id":"56","statistics":{"min":0.29,"avg":0.88,"max":1.26},"data":[1.26,0.29,0.95,0.96,0.94]},{"hostname":"f0720","id":"67","statistics":{"min":0.72,"avg":0.98,"max":1.30},"data":[1.30,0.72,0.93,0.99,0.96]},{"hostname":"f0720","id":"65","statistics":{"min":0.57,"avg":0.95,"max":1.31},"data":[1.31,0.57,0.93,0.99,0.94]},{"hostname":"f0720","id":"37","statistics":{"min":0.22,"avg":0.91,"max":1.43},"data":[1.43,0.22,0.95,0.99,0.94]},{"hostname":"f0720","id":"51","statistics":{"min":0.24,"avg":0.87,"max":1.29},"data":[1.29,0.24,0.92,0.95,0.94]},{"hostname":"f0720","id":"28","statistics":{"min":0.58,"avg":0.93,"max":1.24},"data":[1.24,0.58,0.95,0.93,0.94]},{"hostname":"f0720","id":"40","statistics":{"min":0.52,"avg":0.91,"max":1.26},"data":[1.26,0.51,0.92,0.94,0.92]},{"hostname":"f0720","id":"52","statistics":{"min":0.14,"avg":0.84,"max":1.26},"data":[1.26,0.14,0.94,0.94,0.91]},{"hostname":"f0720","id":"58","statistics":{"min":0.55,"avg":0.93,"max":1.26},"data":[1.26,0.55,0.93,0.97,0.93]},{"hostname":"f0720","id":"70","statistics":{"min":0.55,"avg":0.96,"max":1.29},"data":[1.29,0.55,1.00,1.00,0.96]},{"hostname":"f0720","id":"9","statistics":{"min":0.93,"avg":1.02,"max":1.28},"data":[1.28,0.97,0.93,0.96,0.96]},{"hostname":"f0720","id":"11","statistics":{"min":0.55,"avg":0.90,"max":1.24},"data":[1.24,0.55,0.90,0.89,0.92]},{"hostname":"f0720","id":"60","statistics":{"min":0.33,"avg":0.88,"max":1.25},"data":[1.25,0.33,0.93,0.95,0.93]},{"hostname":"f0720","id":"63","statistics":{"min":0.55,"avg":0.94,"max":1.29},"data":[1.29,0.55,0.93,0.99,0.95]},{"hostname":"f0720","id":"13","statistics":{"min":0.24,"avg":0.86,"max":1.27},"data":[1.27,0.24,0.92,0.93,0.94]},{"hostname":"f0720","id":"31","statistics":{"min":0.55,"avg":0.92,"max":1.26},"data":[1.26,0.55,0.92,0.93,0.94]},{"hostname":"f0720","id":"42","statistics":{"min":0.55,"avg":0.92,"max":1.25},"data":[1.25,0.55,0.94,0.95,0.92]},{"hostname":"f0720","id":"59","statistics":{"min":0.56,"avg":0.95,"max":1.31},"data":[1.31,0.56,0.93,0.98,0.95]},{"hostname":"f0720","id":"1","statistics":{"min":0.18,"avg":0.97,"max":1.48},"data":[1.48,0.18,1.05,1.07,1.06]},{"hostname":"f0720","id":"12","statistics":{"min":0.56,"avg":0.92,"max":1.28},"data":[1.28,0.56,0.94,0.91,0.91]},{"hostname":"f0720","id":"10","statistics":{"min":0.56,"avg":0.91,"max":1.26},"data":[1.26,0.56,0.93,0.89,0.91]},{"hostname":"f0720","id":"17","statistics":{"min":0.55,"avg":0.90,"max":1.23},"data":[1.23,0.55,0.91,0.90,0.92]},{"hostname":"f0720","id":"19","statistics":{"min":0.92,"avg":1.06,"max":1.31},"data":[1.24,1.31,0.92,0.92,0.93]},{"hostname":"f0720","id":"21","statistics":{"min":0.57,"avg":0.91,"max":1.23},"data":[1.23,0.57,0.90,0.91,0.91]},{"hostname":"f0720","id":"30","statistics":{"min":0.52,"avg":0.91,"max":1.25},"data":[1.25,0.52,0.94,0.92,0.93]},{"hostname":"f0720","id":"36","statistics":{"min":0.28,"avg":0.99,"max":1.49},"data":[1.49,0.28,1.05,1.08,1.04]},{"hostname":"f0720","id":"0","statistics":{"min":1.57,"avg":1.71,"max":2.04},"data":[1.71,2.04,1.65,1.58,1.57]},{"hostname":"f0720","id":"7","statistics":{"min":0.96,"avg":1.06,"max":1.33},"data":[1.33,0.96,1.00,1.00,1.00]},{"hostname":"f0720","id":"45","statistics":{"min":0.56,"avg":0.94,"max":1.30},"data":[1.30,0.56,0.93,0.98,0.95]},{"hostname":"f0720","id":"69","statistics":{"min":0.55,"avg":0.93,"max":1.29},"data":[1.29,0.55,0.90,0.97,0.95]},{"hostname":"f0720","id":"55","statistics":{"min":0.23,"avg":0.88,"max":1.29},"data":[1.29,0.22,0.92,0.98,0.96]},{"hostname":"f0720","id":"57","statistics":{"min":0.53,"avg":0.93,"max":1.28},"data":[1.28,0.53,0.92,0.98,0.95]},{"hostname":"f0720","id":"22","statistics":{"min":0.57,"avg":0.91,"max":1.23},"data":[1.23,0.57,0.94,0.89,0.92]},{"hostname":"f0720","id":"29","statistics":{"min":0.58,"avg":0.92,"max":1.25},"data":[1.25,0.58,0.92,0.94,0.93]},{"hostname":"f0720","id":"54","statistics":{"min":0.93,"avg":1.02,"max":1.28},"data":[1.28,1.01,0.94,0.95,0.93]},{"hostname":"f0720","id":"26","statistics":{"min":0.56,"avg":0.92,"max":1.27},"data":[1.27,0.56,0.93,0.89,0.93]},{"hostname":"f0720","id":"27","statistics":{"min":0.54,"avg":0.93,"max":1.26},"data":[1.26,0.54,0.94,0.94,0.96]},{"hostname":"f0720","id":"23","statistics":{"min":0.58,"avg":0.93,"max":1.26},"data":[1.26,0.58,0.91,0.93,0.93]},{"hostname":"f0720","id":"66","statistics":{"min":0.56,"avg":0.93,"max":1.25},"data":[1.25,0.56,0.95,0.97,0.93]},{"hostname":"f0720","id":"35","statistics":{"min":0.56,"avg":0.95,"max":1.27},"data":[1.27,0.56,0.97,0.97,0.98]},{"hostname":"f0720","id":"47","statistics":{"min":0.51,"avg":0.94,"max":1.33},"data":[1.33,0.51,0.92,0.97,0.95]},{"hostname":"f0720","id":"68","statistics":{"min":0.56,"avg":0.94,"max":1.26},"data":[1.26,0.56,0.95,0.97,0.94]},{"hostname":"f0720","id":"3","statistics":{"min":0.56,"avg":1.01,"max":1.38},"data":[1.38,0.56,1.05,1.03,1.06]},{"hostname":"f0720","id":"8","statistics":{"min":0.90,"avg":1.38,"max":2.88},"data":[1.24,2.88,0.95,0.90,0.92]},{"hostname":"f0720","id":"32","statistics":{"min":0.47,"avg":0.91,"max":1.26},"data":[1.26,0.47,0.95,0.93,0.93]},{"hostname":"f0720","id":"16","statistics":{"min":0.19,"avg":0.84,"max":1.23},"data":[1.23,0.19,0.94,0.91,0.93]},{"hostname":"f0720","id":"25","statistics":{"min":0.55,"avg":0.91,"max":1.23},"data":[1.23,0.55,0.91,0.91,0.94]},{"hostname":"f0720","id":"43","statistics":{"min":0.58,"avg":0.95,"max":1.32},"data":[1.32,0.58,0.92,0.98,0.96]},{"hostname":"f0720","id":"50","statistics":{"min":0.52,"avg":0.90,"max":1.23},"data":[1.23,0.52,0.93,0.93,0.90]},{"hostname":"f0720","id":"61","statistics":{"min":0.57,"avg":0.95,"max":1.30},"data":[1.30,0.57,0.91,0.99,0.96]},{"hostname":"f0720","id":"71","statistics":{"min":0.13,"avg":0.88,"max":1.31},"data":[1.31,0.13,0.95,1.05,0.98]},{"hostname":"f0720","id":"4","statistics":{"min":0.26,"avg":0.91,"max":1.30},"data":[1.30,0.26,1.01,0.97,0.98]},{"hostname":"f0720","id":"41","statistics":{"min":0.22,"avg":0.89,"max":1.34},"data":[1.34,0.22,0.96,0.96,0.96]},{"hostname":"f0720","id":"33","statistics":{"min":0.55,"avg":0.93,"max":1.26},"data":[1.26,0.55,0.93,0.94,0.94]},{"hostname":"f0720","id":"49","statistics":{"min":0.55,"avg":0.93,"max":1.29},"data":[1.29,0.55,0.92,0.96,0.94]},{"hostname":"f0720","id":"53","statistics":{"min":0.54,"avg":0.92,"max":1.31},"data":[1.31,0.54,0.90,0.93,0.93]},{"hostname":"f0720","id":"62","statistics":{"min":0.57,"avg":0.93,"max":1.26},"data":[1.26,0.57,0.94,0.96,0.93]},{"hostname":"f0720","id":"2","statistics":{"min":0.22,"avg":0.91,"max":1.38},"data":[1.38,0.22,0.99,0.97,0.97]},{"hostname":"f0720","id":"24","statistics":{"min":0.57,"avg":0.91,"max":1.24},"data":[1.24,0.57,0.93,0.91,0.92]}]},"node":{"unit":{"base":"IPC"},"timestep":60,"series":[{"hostname":"f0720","id":"0","statistics":{"min":0.56,"avg":0.94,"max":1.29},"data":[1.29,0.56,0.95,0.96,0.95]}]}},"mem_bw":{"node":{"unit":{"base":"B/s","prefix":"G"},"timestep":60,"series":[{"hostname":"f0720","id":"0","statistics":{"min":0.02,"avg":79.56,"max":116.02},"data":[77.15,0.02,95.80,108.83,116.02]}]},"socket":{"unit":{"base":"B/s","prefix":"G"},"timestep":60,"series":[{"hostname":"f0720","id":"0","statistics":{"min":0.02,"avg":40.68,"max":59.48},"data":[39.19,0.02,49.11,55.62,59.48]},{"hostname":"f0720","id":"0","statistics":{"min":0.01,"avg":38.88,"max":56.54},"data":[37.96,0.01,46.69,53.21,56.54]}]}},"mem_power":{"node":{"unit":{"base":"W"},"timestep":60,"series":[{"hostname":"f0720","id":"0","statistics":{"min":7.88,"avg":24.69,"max":31.32},"data":[25.91,7.88,28.52,29.83,31.32]}]},"socket":{"unit":{"base":"W"},"timestep":60,"series":[{"hostname":"f0720","id":"0","statistics":{"min":4.24,"avg":12.92,"max":16.39},"data":[13.51,4.24,14.92,15.55,16.39]},{"hostname":"f0720","id":"0","statistics":{"min":3.65,"avg":11.77,"max":14.93},"data":[12.40,3.65,13.60,14.27,14.93]}]}},"mem_used":{"node":{"unit":{"base":"B","prefix":"G"},"timestep":60,"series":[{"hostname":"f0720","statistics":{"min":8.22,"avg":22.57,"max":27.61},"data":[27.61,8.23,25.42,25.74,25.83]}]}},"nfs4_read":{"node":{"unit":{"base":"B/s","prefix":"M"},"timestep":60,"series":[{"hostname":"f0720","statistics":{"min":0.00,"avg":647.00,"max":1946.00},"data":[0.00,1272.00,1946.00,17.00,0.00]}]}},"nfs4_total":{"node":{"unit":{"base":"B/s","prefix":"M"},"timestep":60,"series":[{"hostname":"f0720","statistics":{"min":1270.00,"avg":6181.60,"max":11411.00},"data":[1270.00,6125.00,11411.00,6207.00,5895.00]}]}},"nfs4_write":{"node":{"unit":{"base":"B/s","prefix":"M"},"timestep":60,"series":[{"hostname":"f0720","statistics":{"min":11.00,"avg":22.40,"max":29.00},"data":[16.00,27.00,11.00,29.00,29.00]}]}},"vectorization_ratio":{"core":{"unit":{"base":"%"},"timestep":60,"series":[{"hostname":"f0720","id":"7","statistics":{"min":0.00,"avg":77.30,"max":98.75},"data":[90.39,0.00,98.65,98.70,98.75]},{"hostname":"f0720","id":"14","statistics":{"min":0.00,"avg":77.42,"max":98.93},"data":[90.40,0.00,98.85,98.90,98.93]},{"hostname":"f0720","id":"33","statistics":{"min":0.00,"avg":77.40,"max":98.91},"data":[90.36,0.00,98.88,98.88,98.91]},{"hostname":"f0720","id":"18","statistics":{"min":0.00,"avg":77.44,"max":98.96},"data":[90.45,0.00,98.89,98.91,98.96]},{"hostname":"f0720","id":"42","statistics":{"min":0.00,"avg":77.31,"max":98.78},"data":[90.40,0.00,98.65,98.74,98.78]},{"hostname":"f0720","id":"58","statistics":{"min":0.00,"avg":77.44,"max":98.98},"data":[90.42,0.00,98.89,98.93,98.98]},{"hostname":"f0720","id":"0","statistics":{"min":0.00,"avg":75.67,"max":96.17},"data":[90.25,0.00,95.81,96.11,96.17]},{"hostname":"f0720","id":"9","statistics":{"min":0.00,"avg":77.32,"max":98.76},"data":[90.44,0.00,98.76,98.66,98.72]},{"hostname":"f0720","id":"54","statistics":{"min":0.00,"avg":77.44,"max":98.94},"data":[90.47,0.00,98.88,98.92,98.94]},{"hostname":"f0720","id":"3","statistics":{"min":0.00,"avg":77.20,"max":98.60},"data":[90.32,0.00,98.53,98.54,98.60]},{"hostname":"f0720","id":"5","statistics":{"min":0.00,"avg":77.22,"max":98.67},"data":[90.30,0.00,98.67,98.53,98.59]},{"hostname":"f0720","id":"34","statistics":{"min":0.00,"avg":77.30,"max":98.90},"data":[90.07,0.00,98.73,98.79,98.90]},{"hostname":"f0720","id":"41","statistics":{"min":0.00,"avg":77.24,"max":98.71},"data":[90.29,0.00,98.71,98.57,98.62]},{"hostname":"f0720","id":"49","statistics":{"min":0.00,"avg":77.38,"max":98.87},"data":[90.36,0.00,98.83,98.83,98.87]},{"hostname":"f0720","id":"2","statistics":{"min":0.00,"avg":77.22,"max":98.68},"data":[90.32,0.00,98.54,98.56,98.68]},{"hostname":"f0720","id":"17","statistics":{"min":0.00,"avg":77.43,"max":98.92},"data":[90.48,0.00,98.89,98.86,98.92]},{"hostname":"f0720","id":"28","statistics":{"min":0.00,"avg":77.43,"max":98.95},"data":[90.41,0.00,98.84,98.93,98.95]},{"hostname":"f0720","id":"63","statistics":{"min":0.00,"avg":77.39,"max":98.96},"data":[90.29,0.00,98.83,98.88,98.96]},{"hostname":"f0720","id":"52","statistics":{"min":0.00,"avg":77.42,"max":98.91},"data":[90.46,0.00,98.84,98.88,98.91]},{"hostname":"f0720","id":"65","statistics":{"min":0.00,"avg":77.44,"max":99.00},"data":[90.40,0.00,98.88,98.93,99.00]},{"hostname":"f0720","id":"10","statistics":{"min":0.00,"avg":77.36,"max":98.83},"data":[90.36,0.00,98.83,98.79,98.83]},{"hostname":"f0720","id":"31","statistics":{"min":0.00,"avg":77.39,"max":98.89},"data":[90.35,0.00,98.88,98.84,98.89]},{"hostname":"f0720","id":"36","statistics":{"min":0.00,"avg":77.27,"max":98.72},"data":[90.34,0.00,98.62,98.67,98.72]},{"hostname":"f0720","id":"38","statistics":{"min":0.00,"avg":77.25,"max":98.68},"data":[90.35,0.00,98.58,98.63,98.68]},{"hostname":"f0720","id":"46","statistics":{"min":0.00,"avg":77.35,"max":98.82},"data":[90.36,0.00,98.82,98.77,98.81]},{"hostname":"f0720","id":"11","statistics":{"min":0.00,"avg":77.35,"max":98.85},"data":[90.37,0.00,98.85,98.75,98.80]},{"hostname":"f0720","id":"12","statistics":{"min":0.00,"avg":77.40,"max":98.92},"data":[90.38,0.00,98.84,98.89,98.92]},{"hostname":"f0720","id":"15","statistics":{"min":0.00,"avg":77.39,"max":98.89},"data":[90.37,0.00,98.85,98.85,98.89]},{"hostname":"f0720","id":"59","statistics":{"min":0.00,"avg":77.45,"max":98.99},"data":[90.40,0.00,98.93,98.93,98.99]},{"hostname":"f0720","id":"57","statistics":{"min":0.00,"avg":77.43,"max":98.92},"data":[90.45,0.00,98.88,98.88,98.92]},{"hostname":"f0720","id":"6","statistics":{"min":0.00,"avg":77.31,"max":98.79},"data":[90.38,0.00,98.65,98.75,98.79]},{"hostname":"f0720","id":"13","statistics":{"min":0.00,"avg":77.39,"max":98.89},"data":[90.40,0.00,98.84,98.84,98.89]},{"hostname":"f0720","id":"25","statistics":{"min":0.00,"avg":77.42,"max":98.93},"data":[90.39,0.00,98.89,98.89,98.93]},{"hostname":"f0720","id":"30","statistics":{"min":0.00,"avg":77.40,"max":98.94},"data":[90.30,0.00,98.84,98.90,98.94]},{"hostname":"f0720","id":"32","statistics":{"min":0.00,"avg":77.45,"max":98.99},"data":[90.39,0.00,98.93,98.93,98.99]},{"hostname":"f0720","id":"66","statistics":{"min":0.00,"avg":77.40,"max":98.97},"data":[90.33,0.00,98.82,98.89,98.97]},{"hostname":"f0720","id":"71","statistics":{"min":0.00,"avg":77.34,"max":99.05},"data":[89.93,0.00,98.91,98.81,99.05]},{"hostname":"f0720","id":"40","statistics":{"min":0.00,"avg":77.24,"max":98.66},"data":[90.32,0.00,98.66,98.57,98.63]},{"hostname":"f0720","id":"43","statistics":{"min":0.00,"avg":77.29,"max":98.74},"data":[90.36,0.00,98.65,98.70,98.74]},{"hostname":"f0720","id":"47","statistics":{"min":0.00,"avg":77.36,"max":98.88},"data":[90.33,0.00,98.88,98.78,98.81]},{"hostname":"f0720","id":"56","statistics":{"min":0.00,"avg":77.48,"max":99.00},"data":[90.50,0.00,98.93,98.97,99.00]},{"hostname":"f0720","id":"60","statistics":{"min":0.00,"avg":77.44,"max":98.99},"data":[90.39,0.00,98.88,98.93,98.99]},{"hostname":"f0720","id":"22","statistics":{"min":0.00,"avg":77.45,"max":98.96},"data":[90.42,0.00,98.94,98.94,98.96]},{"hostname":"f0720","id":"29","statistics":{"min":0.00,"avg":77.45,"max":98.97},"data":[90.44,0.00,98.90,98.93,98.97]},{"hostname":"f0720","id":"35","statistics":{"min":0.00,"avg":77.33,"max":98.88},"data":[90.09,0.00,98.88,98.82,98.85]},{"hostname":"f0720","id":"70","statistics":{"min":0.00,"avg":77.34,"max":99.02},"data":[90.09,0.00,98.75,98.86,99.02]},{"hostname":"f0720","id":"61","statistics":{"min":0.00,"avg":77.41,"max":98.95},"data":[90.35,0.00,98.87,98.88,98.96]},{"hostname":"f0720","id":"64","statistics":{"min":0.00,"avg":77.43,"max":98.99},"data":[90.44,0.00,98.83,98.91,98.99]},{"hostname":"f0720","id":"68","statistics":{"min":0.00,"avg":77.48,"max":99.12},"data":[90.40,0.00,98.92,98.96,99.12]},{"hostname":"f0720","id":"8","statistics":{"min":0.00,"avg":77.35,"max":98.80},"data":[90.45,0.00,98.76,98.72,98.80]},{"hostname":"f0720","id":"16","statistics":{"min":0.00,"avg":77.42,"max":98.92},"data":[90.46,0.00,98.85,98.89,98.92]},{"hostname":"f0720","id":"20","statistics":{"min":0.00,"avg":77.49,"max":99.00},"data":[90.50,0.00,98.95,98.98,99.00]},{"hostname":"f0720","id":"27","statistics":{"min":0.00,"avg":77.40,"max":98.92},"data":[90.31,0.00,98.87,98.89,98.92]},{"hostname":"f0720","id":"44","statistics":{"min":0.00,"avg":77.36,"max":98.80},"data":[90.45,0.00,98.80,98.75,98.79]},{"hostname":"f0720","id":"51","statistics":{"min":0.00,"avg":77.37,"max":98.86},"data":[90.33,0.00,98.83,98.83,98.86]},{"hostname":"f0720","id":"55","statistics":{"min":0.00,"avg":77.42,"max":98.91},"data":[90.44,0.00,98.88,98.88,98.91]},{"hostname":"f0720","id":"62","statistics":{"min":0.00,"avg":77.46,"max":99.05},"data":[90.35,0.00,98.92,98.98,99.05]},{"hostname":"f0720","id":"4","statistics":{"min":0.00,"avg":77.24,"max":98.67},"data":[90.28,0.00,98.67,98.59,98.64]},{"hostname":"f0720","id":"21","statistics":{"min":0.00,"avg":77.44,"max":98.92},"data":[90.47,0.00,98.90,98.89,98.92]},{"hostname":"f0720","id":"26","statistics":{"min":0.00,"avg":77.46,"max":99.01},"data":[90.35,0.00,98.95,98.99,99.01]},{"hostname":"f0720","id":"53","statistics":{"min":0.00,"avg":77.44,"max":98.94},"data":[90.45,0.00,98.91,98.91,98.93]},{"hostname":"f0720","id":"1","statistics":{"min":0.00,"avg":77.25,"max":98.68},"data":[90.33,0.00,98.62,98.62,98.68]},{"hostname":"f0720","id":"23","statistics":{"min":0.00,"avg":77.46,"max":98.97},"data":[90.44,0.00,98.95,98.94,98.97]},{"hostname":"f0720","id":"24","statistics":{"min":0.00,"avg":77.43,"max":99.00},"data":[90.36,0.00,98.90,98.91,99.00]},{"hostname":"f0720","id":"37","statistics":{"min":0.00,"avg":77.24,"max":98.66},"data":[90.32,0.00,98.61,98.61,98.66]},{"hostname":"f0720","id":"69","statistics":{"min":0.00,"avg":77.43,"max":99.05},"data":[90.38,0.00,98.86,98.87,99.05]},{"hostname":"f0720","id":"67","statistics":{"min":0.00,"avg":77.39,"max":98.93},"data":[90.34,0.00,98.86,98.83,98.93]},{"hostname":"f0720","id":"19","statistics":{"min":0.00,"avg":77.44,"max":98.93},"data":[90.49,0.00,98.90,98.89,98.93]},{"hostname":"f0720","id":"39","statistics":{"min":0.00,"avg":77.18,"max":98.57},"data":[90.30,0.00,98.52,98.52,98.57]},{"hostname":"f0720","id":"45","statistics":{"min":0.00,"avg":77.30,"max":98.74},"data":[90.39,0.00,98.74,98.65,98.69]},{"hostname":"f0720","id":"48","statistics":{"min":0.00,"avg":77.40,"max":98.90},"data":[90.38,0.00,98.82,98.87,98.90]},{"hostname":"f0720","id":"50","statistics":{"min":0.00,"avg":77.43,"max":98.95},"data":[90.40,0.00,98.88,98.92,98.95]}]},"node":{"unit":{"base":"%"},"timestep":60,"series":[{"hostname":"f0720","id":"0","statistics":{"min":0.00,"avg":77.35,"max":98.84},"data":[90.36,0.00,98.77,98.78,98.84]}]}}} diff --git a/internal/importer/testdata/data-fritzMinimal.json b/internal/importer/testdata/data-fritzMinimal.json new file mode 100644 index 0000000..274d913 --- /dev/null +++ b/internal/importer/testdata/data-fritzMinimal.json @@ -0,0 +1 @@ +{"clock":{"core":{"unit":{"base":"Hz","prefix":"M"},"timestep":60,"series":[{"hostname":"f0649","id":"2","statistics":{"min":848.99,"avg":1404.20,"max":2379.15},"data":[984.46,848.99,2379.15]},{"hostname":"f0649","id":"18","statistics":{"min":799.96,"avg":1317.56,"max":2352.70},"data":[799.96,800.02,2352.70]},{"hostname":"f0649","id":"35","statistics":{"min":795.95,"avg":1316.09,"max":2340.68},"data":[811.66,795.95,2340.68]},{"hostname":"f0649","id":"41","statistics":{"min":800.05,"avg":1325.93,"max":2371.22},"data":[800.05,806.52,2371.22]},{"hostname":"f0649","id":"52","statistics":{"min":796.73,"avg":1318.78,"max":2356.10},"data":[796.73,803.53,2356.10]},{"hostname":"f0649","id":"66","statistics":{"min":800.00,"avg":1314.24,"max":2338.54},"data":[800.00,804.19,2338.54]},{"hostname":"f0649","id":"70","statistics":{"min":797.01,"avg":1320.34,"max":2345.79},"data":[797.01,818.21,2345.79]},{"hostname":"f0649","id":"27","statistics":{"min":793.24,"avg":1305.52,"max":2326.51},"data":[796.82,793.24,2326.51]},{"hostname":"f0649","id":"60","statistics":{"min":800.30,"avg":1312.09,"max":2333.71},"data":[802.26,800.30,2333.71]},{"hostname":"f0649","id":"12","statistics":{"min":794.07,"avg":1322.57,"max":2366.25},"data":[807.40,794.07,2366.25]},{"hostname":"f0649","id":"31","statistics":{"min":792.63,"avg":1314.22,"max":2348.01},"data":[802.03,792.63,2348.01]},{"hostname":"f0649","id":"38","statistics":{"min":795.07,"avg":1317.41,"max":2357.11},"data":[800.04,795.07,2357.11]},{"hostname":"f0649","id":"40","statistics":{"min":789.77,"avg":1318.87,"max":2367.34},"data":[799.50,789.77,2367.34]},{"hostname":"f0649","id":"49","statistics":{"min":806.68,"avg":1323.39,"max":2351.08},"data":[806.68,812.42,2351.08]},{"hostname":"f0649","id":"51","statistics":{"min":800.07,"avg":1325.33,"max":2365.61},"data":[800.07,810.30,2365.61]},{"hostname":"f0649","id":"64","statistics":{"min":792.24,"avg":1310.03,"max":2336.04},"data":[801.81,792.24,2336.04]},{"hostname":"f0649","id":"0","statistics":{"min":800.02,"avg":1435.94,"max":2333.66},"data":[1174.13,800.02,2333.67]},{"hostname":"f0649","id":"19","statistics":{"min":800.08,"avg":1315.41,"max":2344.30},"data":[801.86,800.08,2344.30]},{"hostname":"f0649","id":"26","statistics":{"min":790.09,"avg":1307.20,"max":2338.68},"data":[792.84,790.09,2338.68]},{"hostname":"f0649","id":"42","statistics":{"min":799.79,"avg":1324.62,"max":2367.81},"data":[799.79,806.25,2367.81]},{"hostname":"f0649","id":"54","statistics":{"min":796.96,"avg":1498.83,"max":2343.97},"data":[1355.56,796.96,2343.97]},{"hostname":"f0649","id":"56","statistics":{"min":800.45,"avg":1309.09,"max":2321.74},"data":[805.09,800.45,2321.74]},{"hostname":"f0649","id":"61","statistics":{"min":800.80,"avg":1311.69,"max":2320.98},"data":[813.29,800.80,2320.98]},{"hostname":"f0649","id":"8","statistics":{"min":800.04,"avg":1318.02,"max":2353.71},"data":[800.04,800.32,2353.71]},{"hostname":"f0649","id":"20","statistics":{"min":799.08,"avg":1316.78,"max":2346.36},"data":[799.08,804.89,2346.36]},{"hostname":"f0649","id":"29","statistics":{"min":789.54,"avg":1303.74,"max":2325.87},"data":[789.54,795.81,2325.87]},{"hostname":"f0649","id":"45","statistics":{"min":786.97,"avg":1317.63,"max":2365.81},"data":[800.12,786.97,2365.81]},{"hostname":"f0649","id":"28","statistics":{"min":800.79,"avg":1316.67,"max":2346.94},"data":[800.79,802.30,2346.94]},{"hostname":"f0649","id":"39","statistics":{"min":800.14,"avg":1324.61,"max":2373.41},"data":[800.14,800.27,2373.42]},{"hostname":"f0649","id":"59","statistics":{"min":798.34,"avg":1325.84,"max":2374.19},"data":[804.97,798.34,2374.19]},{"hostname":"f0649","id":"69","statistics":{"min":793.75,"avg":1304.64,"max":2323.16},"data":[793.75,797.01,2323.16]},{"hostname":"f0649","id":"10","statistics":{"min":798.60,"avg":1324.07,"max":2373.38},"data":[800.22,798.60,2373.38]},{"hostname":"f0649","id":"11","statistics":{"min":800.06,"avg":1322.47,"max":2367.16},"data":[800.06,800.19,2367.16]},{"hostname":"f0649","id":"24","statistics":{"min":794.78,"avg":1308.30,"max":2331.31},"data":[798.79,794.78,2331.31]},{"hostname":"f0649","id":"25","statistics":{"min":798.33,"avg":1314.69,"max":2339.58},"data":[798.33,806.17,2339.58]},{"hostname":"f0649","id":"32","statistics":{"min":798.58,"avg":1305.99,"max":2319.89},"data":[799.49,798.58,2319.89]},{"hostname":"f0649","id":"13","statistics":{"min":798.46,"avg":1322.82,"max":2361.46},"data":[798.46,808.54,2361.46]},{"hostname":"f0649","id":"36","statistics":{"min":798.02,"avg":1317.44,"max":2354.39},"data":[799.92,798.02,2354.39]},{"hostname":"f0649","id":"65","statistics":{"min":788.53,"avg":1308.93,"max":2331.22},"data":[788.53,807.03,2331.22]},{"hostname":"f0649","id":"68","statistics":{"min":796.15,"avg":1311.66,"max":2341.13},"data":[796.15,797.70,2341.13]},{"hostname":"f0649","id":"5","statistics":{"min":803.66,"avg":1550.73,"max":2365.51},"data":[1483.01,803.66,2365.51]},{"hostname":"f0649","id":"21","statistics":{"min":795.12,"avg":1305.85,"max":2320.76},"data":[795.12,801.66,2320.76]},{"hostname":"f0649","id":"48","statistics":{"min":806.52,"avg":1550.06,"max":2362.73},"data":[1480.91,806.52,2362.73]},{"hostname":"f0649","id":"9","statistics":{"min":796.45,"avg":1323.81,"max":2374.97},"data":[799.99,796.45,2374.98]},{"hostname":"f0649","id":"34","statistics":{"min":792.18,"avg":1307.37,"max":2331.05},"data":[792.18,798.89,2331.05]},{"hostname":"f0649","id":"50","statistics":{"min":799.85,"avg":1312.79,"max":2338.49},"data":[800.02,799.85,2338.49]},{"hostname":"f0649","id":"58","statistics":{"min":792.13,"avg":1307.47,"max":2320.42},"data":[792.13,809.85,2320.42]},{"hostname":"f0649","id":"71","statistics":{"min":799.37,"avg":1315.14,"max":2346.21},"data":[799.83,799.37,2346.21]},{"hostname":"f0649","id":"4","statistics":{"min":798.95,"avg":1323.37,"max":2371.13},"data":[798.95,800.02,2371.13]},{"hostname":"f0649","id":"16","statistics":{"min":794.25,"avg":1317.41,"max":2358.01},"data":[799.98,794.26,2358.01]},{"hostname":"f0649","id":"55","statistics":{"min":801.05,"avg":1314.97,"max":2341.42},"data":[802.44,801.05,2341.42]},{"hostname":"f0649","id":"62","statistics":{"min":797.44,"avg":1314.25,"max":2338.89},"data":[797.44,806.41,2338.89]},{"hostname":"f0649","id":"33","statistics":{"min":790.42,"avg":1310.46,"max":2337.30},"data":[803.66,790.42,2337.30]},{"hostname":"f0649","id":"37","statistics":{"min":799.75,"avg":1321.82,"max":2365.68},"data":[800.03,799.75,2365.69]},{"hostname":"f0649","id":"43","statistics":{"min":799.97,"avg":1323.61,"max":2360.84},"data":[799.97,810.03,2360.84]},{"hostname":"f0649","id":"44","statistics":{"min":799.26,"avg":1533.13,"max":2364.52},"data":[1435.61,799.26,2364.52]},{"hostname":"f0649","id":"15","statistics":{"min":799.93,"avg":1311.86,"max":2319.66},"data":[799.93,815.98,2319.66]},{"hostname":"f0649","id":"17","statistics":{"min":799.99,"avg":1315.49,"max":2333.24},"data":[799.99,813.24,2333.24]},{"hostname":"f0649","id":"53","statistics":{"min":795.63,"avg":1311.91,"max":2335.33},"data":[795.63,804.76,2335.33]},{"hostname":"f0649","id":"67","statistics":{"min":793.06,"avg":1314.48,"max":2334.71},"data":[815.67,793.06,2334.71]},{"hostname":"f0649","id":"1","statistics":{"min":800.26,"avg":1328.95,"max":2364.99},"data":[821.60,800.27,2365.00]},{"hostname":"f0649","id":"3","statistics":{"min":799.92,"avg":1319.91,"max":2359.79},"data":[800.02,799.92,2359.79]},{"hostname":"f0649","id":"6","statistics":{"min":800.98,"avg":1714.23,"max":2372.89},"data":[1968.81,800.98,2372.89]},{"hostname":"f0649","id":"14","statistics":{"min":800.00,"avg":1322.61,"max":2364.63},"data":[800.00,803.21,2364.63]},{"hostname":"f0649","id":"23","statistics":{"min":793.87,"avg":1310.45,"max":2330.73},"data":[793.86,806.76,2330.73]},{"hostname":"f0649","id":"46","statistics":{"min":799.97,"avg":1320.61,"max":2360.31},"data":[799.97,801.54,2360.31]},{"hostname":"f0649","id":"63","statistics":{"min":797.74,"avg":1310.53,"max":2332.33},"data":[801.51,797.74,2332.33]},{"hostname":"f0649","id":"7","statistics":{"min":796.17,"avg":1318.98,"max":2354.76},"data":[796.17,806.02,2354.75]},{"hostname":"f0649","id":"22","statistics":{"min":799.97,"avg":1319.51,"max":2358.58},"data":[799.98,799.97,2358.58]},{"hostname":"f0649","id":"30","statistics":{"min":800.60,"avg":1305.86,"max":2313.18},"data":[803.81,800.59,2313.18]},{"hostname":"f0649","id":"47","statistics":{"min":799.98,"avg":1322.12,"max":2364.79},"data":[799.98,801.58,2364.79]},{"hostname":"f0649","id":"57","statistics":{"min":801.93,"avg":1316.00,"max":2337.69},"data":[808.39,801.93,2337.69]}]},"node":{"unit":{"base":"Hz","prefix":"M"},"timestep":60,"series":[{"hostname":"f0649","id":"0","statistics":{"min":801.56,"avg":1336.52,"max":2348.22},"data":[859.78,801.56,2348.22]}]}},"cpu_load":{"node":{"unit":{"base":""},"timestep":60,"series":[{"hostname":"f0649","statistics":{"min":17.36,"avg":31.64,"max":45.54},"data":[45.54,17.36,32.02]}]}},"cpu_power":{"node":{"unit":{"base":"W"},"timestep":60,"series":[{"hostname":"f0649","id":"0","statistics":{"min":93.67,"avg":150.02,"max":261.59},"data":[94.79,93.67,261.59]}]},"socket":{"unit":{"base":"W"},"timestep":60,"series":[{"hostname":"f0649","id":"0","statistics":{"min":47.95,"avg":75.78,"max":130.98},"data":[48.41,47.96,130.98]},{"hostname":"f0649","id":"0","statistics":{"min":45.72,"avg":74.23,"max":130.61},"data":[46.38,45.72,130.61]}]}},"cpu_user":{"core":{"unit":{"base":""},"timestep":60,"series":[{"hostname":"f0649","id":"42","statistics":{"min":0.03,"avg":28.54,"max":57.40},"data":[57.40,0.03,28.18]},{"hostname":"f0649","id":"47","statistics":{"min":0.03,"avg":28.49,"max":57.29},"data":[57.29,0.03,28.15]},{"hostname":"f0649","id":"49","statistics":{"min":0.03,"avg":28.43,"max":57.24},"data":[57.24,0.03,28.03]},{"hostname":"f0649","id":"31","statistics":{"min":0.03,"avg":28.41,"max":57.15},"data":[57.15,0.03,28.05]},{"hostname":"f0649","id":"21","statistics":{"min":0.02,"avg":28.39,"max":57.19},"data":[57.19,0.02,27.97]},{"hostname":"f0649","id":"30","statistics":{"min":0.03,"avg":28.55,"max":57.47},"data":[57.47,0.03,28.13]},{"hostname":"f0649","id":"56","statistics":{"min":0.03,"avg":28.21,"max":57.32},"data":[57.32,0.03,27.27]},{"hostname":"f0649","id":"64","statistics":{"min":0.02,"avg":28.46,"max":57.32},"data":[57.32,0.02,28.05]},{"hostname":"f0649","id":"11","statistics":{"min":0.03,"avg":28.50,"max":57.35},"data":[57.35,0.03,28.10]},{"hostname":"f0649","id":"18","statistics":{"min":0.05,"avg":28.41,"max":57.20},"data":[57.20,0.05,27.98]},{"hostname":"f0649","id":"48","statistics":{"min":0.07,"avg":28.48,"max":57.42},"data":[57.42,0.07,27.97]},{"hostname":"f0649","id":"6","statistics":{"min":0.20,"avg":28.58,"max":57.52},"data":[57.52,0.20,28.02]},{"hostname":"f0649","id":"37","statistics":{"min":0.08,"avg":28.59,"max":57.50},"data":[57.50,0.08,28.20]},{"hostname":"f0649","id":"46","statistics":{"min":0.05,"avg":28.41,"max":57.42},"data":[57.42,0.05,27.75]},{"hostname":"f0649","id":"60","statistics":{"min":0.03,"avg":28.33,"max":57.17},"data":[57.17,0.03,27.80]},{"hostname":"f0649","id":"62","statistics":{"min":0.03,"avg":28.47,"max":57.35},"data":[57.35,0.03,28.02]},{"hostname":"f0649","id":"71","statistics":{"min":0.12,"avg":28.55,"max":57.40},"data":[57.40,0.12,28.12]},{"hostname":"f0649","id":"27","statistics":{"min":0.03,"avg":28.42,"max":57.25},"data":[57.25,0.03,27.97]},{"hostname":"f0649","id":"14","statistics":{"min":0.37,"avg":28.61,"max":57.27},"data":[57.27,0.37,28.20]},{"hostname":"f0649","id":"35","statistics":{"min":0.03,"avg":28.37,"max":57.09},"data":[57.09,0.03,27.98]},{"hostname":"f0649","id":"36","statistics":{"min":0.03,"avg":28.61,"max":57.64},"data":[57.64,0.03,28.15]},{"hostname":"f0649","id":"41","statistics":{"min":0.03,"avg":28.66,"max":57.70},"data":[57.70,0.03,28.23]},{"hostname":"f0649","id":"57","statistics":{"min":0.03,"avg":28.41,"max":57.32},"data":[57.32,0.03,27.88]},{"hostname":"f0649","id":"59","statistics":{"min":0.03,"avg":28.37,"max":57.20},"data":[57.20,0.03,27.88]},{"hostname":"f0649","id":"8","statistics":{"min":0.07,"avg":28.55,"max":57.42},"data":[57.42,0.07,28.15]},{"hostname":"f0649","id":"12","statistics":{"min":0.05,"avg":29.23,"max":58.84},"data":[58.84,0.05,28.80]},{"hostname":"f0649","id":"16","statistics":{"min":0.07,"avg":28.54,"max":57.49},"data":[57.49,0.07,28.07]},{"hostname":"f0649","id":"23","statistics":{"min":0.02,"avg":28.31,"max":57.02},"data":[57.02,0.02,27.90]},{"hostname":"f0649","id":"43","statistics":{"min":0.07,"avg":28.60,"max":57.62},"data":[57.62,0.07,28.12]},{"hostname":"f0649","id":"68","statistics":{"min":0.02,"avg":28.34,"max":57.02},"data":[57.02,0.02,27.98]},{"hostname":"f0649","id":"70","statistics":{"min":0.03,"avg":28.54,"max":57.54},"data":[57.54,0.03,28.05]},{"hostname":"f0649","id":"5","statistics":{"min":0.47,"avg":28.66,"max":57.10},"data":[57.10,0.47,28.42]},{"hostname":"f0649","id":"20","statistics":{"min":0.02,"avg":28.32,"max":57.05},"data":[57.05,0.02,27.88]},{"hostname":"f0649","id":"39","statistics":{"min":0.03,"avg":28.47,"max":57.25},"data":[57.25,0.03,28.13]},{"hostname":"f0649","id":"54","statistics":{"min":0.10,"avg":28.48,"max":57.27},"data":[57.27,0.10,28.08]},{"hostname":"f0649","id":"2","statistics":{"min":1.57,"avg":29.20,"max":57.04},"data":[57.04,1.57,28.98]},{"hostname":"f0649","id":"65","statistics":{"min":0.03,"avg":28.48,"max":57.34},"data":[57.34,0.03,28.07]},{"hostname":"f0649","id":"66","statistics":{"min":0.03,"avg":28.42,"max":57.42},"data":[57.42,0.03,27.80]},{"hostname":"f0649","id":"69","statistics":{"min":0.03,"avg":28.48,"max":57.22},"data":[57.22,0.03,28.18]},{"hostname":"f0649","id":"9","statistics":{"min":0.05,"avg":28.70,"max":57.94},"data":[57.94,0.05,28.10]},{"hostname":"f0649","id":"4","statistics":{"min":0.08,"avg":28.77,"max":57.45},"data":[57.45,0.08,28.77]},{"hostname":"f0649","id":"10","statistics":{"min":0.03,"avg":28.64,"max":57.32},"data":[57.32,0.03,28.58]},{"hostname":"f0649","id":"17","statistics":{"min":0.73,"avg":28.75,"max":57.10},"data":[57.10,0.73,28.40]},{"hostname":"f0649","id":"25","statistics":{"min":0.02,"avg":28.48,"max":57.34},"data":[57.34,0.02,28.08]},{"hostname":"f0649","id":"26","statistics":{"min":0.03,"avg":28.53,"max":57.45},"data":[57.45,0.03,28.10]},{"hostname":"f0649","id":"38","statistics":{"min":0.03,"avg":28.45,"max":57.32},"data":[57.32,0.03,28.00]},{"hostname":"f0649","id":"40","statistics":{"min":0.03,"avg":28.42,"max":57.25},"data":[57.25,0.03,27.97]},{"hostname":"f0649","id":"1","statistics":{"min":0.17,"avg":28.88,"max":57.35},"data":[57.35,0.17,29.13]},{"hostname":"f0649","id":"52","statistics":{"min":0.03,"avg":28.54,"max":57.40},"data":[57.40,0.03,28.18]},{"hostname":"f0649","id":"24","statistics":{"min":0.03,"avg":28.41,"max":57.17},"data":[57.17,0.03,28.02]},{"hostname":"f0649","id":"51","statistics":{"min":0.03,"avg":28.48,"max":57.39},"data":[57.39,0.03,28.02]},{"hostname":"f0649","id":"58","statistics":{"min":0.03,"avg":28.41,"max":57.19},"data":[57.19,0.03,28.02]},{"hostname":"f0649","id":"22","statistics":{"min":0.03,"avg":28.47,"max":57.40},"data":[57.40,0.03,27.97]},{"hostname":"f0649","id":"13","statistics":{"min":0.05,"avg":28.59,"max":57.35},"data":[57.35,0.05,28.37]},{"hostname":"f0649","id":"15","statistics":{"min":0.10,"avg":28.64,"max":57.70},"data":[57.70,0.10,28.13]},{"hostname":"f0649","id":"3","statistics":{"min":0.10,"avg":28.82,"max":57.22},"data":[57.22,0.10,29.15]},{"hostname":"f0649","id":"29","statistics":{"min":0.02,"avg":28.51,"max":57.30},"data":[57.30,0.02,28.20]},{"hostname":"f0649","id":"7","statistics":{"min":0.03,"avg":28.68,"max":57.17},"data":[57.17,0.03,28.85]},{"hostname":"f0649","id":"61","statistics":{"min":0.03,"avg":28.45,"max":57.37},"data":[57.37,0.03,27.93]},{"hostname":"f0649","id":"63","statistics":{"min":0.03,"avg":28.46,"max":57.27},"data":[57.27,0.03,28.08]},{"hostname":"f0649","id":"32","statistics":{"min":0.03,"avg":28.55,"max":57.39},"data":[57.39,0.03,28.23]},{"hostname":"f0649","id":"33","statistics":{"min":0.03,"avg":28.46,"max":57.27},"data":[57.27,0.03,28.08]},{"hostname":"f0649","id":"50","statistics":{"min":0.05,"avg":28.53,"max":57.45},"data":[57.45,0.05,28.10]},{"hostname":"f0649","id":"28","statistics":{"min":0.03,"avg":28.49,"max":57.35},"data":[57.35,0.03,28.08]},{"hostname":"f0649","id":"34","statistics":{"min":0.02,"avg":28.47,"max":57.32},"data":[57.32,0.02,28.08]},{"hostname":"f0649","id":"44","statistics":{"min":0.08,"avg":28.57,"max":57.44},"data":[57.44,0.08,28.18]},{"hostname":"f0649","id":"45","statistics":{"min":0.05,"avg":28.53,"max":57.40},"data":[57.40,0.05,28.15]},{"hostname":"f0649","id":"0","statistics":{"min":0.23,"avg":27.79,"max":56.42},"data":[56.42,0.23,26.72]},{"hostname":"f0649","id":"53","statistics":{"min":0.03,"avg":28.58,"max":57.42},"data":[57.42,0.03,28.30]},{"hostname":"f0649","id":"55","statistics":{"min":0.07,"avg":28.51,"max":57.40},"data":[57.40,0.07,28.07]},{"hostname":"f0649","id":"67","statistics":{"min":0.02,"avg":28.42,"max":57.32},"data":[57.32,0.02,27.93]},{"hostname":"f0649","id":"19","statistics":{"min":0.03,"avg":28.45,"max":57.24},"data":[57.24,0.03,28.08]}]},"node":{"unit":{"base":""},"timestep":60,"series":[{"hostname":"f0649","id":"0","statistics":{"min":0.09,"avg":28.52,"max":57.34},"data":[57.34,0.09,28.12]}]}},"flops_any":{"core":{"unit":{"base":"F/s","prefix":"G"},"timestep":60,"series":[{"hostname":"f0649","id":"9","statistics":{"min":0.00,"avg":0.61,"max":1.83},"data":[0.00,0.00,1.83]},{"hostname":"f0649","id":"30","statistics":{"min":0.00,"avg":0.62,"max":1.86},"data":[0.00,0.00,1.86]},{"hostname":"f0649","id":"45","statistics":{"min":0.00,"avg":0.62,"max":1.86},"data":[0.00,0.00,1.86]},{"hostname":"f0649","id":"50","statistics":{"min":0.00,"avg":0.61,"max":1.83},"data":[0.00,0.00,1.83]},{"hostname":"f0649","id":"12","statistics":{"min":0.00,"avg":0.62,"max":1.86},"data":[0.00,0.00,1.86]},{"hostname":"f0649","id":"53","statistics":{"min":0.00,"avg":0.62,"max":1.85},"data":[0.00,0.00,1.85]},{"hostname":"f0649","id":"58","statistics":{"min":0.00,"avg":0.61,"max":1.83},"data":[0.00,0.00,1.83]},{"hostname":"f0649","id":"63","statistics":{"min":0.00,"avg":0.62,"max":1.85},"data":[0.00,0.00,1.85]},{"hostname":"f0649","id":"0","statistics":{"min":0.00,"avg":1.78,"max":5.36},"data":[0.00,0.00,5.36]},{"hostname":"f0649","id":"21","statistics":{"min":0.00,"avg":0.61,"max":1.82},"data":[0.00,0.00,1.82]},{"hostname":"f0649","id":"23","statistics":{"min":0.00,"avg":0.61,"max":1.82},"data":[0.00,0.00,1.82]},{"hostname":"f0649","id":"49","statistics":{"min":0.00,"avg":0.62,"max":1.86},"data":[0.00,0.00,1.86]},{"hostname":"f0649","id":"51","statistics":{"min":0.00,"avg":0.62,"max":1.86},"data":[0.00,0.00,1.86]},{"hostname":"f0649","id":"32","statistics":{"min":0.00,"avg":0.62,"max":1.86},"data":[0.00,0.00,1.86]},{"hostname":"f0649","id":"46","statistics":{"min":0.00,"avg":0.61,"max":1.83},"data":[0.00,0.00,1.83]},{"hostname":"f0649","id":"48","statistics":{"min":0.00,"avg":0.61,"max":1.83},"data":[0.00,0.00,1.83]},{"hostname":"f0649","id":"55","statistics":{"min":0.00,"avg":0.62,"max":1.85},"data":[0.00,0.00,1.85]},{"hostname":"f0649","id":"69","statistics":{"min":0.00,"avg":0.62,"max":1.85},"data":[0.00,0.00,1.85]},{"hostname":"f0649","id":"18","statistics":{"min":0.00,"avg":0.62,"max":1.86},"data":[0.00,0.00,1.86]},{"hostname":"f0649","id":"20","statistics":{"min":0.00,"avg":0.62,"max":1.86},"data":[0.00,0.00,1.86]},{"hostname":"f0649","id":"25","statistics":{"min":0.00,"avg":0.61,"max":1.82},"data":[0.00,0.00,1.82]},{"hostname":"f0649","id":"42","statistics":{"min":0.00,"avg":0.61,"max":1.84},"data":[0.00,0.00,1.84]},{"hostname":"f0649","id":"43","statistics":{"min":0.00,"avg":0.62,"max":1.86},"data":[0.00,0.00,1.86]},{"hostname":"f0649","id":"71","statistics":{"min":0.00,"avg":0.51,"max":1.54},"data":[0.00,0.00,1.54]},{"hostname":"f0649","id":"4","statistics":{"min":0.00,"avg":0.62,"max":1.86},"data":[0.00,0.00,1.86]},{"hostname":"f0649","id":"8","statistics":{"min":0.00,"avg":0.62,"max":1.86},"data":[0.00,0.00,1.86]},{"hostname":"f0649","id":"10","statistics":{"min":0.00,"avg":0.62,"max":1.86},"data":[0.00,0.00,1.86]},{"hostname":"f0649","id":"61","statistics":{"min":0.00,"avg":0.62,"max":1.85},"data":[0.00,0.00,1.85]},{"hostname":"f0649","id":"39","statistics":{"min":0.00,"avg":0.62,"max":1.86},"data":[0.00,0.00,1.86]},{"hostname":"f0649","id":"41","statistics":{"min":0.00,"avg":0.62,"max":1.86},"data":[0.00,0.00,1.86]},{"hostname":"f0649","id":"64","statistics":{"min":0.00,"avg":0.61,"max":1.83},"data":[0.00,0.00,1.83]},{"hostname":"f0649","id":"67","statistics":{"min":0.00,"avg":0.62,"max":1.85},"data":[0.00,0.00,1.85]},{"hostname":"f0649","id":"68","statistics":{"min":0.00,"avg":0.61,"max":1.83},"data":[0.00,0.00,1.83]},{"hostname":"f0649","id":"1","statistics":{"min":0.00,"avg":0.61,"max":1.83},"data":[0.00,0.00,1.83]},{"hostname":"f0649","id":"2","statistics":{"min":0.00,"avg":0.62,"max":1.86},"data":[0.00,0.00,1.86]},{"hostname":"f0649","id":"34","statistics":{"min":0.00,"avg":0.50,"max":1.50},"data":[0.00,0.00,1.50]},{"hostname":"f0649","id":"44","statistics":{"min":0.00,"avg":0.61,"max":1.83},"data":[0.00,0.00,1.83]},{"hostname":"f0649","id":"57","statistics":{"min":0.00,"avg":0.62,"max":1.85},"data":[0.00,0.00,1.85]},{"hostname":"f0649","id":"11","statistics":{"min":0.00,"avg":0.61,"max":1.83},"data":[0.00,0.00,1.83]},{"hostname":"f0649","id":"26","statistics":{"min":0.00,"avg":0.62,"max":1.86},"data":[0.00,0.00,1.86]},{"hostname":"f0649","id":"62","statistics":{"min":0.00,"avg":0.61,"max":1.83},"data":[0.00,0.00,1.83]},{"hostname":"f0649","id":"6","statistics":{"min":0.00,"avg":0.62,"max":1.86},"data":[0.00,0.00,1.86]},{"hostname":"f0649","id":"70","statistics":{"min":0.00,"avg":0.52,"max":1.57},"data":[0.00,0.00,1.57]},{"hostname":"f0649","id":"3","statistics":{"min":0.00,"avg":0.61,"max":1.83},"data":[0.00,0.00,1.83]},{"hostname":"f0649","id":"7","statistics":{"min":0.00,"avg":0.61,"max":1.83},"data":[0.00,0.00,1.83]},{"hostname":"f0649","id":"14","statistics":{"min":0.00,"avg":0.62,"max":1.86},"data":[0.00,0.00,1.86]},{"hostname":"f0649","id":"29","statistics":{"min":0.00,"avg":0.61,"max":1.82},"data":[0.00,0.00,1.82]},{"hostname":"f0649","id":"60","statistics":{"min":0.00,"avg":0.61,"max":1.83},"data":[0.00,0.00,1.83]},{"hostname":"f0649","id":"13","statistics":{"min":0.00,"avg":0.61,"max":1.83},"data":[0.00,0.00,1.83]},{"hostname":"f0649","id":"15","statistics":{"min":0.00,"avg":0.61,"max":1.83},"data":[0.00,0.00,1.83]},{"hostname":"f0649","id":"19","statistics":{"min":0.00,"avg":0.61,"max":1.82},"data":[0.00,0.00,1.82]},{"hostname":"f0649","id":"54","statistics":{"min":0.00,"avg":0.61,"max":1.83},"data":[0.00,0.00,1.83]},{"hostname":"f0649","id":"66","statistics":{"min":0.00,"avg":0.61,"max":1.83},"data":[0.00,0.00,1.83]},{"hostname":"f0649","id":"38","statistics":{"min":0.00,"avg":0.61,"max":1.84},"data":[0.00,0.00,1.84]},{"hostname":"f0649","id":"16","statistics":{"min":0.00,"avg":0.62,"max":1.86},"data":[0.00,0.00,1.86]},{"hostname":"f0649","id":"17","statistics":{"min":0.00,"avg":0.61,"max":1.82},"data":[0.00,0.00,1.82]},{"hostname":"f0649","id":"22","statistics":{"min":0.00,"avg":0.62,"max":1.86},"data":[0.00,0.00,1.86]},{"hostname":"f0649","id":"33","statistics":{"min":0.00,"avg":0.61,"max":1.82},"data":[0.00,0.00,1.82]},{"hostname":"f0649","id":"35","statistics":{"min":0.00,"avg":0.53,"max":1.59},"data":[0.00,0.00,1.59]},{"hostname":"f0649","id":"28","statistics":{"min":0.00,"avg":0.62,"max":1.86},"data":[0.00,0.00,1.86]},{"hostname":"f0649","id":"40","statistics":{"min":0.00,"avg":0.61,"max":1.84},"data":[0.00,0.00,1.84]},{"hostname":"f0649","id":"52","statistics":{"min":0.00,"avg":0.61,"max":1.83},"data":[0.00,0.00,1.83]},{"hostname":"f0649","id":"59","statistics":{"min":0.00,"avg":0.62,"max":1.85},"data":[0.00,0.00,1.85]},{"hostname":"f0649","id":"65","statistics":{"min":0.00,"avg":0.62,"max":1.85},"data":[0.00,0.00,1.85]},{"hostname":"f0649","id":"24","statistics":{"min":0.00,"avg":0.62,"max":1.86},"data":[0.00,0.00,1.86]},{"hostname":"f0649","id":"31","statistics":{"min":0.00,"avg":0.61,"max":1.82},"data":[0.00,0.00,1.82]},{"hostname":"f0649","id":"36","statistics":{"min":0.00,"avg":0.61,"max":1.84},"data":[0.00,0.00,1.84]},{"hostname":"f0649","id":"37","statistics":{"min":0.00,"avg":0.62,"max":1.86},"data":[0.00,0.00,1.86]},{"hostname":"f0649","id":"56","statistics":{"min":0.00,"avg":0.61,"max":1.83},"data":[0.00,0.00,1.83]},{"hostname":"f0649","id":"5","statistics":{"min":0.00,"avg":0.61,"max":1.83},"data":[0.00,0.00,1.83]},{"hostname":"f0649","id":"27","statistics":{"min":0.00,"avg":0.61,"max":1.82},"data":[0.00,0.00,1.82]},{"hostname":"f0649","id":"47","statistics":{"min":0.00,"avg":0.62,"max":1.86},"data":[0.00,0.00,1.86]}]},"node":{"unit":{"base":"F/s","prefix":"G"},"timestep":60,"series":[{"hostname":"f0649","id":"0","statistics":{"min":0.00,"avg":45.01,"max":135.04},"data":[0.00,0.00,135.04]}]}},"flops_dp":{"core":{"unit":{"base":"F/s","prefix":"G"},"timestep":60,"series":[{"hostname":"f0649","id":"21","statistics":{"min":0.00,"avg":0.30,"max":0.91},"data":[0.00,0.00,0.91]},{"hostname":"f0649","id":"49","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"71","statistics":{"min":0.00,"avg":0.26,"max":0.77},"data":[0.00,0.00,0.77]},{"hostname":"f0649","id":"8","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"17","statistics":{"min":0.00,"avg":0.30,"max":0.91},"data":[0.00,0.00,0.91]},{"hostname":"f0649","id":"25","statistics":{"min":0.00,"avg":0.30,"max":0.91},"data":[0.00,0.00,0.91]},{"hostname":"f0649","id":"13","statistics":{"min":0.00,"avg":0.30,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"19","statistics":{"min":0.00,"avg":0.30,"max":0.91},"data":[0.00,0.00,0.91]},{"hostname":"f0649","id":"36","statistics":{"min":0.00,"avg":0.31,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"59","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"24","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"52","statistics":{"min":0.00,"avg":0.30,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"56","statistics":{"min":0.00,"avg":0.30,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"58","statistics":{"min":0.00,"avg":0.31,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"63","statistics":{"min":0.00,"avg":0.31,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"68","statistics":{"min":0.00,"avg":0.30,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"6","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"62","statistics":{"min":0.00,"avg":0.30,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"26","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"0","statistics":{"min":0.00,"avg":0.89,"max":2.68},"data":[0.00,0.00,2.68]},{"hostname":"f0649","id":"4","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"27","statistics":{"min":0.00,"avg":0.30,"max":0.91},"data":[0.00,0.00,0.91]},{"hostname":"f0649","id":"70","statistics":{"min":0.00,"avg":0.26,"max":0.79},"data":[0.00,0.00,0.79]},{"hostname":"f0649","id":"41","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"43","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"2","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"5","statistics":{"min":0.00,"avg":0.30,"max":0.91},"data":[0.00,0.00,0.91]},{"hostname":"f0649","id":"7","statistics":{"min":0.00,"avg":0.30,"max":0.91},"data":[0.00,0.00,0.91]},{"hostname":"f0649","id":"30","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"31","statistics":{"min":0.00,"avg":0.30,"max":0.91},"data":[0.00,0.00,0.91]},{"hostname":"f0649","id":"35","statistics":{"min":0.00,"avg":0.27,"max":0.80},"data":[0.00,0.00,0.80]},{"hostname":"f0649","id":"53","statistics":{"min":0.00,"avg":0.31,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"61","statistics":{"min":0.00,"avg":0.31,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"67","statistics":{"min":0.00,"avg":0.31,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"15","statistics":{"min":0.00,"avg":0.30,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"34","statistics":{"min":0.00,"avg":0.25,"max":0.75},"data":[0.00,0.00,0.75]},{"hostname":"f0649","id":"57","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"66","statistics":{"min":0.00,"avg":0.30,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"10","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"40","statistics":{"min":0.00,"avg":0.31,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"50","statistics":{"min":0.00,"avg":0.30,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"60","statistics":{"min":0.00,"avg":0.30,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"39","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"54","statistics":{"min":0.00,"avg":0.30,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"64","statistics":{"min":0.00,"avg":0.30,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"32","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"33","statistics":{"min":0.00,"avg":0.30,"max":0.91},"data":[0.00,0.00,0.91]},{"hostname":"f0649","id":"44","statistics":{"min":0.00,"avg":0.30,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"51","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"3","statistics":{"min":0.00,"avg":0.30,"max":0.91},"data":[0.00,0.00,0.91]},{"hostname":"f0649","id":"12","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"23","statistics":{"min":0.00,"avg":0.30,"max":0.91},"data":[0.00,0.00,0.91]},{"hostname":"f0649","id":"55","statistics":{"min":0.00,"avg":0.31,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"42","statistics":{"min":0.00,"avg":0.31,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"46","statistics":{"min":0.00,"avg":0.30,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"1","statistics":{"min":0.00,"avg":0.30,"max":0.91},"data":[0.00,0.00,0.91]},{"hostname":"f0649","id":"11","statistics":{"min":0.00,"avg":0.30,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"14","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"22","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"29","statistics":{"min":0.00,"avg":0.30,"max":0.91},"data":[0.00,0.00,0.91]},{"hostname":"f0649","id":"38","statistics":{"min":0.00,"avg":0.31,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"47","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"48","statistics":{"min":0.00,"avg":0.30,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"9","statistics":{"min":0.00,"avg":0.30,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"20","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"28","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"16","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"18","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"37","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"45","statistics":{"min":0.00,"avg":0.31,"max":0.93},"data":[0.00,0.00,0.93]},{"hostname":"f0649","id":"65","statistics":{"min":0.00,"avg":0.31,"max":0.92},"data":[0.00,0.00,0.92]},{"hostname":"f0649","id":"69","statistics":{"min":0.00,"avg":0.31,"max":0.92},"data":[0.00,0.00,0.92]}]},"node":{"unit":{"base":"F/s","prefix":"G"},"timestep":60,"series":[{"hostname":"f0649","id":"0","statistics":{"min":0.00,"avg":22.50,"max":67.49},"data":[0.00,0.00,67.49]}]}},"flops_sp":{"core":{"unit":{"base":"F/s","prefix":"G"},"timestep":60,"series":[{"hostname":"f0649","id":"22","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"28","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"64","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"6","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"43","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"56","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"58","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"60","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"63","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"30","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"2","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"14","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"20","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"36","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"48","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"70","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"1","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"4","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"40","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"41","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"51","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"53","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"8","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"16","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"17","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"21","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"31","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"44","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"49","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"25","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"26","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"29","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"61","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"52","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"11","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"15","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"34","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"46","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"47","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"0","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"10","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"19","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"32","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"62","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"71","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"9","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"24","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"33","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"39","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"42","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"59","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"68","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"23","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"38","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"55","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"57","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"3","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"27","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"45","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"65","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"67","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"69","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"12","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"50","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"54","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"5","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"18","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"37","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"7","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"13","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"35","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]},{"hostname":"f0649","id":"66","statistics":{"min":0.00,"avg":0.00,"max":0.00},"data":[0.00,0.00,0.00]}]},"node":{"unit":{"base":"F/s","prefix":"G"},"timestep":60,"series":[{"hostname":"f0649","id":"0","statistics":{"min":0.00,"avg":0.02,"max":0.06},"data":[0.00,0.00,0.06]}]}},"ib_recv":{"node":{"unit":{"base":"B/s"},"timestep":60,"series":[{"hostname":"f0649","statistics":{"min":220.00,"avg":14442.82,"max":42581.37},"data":[527.09,220.00,42581.37]}]}},"ib_recv_pkts":{"node":{"unit":{"base":"packets/s"},"timestep":60,"series":[{"hostname":"f0649","statistics":{"min":1.25,"avg":201.53,"max":601.35},"data":[2.00,1.25,601.34]}]}},"ib_xmit":{"node":{"unit":{"base":"B/s"},"timestep":60,"series":[{"hostname":"f0649","statistics":{"min":56.20,"avg":282.10,"max":569.36},"data":[569.36,220.73,56.20]}]}},"ib_xmit_pkts":{"node":{"unit":{"base":"packets/s"},"timestep":60,"series":[{"hostname":"f0649","statistics":{"min":0.43,"avg":1.23,"max":2.00},"data":[2.00,1.25,0.43]}]}},"ipc":{"core":{"unit":{"base":"IPC"},"timestep":60,"series":[{"hostname":"f0649","id":"42","statistics":{"min":0.55,"avg":0.82,"max":1.07},"data":[1.07,0.55,0.84]},{"hostname":"f0649","id":"55","statistics":{"min":0.22,"avg":0.43,"max":0.83},"data":[0.23,0.22,0.83]},{"hostname":"f0649","id":"3","statistics":{"min":0.29,"avg":0.79,"max":1.18},"data":[1.17,0.29,0.91]},{"hostname":"f0649","id":"7","statistics":{"min":0.53,"avg":0.66,"max":0.89},"data":[0.55,0.53,0.89]},{"hostname":"f0649","id":"10","statistics":{"min":0.22,"avg":0.56,"max":0.81},"data":[0.64,0.22,0.81]},{"hostname":"f0649","id":"14","statistics":{"min":0.57,"avg":0.88,"max":1.26},"data":[1.26,0.57,0.80]},{"hostname":"f0649","id":"22","statistics":{"min":0.13,"avg":0.92,"max":1.83},"data":[1.83,0.13,0.80]},{"hostname":"f0649","id":"19","statistics":{"min":0.23,"avg":0.43,"max":0.82},"data":[0.24,0.23,0.82]},{"hostname":"f0649","id":"38","statistics":{"min":0.29,"avg":0.57,"max":0.88},"data":[0.56,0.29,0.88]},{"hostname":"f0649","id":"41","statistics":{"min":0.55,"avg":1.20,"max":2.21},"data":[2.21,0.55,0.84]},{"hostname":"f0649","id":"43","statistics":{"min":0.45,"avg":0.87,"max":1.31},"data":[1.31,0.45,0.85]},{"hostname":"f0649","id":"56","statistics":{"min":0.28,"avg":0.47,"max":0.85},"data":[0.28,0.28,0.85]},{"hostname":"f0649","id":"12","statistics":{"min":0.32,"avg":0.58,"max":0.83},"data":[0.58,0.32,0.83]},{"hostname":"f0649","id":"16","statistics":{"min":0.56,"avg":1.01,"max":1.65},"data":[1.65,0.56,0.82]},{"hostname":"f0649","id":"45","statistics":{"min":0.55,"avg":0.94,"max":1.45},"data":[1.45,0.55,0.84]},{"hostname":"f0649","id":"68","statistics":{"min":0.57,"avg":0.66,"max":0.81},"data":[0.57,0.59,0.81]},{"hostname":"f0649","id":"13","statistics":{"min":0.55,"avg":0.64,"max":0.82},"data":[0.55,0.55,0.82]},{"hostname":"f0649","id":"21","statistics":{"min":0.54,"avg":0.63,"max":0.82},"data":[0.55,0.54,0.82]},{"hostname":"f0649","id":"39","statistics":{"min":0.71,"avg":0.89,"max":1.11},"data":[0.71,1.11,0.86]},{"hostname":"f0649","id":"23","statistics":{"min":0.55,"avg":0.65,"max":0.83},"data":[0.55,0.56,0.83]},{"hostname":"f0649","id":"34","statistics":{"min":0.55,"avg":0.65,"max":0.82},"data":[0.55,0.57,0.82]},{"hostname":"f0649","id":"58","statistics":{"min":0.53,"avg":0.64,"max":0.83},"data":[0.55,0.53,0.83]},{"hostname":"f0649","id":"70","statistics":{"min":0.54,"avg":0.64,"max":0.82},"data":[0.54,0.58,0.82]},{"hostname":"f0649","id":"5","statistics":{"min":0.56,"avg":0.95,"max":1.40},"data":[1.40,0.56,0.89]},{"hostname":"f0649","id":"27","statistics":{"min":0.54,"avg":0.64,"max":0.82},"data":[0.54,0.56,0.82]},{"hostname":"f0649","id":"35","statistics":{"min":0.50,"avg":0.62,"max":0.81},"data":[0.50,0.54,0.81]},{"hostname":"f0649","id":"47","statistics":{"min":0.56,"avg":0.69,"max":0.83},"data":[0.68,0.56,0.83]},{"hostname":"f0649","id":"29","statistics":{"min":0.55,"avg":0.64,"max":0.81},"data":[0.55,0.56,0.81]},{"hostname":"f0649","id":"50","statistics":{"min":0.54,"avg":1.03,"max":1.73},"data":[1.73,0.54,0.82]},{"hostname":"f0649","id":"60","statistics":{"min":0.53,"avg":0.64,"max":0.82},"data":[0.57,0.53,0.82]},{"hostname":"f0649","id":"24","statistics":{"min":0.54,"avg":0.63,"max":0.81},"data":[0.54,0.55,0.81]},{"hostname":"f0649","id":"26","statistics":{"min":0.55,"avg":0.64,"max":0.81},"data":[0.55,0.57,0.81]},{"hostname":"f0649","id":"57","statistics":{"min":0.53,"avg":0.63,"max":0.82},"data":[0.55,0.53,0.82]},{"hostname":"f0649","id":"66","statistics":{"min":0.53,"avg":0.64,"max":0.82},"data":[0.56,0.53,0.82]},{"hostname":"f0649","id":"6","statistics":{"min":0.85,"avg":1.04,"max":1.36},"data":[1.36,0.86,0.91]},{"hostname":"f0649","id":"11","statistics":{"min":0.47,"avg":0.66,"max":0.81},"data":[0.47,0.71,0.81]},{"hostname":"f0649","id":"48","statistics":{"min":0.55,"avg":1.37,"max":2.73},"data":[2.73,0.55,0.84]},{"hostname":"f0649","id":"54","statistics":{"min":0.57,"avg":1.33,"max":2.59},"data":[2.59,0.57,0.82]},{"hostname":"f0649","id":"67","statistics":{"min":0.55,"avg":0.65,"max":0.80},"data":[0.58,0.55,0.80]},{"hostname":"f0649","id":"52","statistics":{"min":0.53,"avg":0.64,"max":0.84},"data":[0.53,0.54,0.84]},{"hostname":"f0649","id":"64","statistics":{"min":0.56,"avg":0.66,"max":0.82},"data":[0.58,0.56,0.82]},{"hostname":"f0649","id":"65","statistics":{"min":0.55,"avg":0.65,"max":0.82},"data":[0.57,0.55,0.82]},{"hostname":"f0649","id":"15","statistics":{"min":0.55,"avg":1.08,"max":1.86},"data":[1.86,0.55,0.82]},{"hostname":"f0649","id":"25","statistics":{"min":0.54,"avg":0.65,"max":0.82},"data":[0.54,0.57,0.82]},{"hostname":"f0649","id":"28","statistics":{"min":0.54,"avg":0.64,"max":0.81},"data":[0.56,0.54,0.81]},{"hostname":"f0649","id":"30","statistics":{"min":0.53,"avg":0.63,"max":0.81},"data":[0.54,0.53,0.81]},{"hostname":"f0649","id":"49","statistics":{"min":0.55,"avg":0.64,"max":0.82},"data":[0.55,0.56,0.82]},{"hostname":"f0649","id":"0","statistics":{"min":1.47,"avg":2.11,"max":2.96},"data":[1.90,2.96,1.47]},{"hostname":"f0649","id":"1","statistics":{"min":0.38,"avg":0.69,"max":0.89},"data":[0.38,0.79,0.89]},{"hostname":"f0649","id":"51","statistics":{"min":0.53,"avg":1.08,"max":1.88},"data":[1.88,0.53,0.82]},{"hostname":"f0649","id":"69","statistics":{"min":0.56,"avg":0.65,"max":0.82},"data":[0.56,0.56,0.82]},{"hostname":"f0649","id":"2","statistics":{"min":0.88,"avg":1.40,"max":2.11},"data":[2.11,1.20,0.88]},{"hostname":"f0649","id":"4","statistics":{"min":0.17,"avg":0.45,"max":0.89},"data":[0.17,0.28,0.89]},{"hostname":"f0649","id":"8","statistics":{"min":0.36,"avg":0.69,"max":0.88},"data":[0.88,0.36,0.82]},{"hostname":"f0649","id":"32","statistics":{"min":0.23,"avg":0.42,"max":0.81},"data":[0.23,0.23,0.81]},{"hostname":"f0649","id":"59","statistics":{"min":0.54,"avg":0.64,"max":0.83},"data":[0.57,0.54,0.83]},{"hostname":"f0649","id":"9","statistics":{"min":0.58,"avg":1.21,"max":2.27},"data":[2.27,0.58,0.79]},{"hostname":"f0649","id":"31","statistics":{"min":0.55,"avg":0.64,"max":0.80},"data":[0.55,0.57,0.80]},{"hostname":"f0649","id":"46","statistics":{"min":0.55,"avg":0.68,"max":0.83},"data":[0.66,0.55,0.83]},{"hostname":"f0649","id":"53","statistics":{"min":0.53,"avg":0.63,"max":0.83},"data":[0.54,0.53,0.83]},{"hostname":"f0649","id":"62","statistics":{"min":0.54,"avg":0.65,"max":0.83},"data":[0.54,0.57,0.83]},{"hostname":"f0649","id":"18","statistics":{"min":0.81,"avg":1.01,"max":1.28},"data":[0.93,1.28,0.81]},{"hostname":"f0649","id":"20","statistics":{"min":0.27,"avg":0.47,"max":0.81},"data":[0.33,0.27,0.81]},{"hostname":"f0649","id":"33","statistics":{"min":0.54,"avg":0.63,"max":0.80},"data":[0.54,0.54,0.80]},{"hostname":"f0649","id":"37","statistics":{"min":0.70,"avg":1.10,"max":1.75},"data":[1.75,0.70,0.85]},{"hostname":"f0649","id":"40","statistics":{"min":0.53,"avg":0.91,"max":1.37},"data":[1.37,0.53,0.84]},{"hostname":"f0649","id":"17","statistics":{"min":0.58,"avg":0.91,"max":1.34},"data":[1.34,0.58,0.81]},{"hostname":"f0649","id":"61","statistics":{"min":0.55,"avg":0.66,"max":0.85},"data":[0.55,0.57,0.85]},{"hostname":"f0649","id":"63","statistics":{"min":0.54,"avg":0.63,"max":0.82},"data":[0.54,0.54,0.82]},{"hostname":"f0649","id":"36","statistics":{"min":0.31,"avg":0.60,"max":0.84},"data":[0.66,0.31,0.84]},{"hostname":"f0649","id":"44","statistics":{"min":0.53,"avg":1.34,"max":2.65},"data":[2.65,0.53,0.84]},{"hostname":"f0649","id":"71","statistics":{"min":0.13,"avg":0.36,"max":0.82},"data":[0.13,0.14,0.82]}]},"node":{"unit":{"base":"IPC"},"timestep":60,"series":[{"hostname":"f0649","id":"0","statistics":{"min":0.56,"avg":0.77,"max":0.91},"data":[0.91,0.56,0.84]}]}},"mem_bw":{"node":{"unit":{"base":"B/s","prefix":"G"},"timestep":60,"series":[{"hostname":"f0649","id":"0","statistics":{"min":0.03,"avg":4.87,"max":14.55},"data":[0.04,0.03,14.55]}]},"socket":{"unit":{"base":"B/s","prefix":"G"},"timestep":60,"series":[{"hostname":"f0649","id":"0","statistics":{"min":0.02,"avg":2.65,"max":7.89},"data":[0.03,0.02,7.89]},{"hostname":"f0649","id":"0","statistics":{"min":0.01,"avg":2.23,"max":6.66},"data":[0.01,0.01,6.66]}]}},"mem_power":{"node":{"unit":{"base":"W"},"timestep":60,"series":[{"hostname":"f0649","id":"0","statistics":{"min":6.29,"avg":7.72,"max":10.56},"data":[6.33,6.29,10.56]}]},"socket":{"unit":{"base":"W"},"timestep":60,"series":[{"hostname":"f0649","id":"0","statistics":{"min":1.92,"avg":2.89,"max":4.80},"data":[1.94,1.92,4.80]},{"hostname":"f0649","id":"0","statistics":{"min":4.36,"avg":4.84,"max":5.75},"data":[4.39,4.36,5.75]}]}},"mem_used":{"node":{"unit":{"base":"B","prefix":"G"},"timestep":60,"series":[{"hostname":"f0649","statistics":{"min":6.10,"avg":6.16,"max":6.23},"data":[6.10,6.16,6.23]}]}},"nfs4_read":{"node":{"unit":{"base":"B/s","prefix":"M"},"timestep":60,"series":[{"hostname":"f0649","statistics":{"min":311.00,"avg":1045.33,"max":1525.00},"data":[311.00,1300.00,1525.00]}]}},"nfs4_total":{"node":{"unit":{"base":"B/s","prefix":"M"},"timestep":60,"series":[{"hostname":"f0649","statistics":{"min":2796.00,"avg":6430.00,"max":11518.00},"data":[2796.00,4976.00,11518.00]}]}},"nfs4_write":{"node":{"unit":{"base":"B/s","prefix":"M"},"timestep":60,"series":[{"hostname":"f0649","statistics":{"min":0.00,"avg":24.33,"max":38.00},"data":[35.00,0.00,38.00]}]}},"vectorization_ratio":{"core":{"unit":{"base":"%"},"timestep":60,"series":[{"hostname":"f0649","id":"36","statistics":{"min":0.00,"avg":25.56,"max":76.69},"data":[0.00,0.00,76.69]},{"hostname":"f0649","id":"42","statistics":{"min":0.00,"avg":25.57,"max":76.70},"data":[0.00,0.00,76.70]},{"hostname":"f0649","id":"45","statistics":{"min":0.00,"avg":25.56,"max":76.69},"data":[0.00,0.00,76.69]},{"hostname":"f0649","id":"71","statistics":{"min":0.00,"avg":24.62,"max":73.85},"data":[0.00,0.00,73.85]},{"hostname":"f0649","id":"29","statistics":{"min":0.00,"avg":25.48,"max":76.44},"data":[0.00,0.00,76.44]},{"hostname":"f0649","id":"15","statistics":{"min":0.00,"avg":25.50,"max":76.48},"data":[0.00,0.00,76.48]},{"hostname":"f0649","id":"28","statistics":{"min":0.00,"avg":25.62,"max":76.86},"data":[0.00,0.00,76.86]},{"hostname":"f0649","id":"37","statistics":{"min":0.00,"avg":25.61,"max":76.84},"data":[0.00,0.00,76.84]},{"hostname":"f0649","id":"48","statistics":{"min":0.00,"avg":25.52,"max":76.55},"data":[0.00,0.00,76.55]},{"hostname":"f0649","id":"49","statistics":{"min":0.00,"avg":25.57,"max":76.70},"data":[0.00,0.00,76.70]},{"hostname":"f0649","id":"11","statistics":{"min":0.00,"avg":25.49,"max":76.47},"data":[0.00,0.00,76.47]},{"hostname":"f0649","id":"18","statistics":{"min":0.00,"avg":25.59,"max":76.77},"data":[0.00,0.00,76.77]},{"hostname":"f0649","id":"33","statistics":{"min":0.00,"avg":25.49,"max":76.48},"data":[0.00,0.00,76.48]},{"hostname":"f0649","id":"59","statistics":{"min":0.00,"avg":25.52,"max":76.56},"data":[0.00,0.00,76.56]},{"hostname":"f0649","id":"70","statistics":{"min":0.00,"avg":24.74,"max":74.22},"data":[0.00,0.00,74.22]},{"hostname":"f0649","id":"2","statistics":{"min":0.00,"avg":25.53,"max":76.59},"data":[0.00,0.01,76.59]},{"hostname":"f0649","id":"6","statistics":{"min":0.00,"avg":25.55,"max":76.65},"data":[0.00,0.00,76.65]},{"hostname":"f0649","id":"22","statistics":{"min":0.00,"avg":25.58,"max":76.73},"data":[0.00,0.00,76.73]},{"hostname":"f0649","id":"39","statistics":{"min":0.00,"avg":25.60,"max":76.80},"data":[0.00,0.00,76.80]},{"hostname":"f0649","id":"4","statistics":{"min":0.00,"avg":25.56,"max":76.67},"data":[0.00,0.00,76.67]},{"hostname":"f0649","id":"31","statistics":{"min":0.00,"avg":25.49,"max":76.47},"data":[0.00,0.00,76.47]},{"hostname":"f0649","id":"38","statistics":{"min":0.00,"avg":25.55,"max":76.64},"data":[0.00,0.00,76.64]},{"hostname":"f0649","id":"46","statistics":{"min":0.00,"avg":25.52,"max":76.54},"data":[0.00,0.00,76.54]},{"hostname":"f0649","id":"30","statistics":{"min":0.00,"avg":25.62,"max":76.87},"data":[0.00,0.00,76.87]},{"hostname":"f0649","id":"13","statistics":{"min":0.00,"avg":25.49,"max":76.48},"data":[0.00,0.00,76.48]},{"hostname":"f0649","id":"16","statistics":{"min":0.00,"avg":25.61,"max":76.82},"data":[0.00,0.00,76.82]},{"hostname":"f0649","id":"41","statistics":{"min":0.00,"avg":25.62,"max":76.86},"data":[0.00,0.00,76.86]},{"hostname":"f0649","id":"43","statistics":{"min":0.00,"avg":25.62,"max":76.85},"data":[0.00,0.00,76.85]},{"hostname":"f0649","id":"47","statistics":{"min":0.00,"avg":25.56,"max":76.69},"data":[0.00,0.00,76.69]},{"hostname":"f0649","id":"54","statistics":{"min":0.00,"avg":25.51,"max":76.54},"data":[0.00,0.00,76.54]},{"hostname":"f0649","id":"55","statistics":{"min":0.00,"avg":25.53,"max":76.59},"data":[0.00,0.00,76.59]},{"hostname":"f0649","id":"8","statistics":{"min":0.00,"avg":25.60,"max":76.81},"data":[0.00,0.00,76.81]},{"hostname":"f0649","id":"17","statistics":{"min":0.00,"avg":25.46,"max":76.39},"data":[0.00,0.00,76.39]},{"hostname":"f0649","id":"23","statistics":{"min":0.00,"avg":25.44,"max":76.33},"data":[0.00,0.00,76.33]},{"hostname":"f0649","id":"7","statistics":{"min":0.00,"avg":25.45,"max":76.34},"data":[0.00,0.00,76.34]},{"hostname":"f0649","id":"19","statistics":{"min":0.00,"avg":25.46,"max":76.37},"data":[0.00,0.00,76.37]},{"hostname":"f0649","id":"20","statistics":{"min":0.00,"avg":25.58,"max":76.75},"data":[0.00,0.00,76.75]},{"hostname":"f0649","id":"27","statistics":{"min":0.00,"avg":25.47,"max":76.40},"data":[0.00,0.00,76.40]},{"hostname":"f0649","id":"51","statistics":{"min":0.00,"avg":25.57,"max":76.70},"data":[0.00,0.00,76.70]},{"hostname":"f0649","id":"56","statistics":{"min":0.00,"avg":25.50,"max":76.51},"data":[0.00,0.00,76.51]},{"hostname":"f0649","id":"60","statistics":{"min":0.00,"avg":25.52,"max":76.57},"data":[0.00,0.00,76.57]},{"hostname":"f0649","id":"65","statistics":{"min":0.00,"avg":25.56,"max":76.67},"data":[0.00,0.00,76.67]},{"hostname":"f0649","id":"10","statistics":{"min":0.00,"avg":25.59,"max":76.78},"data":[0.00,0.00,76.78]},{"hostname":"f0649","id":"68","statistics":{"min":0.00,"avg":25.55,"max":76.65},"data":[0.00,0.00,76.65]},{"hostname":"f0649","id":"69","statistics":{"min":0.00,"avg":25.57,"max":76.70},"data":[0.00,0.00,76.70]},{"hostname":"f0649","id":"66","statistics":{"min":0.00,"avg":25.55,"max":76.64},"data":[0.00,0.00,76.64]},{"hostname":"f0649","id":"62","statistics":{"min":0.00,"avg":25.52,"max":76.57},"data":[0.00,0.00,76.57]},{"hostname":"f0649","id":"14","statistics":{"min":0.00,"avg":25.59,"max":76.78},"data":[0.00,0.00,76.78]},{"hostname":"f0649","id":"58","statistics":{"min":0.00,"avg":25.50,"max":76.50},"data":[0.00,0.00,76.50]},{"hostname":"f0649","id":"63","statistics":{"min":0.00,"avg":25.54,"max":76.63},"data":[0.00,0.00,76.63]},{"hostname":"f0649","id":"5","statistics":{"min":0.00,"avg":25.45,"max":76.35},"data":[0.00,0.00,76.35]},{"hostname":"f0649","id":"35","statistics":{"min":0.00,"avg":24.79,"max":74.38},"data":[0.00,0.00,74.38]},{"hostname":"f0649","id":"24","statistics":{"min":0.00,"avg":25.61,"max":76.83},"data":[0.00,0.00,76.83]},{"hostname":"f0649","id":"34","statistics":{"min":0.00,"avg":24.50,"max":73.49},"data":[0.00,0.00,73.49]},{"hostname":"f0649","id":"57","statistics":{"min":0.00,"avg":25.52,"max":76.57},"data":[0.00,0.00,76.57]},{"hostname":"f0649","id":"61","statistics":{"min":0.00,"avg":25.54,"max":76.62},"data":[0.00,0.00,76.62]},{"hostname":"f0649","id":"32","statistics":{"min":0.00,"avg":25.63,"max":76.90},"data":[0.00,0.00,76.90]},{"hostname":"f0649","id":"26","statistics":{"min":0.00,"avg":25.60,"max":76.81},"data":[0.00,0.00,76.81]},{"hostname":"f0649","id":"1","statistics":{"min":0.00,"avg":25.45,"max":76.33},"data":[0.00,0.00,76.33]},{"hostname":"f0649","id":"25","statistics":{"min":0.00,"avg":25.46,"max":76.39},"data":[0.00,0.00,76.39]},{"hostname":"f0649","id":"50","statistics":{"min":0.00,"avg":25.52,"max":76.55},"data":[0.00,0.00,76.55]},{"hostname":"f0649","id":"12","statistics":{"min":0.00,"avg":25.60,"max":76.79},"data":[0.00,0.00,76.79]},{"hostname":"f0649","id":"40","statistics":{"min":0.00,"avg":25.57,"max":76.71},"data":[0.00,0.00,76.71]},{"hostname":"f0649","id":"44","statistics":{"min":0.00,"avg":25.52,"max":76.55},"data":[0.00,0.00,76.55]},{"hostname":"f0649","id":"52","statistics":{"min":0.00,"avg":25.52,"max":76.55},"data":[0.00,0.00,76.55]},{"hostname":"f0649","id":"53","statistics":{"min":0.00,"avg":25.53,"max":76.60},"data":[0.00,0.00,76.60]},{"hostname":"f0649","id":"64","statistics":{"min":0.00,"avg":25.54,"max":76.61},"data":[0.00,0.00,76.61]},{"hostname":"f0649","id":"3","statistics":{"min":0.00,"avg":25.43,"max":76.29},"data":[0.00,0.00,76.29]},{"hostname":"f0649","id":"9","statistics":{"min":0.00,"avg":25.49,"max":76.47},"data":[0.00,0.00,76.47]},{"hostname":"f0649","id":"21","statistics":{"min":0.00,"avg":25.45,"max":76.33},"data":[0.00,0.00,76.33]},{"hostname":"f0649","id":"67","statistics":{"min":0.00,"avg":25.57,"max":76.69},"data":[0.00,0.00,76.69]},{"hostname":"f0649","id":"0","statistics":{"min":0.00,"avg":28.37,"max":85.12},"data":[0.00,0.00,85.12]}]},"node":{"unit":{"base":"%"},"timestep":60,"series":[{"hostname":"f0649","id":"0","statistics":{"min":0.00,"avg":25.53,"max":76.58},"data":[0.00,0.00,76.58]}]}}} diff --git a/internal/importer/testdata/fritzError-golden.json b/internal/importer/testdata/fritzError-golden.json new file mode 100644 index 0000000..e234666 --- /dev/null +++ b/internal/importer/testdata/fritzError-golden.json @@ -0,0 +1,6 @@ +{ + "jobId": 398955, + "cluster": "fritz", + "startTime": 1675956725, + "duration": 260 +} diff --git a/internal/importer/testdata/fritzMinimal-golden.json b/internal/importer/testdata/fritzMinimal-golden.json new file mode 100644 index 0000000..3cee40e --- /dev/null +++ b/internal/importer/testdata/fritzMinimal-golden.json @@ -0,0 +1,6 @@ +{ + "jobId": 398764, + "cluster": "fritz", + "startTime": 1675954353, + "duration": 177 +} diff --git a/internal/importer/testdata/meta-fritzError.input b/internal/importer/testdata/meta-fritzError.input new file mode 100644 index 0000000..2b8d0e8 --- /dev/null +++ b/internal/importer/testdata/meta-fritzError.input @@ -0,0 +1 @@ +{"jobId":398955,"user":"k106eb10","project":"k106eb","cluster":"fritz","subCluster":"main","partition":"singlenode","arrayJobId":0,"numNodes":1,"numHwthreads":72,"numAcc":0,"exclusive":1,"monitoringStatus":1,"smt":0,"jobState":"completed","duration":260,"walltime":86340,"resources":[{"hostname":"f0720"}],"metaData":{"jobName":"ams_pipeline","jobScript":"#!/bin/bash -l\n#SBATCH --job-name=ams_pipeline\n#SBATCH --time=23:59:00\n#SBATCH --partition=singlenode\n#SBATCH --ntasks=72\n#SBATCH --hint=multithread\n#SBATCH --chdir=/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/base-hcp/occ-shaken/hcp16.occ.4.shake.0/cfg/NiAl3NiAl11\n#SBATCH --export=NONE\nunset SLURM_EXPORT_ENV\nuss=$(whoami)\nfind /dev/shm/ -user $uss -type f -mmin +30 -delete\ncd \"/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/base-hcp/occ-shaken/hcp16.occ.4.shake.0/cfg/NiAl3NiAl11\"\nams_pipeline pipeline.json \u003e \"/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/base-hcp/occ-shaken/hcp16.occ.4.shake.0/cfg/NiAl3NiAl11/ams_pipeline_job.sh.out\" 2\u003e \"/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/base-hcp/occ-shaken/hcp16.occ.4.shake.0/cfg/NiAl3NiAl11/ams_pipeline_job.sh.err\"\n","slurmInfo":"\nJobId=398955 JobName=ams_pipeline\n UserId=k106eb10(210387) GroupId=80111\n Account=k106eb QOS=normal \n Requeue=False Restarts=0 BatchFlag=True \n TimeLimit=1439\n SubmitTime=2023-02-09T14:11:22\n Partition=singlenode \n NodeList=f0720\n NumNodes=1 NumCPUs=72 NumTasks=72 CPUs/Task=1\n NTasksPerNode:Socket:Core=0:None:None\n TRES_req=cpu=72,mem=250000M,node=1,billing=72\n TRES_alloc=cpu=72,node=1,billing=72\n Command=/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/base-hcp/occ-shaken/hcp16.occ.4.shake.0/cfg/NiAl3NiAl11/ams_pipeline_job.sh\n WorkDir=/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/base-hcp/occ-shaken/hcp16.occ.4.shake.0/cfg/NiAl3NiAl11\n StdErr=\n StdOut=ams_pipeline.o%j\n"},"startTime":1675956725,"statistics":{"clock":{"unit":{"base":"Hz","prefix":"M"},"avg":2335.254,"min":800.418,"max":2734.922},"cpu_load":{"unit":{"base":""},"avg":52.72,"min":34.46,"max":71.91},"cpu_power":{"unit":{"base":"W"},"avg":407.767,"min":93.932,"max":497.636},"cpu_user":{"unit":{"base":""},"avg":63.678,"min":19.872,"max":96.633},"flops_any":{"unit":{"base":"F/s","prefix":"G"},"avg":635.672,"min":0,"max":1332.874},"flops_dp":{"unit":{"base":"F/s","prefix":"G"},"avg":261.006,"min":0,"max":382.294},"flops_sp":{"unit":{"base":"F/s","prefix":"G"},"avg":113.659,"min":0,"max":568.286},"ib_recv":{"unit":{"base":"B/s"},"avg":27981.111,"min":69.4,"max":48084.589},"ib_recv_pkts":{"unit":{"base":"packets/s"},"avg":398.939,"min":0.5,"max":693.817},"ib_xmit":{"unit":{"base":"B/s"},"avg":188.513,"min":39.597,"max":724.568},"ib_xmit_pkts":{"unit":{"base":"packets/s"},"avg":0.867,"min":0.2,"max":2.933},"ipc":{"unit":{"base":"IPC"},"avg":0.944,"min":0.564,"max":1.291},"mem_bw":{"unit":{"base":"B/s","prefix":"G"},"avg":79.565,"min":0.021,"max":116.02},"mem_power":{"unit":{"base":"W"},"avg":24.692,"min":7.883,"max":31.318},"mem_used":{"unit":{"base":"B","prefix":"G"},"avg":22.566,"min":8.225,"max":27.613},"nfs4_read":{"unit":{"base":"B/s","prefix":"M"},"avg":647,"min":0,"max":1946},"nfs4_total":{"unit":{"base":"B/s","prefix":"M"},"avg":6181.6,"min":1270,"max":11411},"nfs4_write":{"unit":{"base":"B/s","prefix":"M"},"avg":22.4,"min":11,"max":29},"vectorization_ratio":{"unit":{"base":"%"},"avg":77.351,"min":0,"max":98.837}}} diff --git a/internal/importer/testdata/meta-fritzMinimal.input b/internal/importer/testdata/meta-fritzMinimal.input new file mode 100644 index 0000000..f2cce79 --- /dev/null +++ b/internal/importer/testdata/meta-fritzMinimal.input @@ -0,0 +1 @@ +{"jobId":398764,"user":"k106eb10","project":"k106eb","cluster":"fritz","subCluster":"main","numNodes":1,"exclusive":1,"jobState":"completed","duration":177,"resources":[{"hostname":"f0649"}],"startTime":1675954353,"statistics":{"clock":{"unit":{"base":"Hz","prefix":"M"},"avg":1336.519,"min":801.564,"max":2348.215},"cpu_load":{"unit":{"base":""},"avg":31.64,"min":17.36,"max":45.54},"cpu_power":{"unit":{"base":"W"},"avg":150.018,"min":93.672,"max":261.592},"cpu_user":{"unit":{"base":""},"avg":28.518,"min":0.09,"max":57.343},"flops_any":{"unit":{"base":"F/s","prefix":"G"},"avg":45.012,"min":0,"max":135.037},"flops_dp":{"unit":{"base":"F/s","prefix":"G"},"avg":22.496,"min":0,"max":67.488},"flops_sp":{"unit":{"base":"F/s","prefix":"G"},"avg":0.02,"min":0,"max":0.061},"ib_recv":{"unit":{"base":"B/s"},"avg":14442.82,"min":219.998,"max":42581.368},"ib_recv_pkts":{"unit":{"base":"packets/s"},"avg":201.532,"min":1.25,"max":601.345},"ib_xmit":{"unit":{"base":"B/s"},"avg":282.098,"min":56.2,"max":569.363},"ib_xmit_pkts":{"unit":{"base":"packets/s"},"avg":1.228,"min":0.433,"max":2},"ipc":{"unit":{"base":"IPC"},"avg":0.77,"min":0.564,"max":0.906},"mem_bw":{"unit":{"base":"B/s","prefix":"G"},"avg":4.872,"min":0.025,"max":14.552},"mem_power":{"unit":{"base":"W"},"avg":7.725,"min":6.286,"max":10.556},"mem_used":{"unit":{"base":"B","prefix":"G"},"avg":6.162,"min":6.103,"max":6.226},"nfs4_read":{"unit":{"base":"B/s","prefix":"M"},"avg":1045.333,"min":311,"max":1525},"nfs4_total":{"unit":{"base":"B/s","prefix":"M"},"avg":6430,"min":2796,"max":11518},"nfs4_write":{"unit":{"base":"B/s","prefix":"M"},"avg":24.333,"min":0,"max":38},"vectorization_ratio":{"unit":{"base":"%"},"avg":25.528,"min":0,"max":76.585}}} diff --git a/internal/metricdata/cc-metric-store.go b/internal/metricdata/cc-metric-store.go index 0df3fda..6b3153f 100644 --- a/internal/metricdata/cc-metric-store.go +++ b/internal/metricdata/cc-metric-store.go @@ -293,7 +293,7 @@ func (ccms *CCMetricStore) buildQueries( scopesLoop: for _, requestedScope := range scopes { nativeScope := mc.Scope - if nativeScope == schema.MetricScopeAccelerator && job.NumAcc == nil { + if nativeScope == schema.MetricScopeAccelerator && job.NumAcc == 0 { continue } diff --git a/internal/repository/dbConnection.go b/internal/repository/dbConnection.go index 7ea24ef..790ec74 100644 --- a/internal/repository/dbConnection.go +++ b/internal/repository/dbConnection.go @@ -56,7 +56,10 @@ func Connect(driver string, db string) { } dbConnInstance = &DBConnection{DB: dbHandle, Driver: driver} - checkDBVersion(driver, dbHandle.DB) + err = checkDBVersion(driver, dbHandle.DB) + if err != nil { + log.Fatal(err) + } }) } diff --git a/internal/repository/init.go b/internal/repository/init.go deleted file mode 100644 index 2649587..0000000 --- a/internal/repository/init.go +++ /dev/null @@ -1,351 +0,0 @@ -// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. -// All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. -package repository - -import ( - "bytes" - "database/sql" - "encoding/json" - "fmt" - "os" - "strings" - "time" - - "github.com/ClusterCockpit/cc-backend/internal/config" - "github.com/ClusterCockpit/cc-backend/pkg/archive" - "github.com/ClusterCockpit/cc-backend/pkg/log" - "github.com/ClusterCockpit/cc-backend/pkg/schema" - "github.com/ClusterCockpit/cc-backend/pkg/units" -) - -const NamedJobInsert string = `INSERT INTO job ( - job_id, user, project, cluster, subcluster, ` + "`partition`" + `, array_job_id, num_nodes, num_hwthreads, num_acc, - exclusive, monitoring_status, smt, job_state, start_time, duration, walltime, resources, meta_data, - mem_used_max, flops_any_avg, mem_bw_avg, load_avg, net_bw_avg, net_data_vol_total, file_bw_avg, file_data_vol_total -) VALUES ( - :job_id, :user, :project, :cluster, :subcluster, :partition, :array_job_id, :num_nodes, :num_hwthreads, :num_acc, - :exclusive, :monitoring_status, :smt, :job_state, :start_time, :duration, :walltime, :resources, :meta_data, - :mem_used_max, :flops_any_avg, :mem_bw_avg, :load_avg, :net_bw_avg, :net_data_vol_total, :file_bw_avg, :file_data_vol_total -);` - -// Import all jobs specified as `:,...` -func HandleImportFlag(flag string) error { - for _, pair := range strings.Split(flag, ",") { - files := strings.Split(pair, ":") - if len(files) != 2 { - return fmt.Errorf("REPOSITORY/INIT > invalid import flag format") - } - - raw, err := os.ReadFile(files[0]) - if err != nil { - log.Warn("Error while reading metadata file for import") - return err - } - - if config.Keys.Validate { - if err := schema.Validate(schema.Meta, bytes.NewReader(raw)); err != nil { - return fmt.Errorf("REPOSITORY/INIT > validate job meta: %v", err) - } - } - dec := json.NewDecoder(bytes.NewReader(raw)) - dec.DisallowUnknownFields() - jobMeta := schema.JobMeta{BaseJob: schema.JobDefaults} - if err := dec.Decode(&jobMeta); err != nil { - log.Warn("Error while decoding raw json metadata for import") - return err - } - - raw, err = os.ReadFile(files[1]) - if err != nil { - log.Warn("Error while reading jobdata file for import") - return err - } - - if config.Keys.Validate { - if err := schema.Validate(schema.Data, bytes.NewReader(raw)); err != nil { - return fmt.Errorf("REPOSITORY/INIT > validate job data: %v", err) - } - } - dec = json.NewDecoder(bytes.NewReader(raw)) - dec.DisallowUnknownFields() - jobData := schema.JobData{} - if err := dec.Decode(&jobData); err != nil { - log.Warn("Error while decoding raw json jobdata for import") - return err - } - - checkJobData(&jobData) - SanityChecks(&jobMeta.BaseJob) - jobMeta.MonitoringStatus = schema.MonitoringStatusArchivingSuccessful - if job, err := GetJobRepository().Find(&jobMeta.JobID, &jobMeta.Cluster, &jobMeta.StartTime); err != sql.ErrNoRows { - if err != nil { - log.Warn("Error while finding job in jobRepository") - return err - } - - return fmt.Errorf("REPOSITORY/INIT > a job with that jobId, cluster and startTime does already exist (dbid: %d)", job.ID) - } - - job := schema.Job{ - BaseJob: jobMeta.BaseJob, - StartTime: time.Unix(jobMeta.StartTime, 0), - StartTimeUnix: jobMeta.StartTime, - } - - // TODO: Other metrics... - job.FlopsAnyAvg = loadJobStat(&jobMeta, "flops_any") - job.MemBwAvg = loadJobStat(&jobMeta, "mem_bw") - job.NetBwAvg = loadJobStat(&jobMeta, "net_bw") - job.FileBwAvg = loadJobStat(&jobMeta, "file_bw") - job.RawResources, err = json.Marshal(job.Resources) - if err != nil { - log.Warn("Error while marshaling job resources") - return err - } - job.RawMetaData, err = json.Marshal(job.MetaData) - if err != nil { - log.Warn("Error while marshaling job metadata") - return err - } - - if err := SanityChecks(&job.BaseJob); err != nil { - log.Warn("BaseJob SanityChecks failed") - return err - } - - if err := archive.GetHandle().ImportJob(&jobMeta, &jobData); err != nil { - log.Error("Error while importing job") - return err - } - - res, err := GetConnection().DB.NamedExec(NamedJobInsert, job) - if err != nil { - log.Warn("Error while NamedJobInsert") - return err - } - - id, err := res.LastInsertId() - if err != nil { - log.Warn("Error while getting last insert ID") - return err - } - - for _, tag := range job.Tags { - if _, err := GetJobRepository().AddTagOrCreate(id, tag.Type, tag.Name); err != nil { - log.Error("Error while adding or creating tag") - return err - } - } - - log.Infof("successfully imported a new job (jobId: %d, cluster: %s, dbid: %d)", job.JobID, job.Cluster, id) - } - return nil -} - -// Delete the tables "job", "tag" and "jobtag" from the database and -// repopulate them using the jobs found in `archive`. -func InitDB() error { - db := GetConnection() - starttime := time.Now() - log.Print("Building job table...") - - // Inserts are bundled into transactions because in sqlite, - // that speeds up inserts A LOT. - tx, err := db.DB.Beginx() - if err != nil { - log.Warn("Error while bundling transactions") - return err - } - - stmt, err := tx.PrepareNamed(NamedJobInsert) - if err != nil { - log.Warn("Error while preparing namedJobInsert") - return err - } - tags := make(map[string]int64) - - // Not using log.Print because we want the line to end with `\r` and - // this function is only ever called when a special command line flag - // is passed anyways. - fmt.Printf("%d jobs inserted...\r", 0) - - ar := archive.GetHandle() - i := 0 - errorOccured := 0 - - for jobContainer := range ar.Iter(false) { - - jobMeta := jobContainer.Meta - - // // Bundle 100 inserts into one transaction for better performance: - if i%10 == 0 { - if tx != nil { - if err := tx.Commit(); err != nil { - log.Warn("Error while committing transactions for jobMeta") - return err - } - } - - tx, err = db.DB.Beginx() - if err != nil { - log.Warn("Error while bundling transactions for jobMeta") - return err - } - - stmt = tx.NamedStmt(stmt) - fmt.Printf("%d jobs inserted...\r", i) - } - - jobMeta.MonitoringStatus = schema.MonitoringStatusArchivingSuccessful - job := schema.Job{ - BaseJob: jobMeta.BaseJob, - StartTime: time.Unix(jobMeta.StartTime, 0), - StartTimeUnix: jobMeta.StartTime, - } - - // TODO: Other metrics... - job.FlopsAnyAvg = loadJobStat(jobMeta, "flops_any") - job.MemBwAvg = loadJobStat(jobMeta, "mem_bw") - job.NetBwAvg = loadJobStat(jobMeta, "net_bw") - job.FileBwAvg = loadJobStat(jobMeta, "file_bw") - - job.RawResources, err = json.Marshal(job.Resources) - if err != nil { - log.Errorf("repository initDB(): %v", err) - errorOccured++ - continue - } - - job.RawMetaData, err = json.Marshal(job.MetaData) - if err != nil { - log.Errorf("repository initDB(): %v", err) - errorOccured++ - continue - } - - if err := SanityChecks(&job.BaseJob); err != nil { - log.Errorf("repository initDB(): %v", err) - errorOccured++ - continue - } - - res, err := stmt.Exec(job) - if err != nil { - log.Errorf("repository initDB(): %v", err) - errorOccured++ - continue - } - - id, err := res.LastInsertId() - if err != nil { - log.Errorf("repository initDB(): %v", err) - errorOccured++ - continue - } - - for _, tag := range job.Tags { - tagstr := tag.Name + ":" + tag.Type - tagId, ok := tags[tagstr] - if !ok { - res, err := tx.Exec(`INSERT INTO tag (tag_name, tag_type) VALUES (?, ?)`, tag.Name, tag.Type) - if err != nil { - log.Errorf("Error while inserting tag into tag table: %v (Type %v)", tag.Name, tag.Type) - return err - } - tagId, err = res.LastInsertId() - if err != nil { - log.Warn("Error while getting last insert ID") - return err - } - tags[tagstr] = tagId - } - - if _, err := tx.Exec(`INSERT INTO jobtag (job_id, tag_id) VALUES (?, ?)`, id, tagId); err != nil { - log.Errorf("Error while inserting jobtag into jobtag table: %v (TagID %v)", id, tagId) - return err - } - } - - if err == nil { - i += 1 - } - } - - if errorOccured > 0 { - log.Warnf("Error in import of %d jobs!", errorOccured) - } - - if err := tx.Commit(); err != nil { - log.Warn("Error while committing SQL transactions") - return err - } - - log.Printf("A total of %d jobs have been registered in %.3f seconds.\n", i, time.Since(starttime).Seconds()) - return nil -} - -// This function also sets the subcluster if necessary! -func SanityChecks(job *schema.BaseJob) error { - if c := archive.GetCluster(job.Cluster); c == nil { - return fmt.Errorf("no such cluster: %v", job.Cluster) - } - if err := archive.AssignSubCluster(job); err != nil { - log.Warn("Error while assigning subcluster to job") - return err - } - if !job.State.Valid() { - return fmt.Errorf("not a valid job state: %v", job.State) - } - if len(job.Resources) == 0 || len(job.User) == 0 { - return fmt.Errorf("'resources' and 'user' should not be empty") - } - if *job.NumAcc < 0 || *job.NumHWThreads < 0 || job.NumNodes < 1 { - return fmt.Errorf("'numNodes', 'numAcc' or 'numHWThreads' invalid") - } - if len(job.Resources) != int(job.NumNodes) { - return fmt.Errorf("len(resources) does not equal numNodes (%d vs %d)", len(job.Resources), job.NumNodes) - } - - return nil -} - -func loadJobStat(job *schema.JobMeta, metric string) float64 { - if stats, ok := job.Statistics[metric]; ok { - return stats.Avg - } - - return 0.0 -} - -func checkJobData(d *schema.JobData) error { - for _, scopes := range *d { - var newUnit string - // Add node scope if missing - for _, metric := range scopes { - if strings.Contains(metric.Unit.Base, "B/s") || - strings.Contains(metric.Unit.Base, "F/s") || - strings.Contains(metric.Unit.Base, "B") { - - // First get overall avg - sum := 0.0 - for _, s := range metric.Series { - sum += s.Statistics.Avg - } - - avg := sum / float64(len(metric.Series)) - - for _, s := range metric.Series { - fp := schema.ConvertFloatToFloat64(s.Data) - // Normalize values with new unit prefix - oldUnit := metric.Unit.Base - units.NormalizeSeries(fp, avg, oldUnit, &newUnit) - s.Data = schema.GetFloat64ToFloat(fp) - } - metric.Unit.Base = newUnit - } - } - } - return nil -} diff --git a/internal/repository/job.go b/internal/repository/job.go index 50041d8..989c9e5 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -96,6 +96,50 @@ func scanJob(row interface{ Scan(...interface{}) error }) (*schema.Job, error) { return job, nil } +func (r *JobRepository) Optimize() error { + var err error + + switch r.driver { + case "sqlite3": + if _, err = r.DB.Exec(`VACUUM`); err != nil { + return err + } + case "mysql": + log.Info("Optimize currently not supported for mysql driver") + } + + return nil +} + +func (r *JobRepository) Flush() error { + var err error + + switch r.driver { + case "sqlite3": + if _, err = r.DB.Exec(`DELETE FROM jobtag`); err != nil { + return err + } + if _, err = r.DB.Exec(`DELETE FROM tag`); err != nil { + return err + } + if _, err = r.DB.Exec(`DELETE FROM job`); err != nil { + return err + } + case "mysql": + if _, err = r.DB.Exec(`TRUNCATE TABLE jobtag`); err != nil { + return err + } + if _, err = r.DB.Exec(`TRUNCATE TABLE tag`); err != nil { + return err + } + if _, err = r.DB.Exec(`TRUNCATE TABLE job`); err != nil { + return err + } + } + + return nil +} + func scanJobLink(row interface{ Scan(...interface{}) error }) (*model.JobLink, error) { jobLink := &model.JobLink{} if err := row.Scan( @@ -548,7 +592,7 @@ func (r *JobRepository) FindUserOrProjectOrJobname(ctx context.Context, searchte func (r *JobRepository) FindColumnValue(user *auth.User, searchterm string, table string, selectColumn string, whereColumn string, isLike bool) (result string, err error) { compareStr := " = ?" query := searchterm - if isLike == true { + if isLike { compareStr = " LIKE ?" query = "%" + searchterm + "%" } @@ -689,6 +733,38 @@ func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error { return nil } +func (r *JobRepository) FindJobsBefore(startTime int64) ([]*schema.Job, error) { + + query := sq.Select(jobColumns...).From("job").Where(fmt.Sprintf( + "job.start_time < %d", startTime)) + + sql, args, err := query.ToSql() + if err != nil { + log.Warn("Error while converting query to sql") + return nil, err + } + + log.Debugf("SQL query: `%s`, args: %#v", sql, args) + rows, err := query.RunWith(r.stmtCache).Query() + if err != nil { + log.Error("Error while running query") + return nil, err + } + + jobs := make([]*schema.Job, 0, 50) + for rows.Next() { + job, err := scanJob(rows) + if err != nil { + rows.Close() + log.Warn("Error while scanning rows") + return nil, err + } + jobs = append(jobs, job) + } + + return jobs, nil +} + // GraphQL validation should make sure that no unkown values can be specified. var groupBy2column = map[model.Aggregate]string{ model.AggregateUser: "job.user", @@ -706,9 +782,10 @@ func (r *JobRepository) JobsStatistics(ctx context.Context, stats := map[string]*model.JobsStatistics{} var castType string - if r.driver == "sqlite3" { + switch r.driver { + case "sqlite3": castType = "int" - } else if r.driver == "mysql" { + case "mysql": castType = "unsigned" } @@ -890,7 +967,6 @@ func (r *JobRepository) jobsStatisticsHistogram(ctx context.Context, value string, filters []*model.JobFilter, id, col string) ([]*model.HistoPoint, error) { start := time.Now() - query := sq.Select(value, "COUNT(job.id) AS count").From("job") query, qerr := SecurityCheck(ctx, sq.Select(value, "COUNT(job.id) AS count").From("job")) if qerr != nil { @@ -924,3 +1000,121 @@ func (r *JobRepository) jobsStatisticsHistogram(ctx context.Context, log.Infof("Timer jobsStatisticsHistogram %s", time.Since(start)) return points, nil } + +const NamedJobInsert string = `INSERT INTO job ( + job_id, user, project, cluster, subcluster, ` + "`partition`" + `, array_job_id, num_nodes, num_hwthreads, num_acc, + exclusive, monitoring_status, smt, job_state, start_time, duration, walltime, resources, meta_data, + mem_used_max, flops_any_avg, mem_bw_avg, load_avg, net_bw_avg, net_data_vol_total, file_bw_avg, file_data_vol_total +) VALUES ( + :job_id, :user, :project, :cluster, :subcluster, :partition, :array_job_id, :num_nodes, :num_hwthreads, :num_acc, + :exclusive, :monitoring_status, :smt, :job_state, :start_time, :duration, :walltime, :resources, :meta_data, + :mem_used_max, :flops_any_avg, :mem_bw_avg, :load_avg, :net_bw_avg, :net_data_vol_total, :file_bw_avg, :file_data_vol_total +);` + +func (r *JobRepository) InsertJob(job *schema.Job) (int64, error) { + res, err := r.DB.NamedExec(NamedJobInsert, job) + if err != nil { + log.Warn("Error while NamedJobInsert") + return 0, err + } + id, err := res.LastInsertId() + if err != nil { + log.Warn("Error while getting last insert ID") + return 0, err + } + + return id, nil +} + +type Transaction struct { + tx *sqlx.Tx + stmt *sqlx.NamedStmt +} + +func (r *JobRepository) TransactionInit() (*Transaction, error) { + var err error + t := new(Transaction) + // Inserts are bundled into transactions because in sqlite, + // that speeds up inserts A LOT. + t.tx, err = r.DB.Beginx() + if err != nil { + log.Warn("Error while bundling transactions") + return nil, err + } + + t.stmt, err = t.tx.PrepareNamed(NamedJobInsert) + if err != nil { + log.Warn("Error while preparing namedJobInsert") + return nil, err + } + + return t, nil +} + +func (r *JobRepository) TransactionCommit(t *Transaction) error { + var err error + if t.tx != nil { + if err = t.tx.Commit(); err != nil { + log.Warn("Error while committing transactions") + return err + } + } + + t.tx, err = r.DB.Beginx() + if err != nil { + log.Warn("Error while bundling transactions") + return err + } + + t.stmt = t.tx.NamedStmt(t.stmt) + return nil +} + +func (r *JobRepository) TransactionEnd(t *Transaction) error { + if err := t.tx.Commit(); err != nil { + log.Warn("Error while committing SQL transactions") + return err + } + + return nil +} + +func (r *JobRepository) TransactionAdd(t *Transaction, job schema.Job) (int64, error) { + res, err := t.stmt.Exec(job) + if err != nil { + log.Errorf("repository initDB(): %v", err) + return 0, err + } + + id, err := res.LastInsertId() + if err != nil { + log.Errorf("repository initDB(): %v", err) + return 0, err + } + + return id, nil +} + +func (r *JobRepository) TransactionAddTag(t *Transaction, tag *schema.Tag) (int64, error) { + res, err := t.tx.Exec(`INSERT INTO tag (tag_name, tag_type) VALUES (?, ?)`, tag.Name, tag.Type) + if err != nil { + log.Errorf("Error while inserting tag into tag table: %v (Type %v)", tag.Name, tag.Type) + return 0, err + } + tagId, err := res.LastInsertId() + if err != nil { + log.Warn("Error while getting last insert ID") + return 0, err + } + + return tagId, nil +} + +func (r *JobRepository) TransactionSetTag(t *Transaction, jobId int64, tagId int64) error { + if _, err := t.tx.Exec(`INSERT INTO jobtag (job_id, tag_id) VALUES (?, ?)`, jobId, tagId); err != nil { + log.Errorf("Error while inserting jobtag into jobtag table: %v (TagID %v)", jobId, tagId) + return err + } + + return nil +} diff --git a/internal/repository/job_test.go b/internal/repository/job_test.go index 6370581..d74dad5 100644 --- a/internal/repository/job_test.go +++ b/internal/repository/job_test.go @@ -12,19 +12,21 @@ import ( _ "github.com/mattn/go-sqlite3" ) -func init() { - log.Init("info", true) - Connect("sqlite3", "../../test/test.db") -} - func setup(t *testing.T) *JobRepository { + log.Init("info", true) + dbfilepath := "testdata/test.db" + err := MigrateDB("sqlite3", dbfilepath) + if err != nil { + t.Fatal(err) + } + Connect("sqlite3", dbfilepath) return GetJobRepository() } func TestFind(t *testing.T) { r := setup(t) - jobId, cluster, startTime := int64(1404396), "emmy", int64(1609299584) + jobId, cluster, startTime := int64(398998), "fritz", int64(1675957496) job, err := r.Find(&jobId, &cluster, &startTime) if err != nil { t.Fatal(err) @@ -32,7 +34,7 @@ func TestFind(t *testing.T) { // fmt.Printf("%+v", job) - if job.ID != 1366 { + if job.ID != 5 { t.Errorf("wrong summary for diagnostic 3\ngot: %d \nwant: 1366", job.JobID) } } @@ -40,14 +42,14 @@ func TestFind(t *testing.T) { func TestFindById(t *testing.T) { r := setup(t) - job, err := r.FindById(1366) + job, err := r.FindById(5) if err != nil { t.Fatal(err) } // fmt.Printf("%+v", job) - if job.JobID != 1404396 { + if job.JobID != 398998 { t.Errorf("wrong summary for diagnostic 3\ngot: %d \nwant: 1404396", job.JobID) } } @@ -63,7 +65,7 @@ func TestGetTags(t *testing.T) { fmt.Printf("TAGS %+v \n", tags) // fmt.Printf("COUNTS %+v \n", counts) - if counts["bandwidth"] != 6 { - t.Errorf("wrong summary for diagnostic 3\ngot: %d \nwant: 6", counts["load-imbalance"]) + if counts["bandwidth"] != 3 { + t.Errorf("wrong tag count \ngot: %d \nwant: 3", counts["bandwidth"]) } } diff --git a/internal/repository/migration.go b/internal/repository/migration.go index 07e1c90..f7e0593 100644 --- a/internal/repository/migration.go +++ b/internal/repository/migration.go @@ -8,7 +8,6 @@ import ( "database/sql" "embed" "fmt" - "os" "github.com/ClusterCockpit/cc-backend/pkg/log" "github.com/golang-migrate/migrate/v4" @@ -22,37 +21,37 @@ const Version uint = 3 //go:embed migrations/* var migrationFiles embed.FS -func checkDBVersion(backend string, db *sql.DB) { +func checkDBVersion(backend string, db *sql.DB) error { var m *migrate.Migrate - if backend == "sqlite3" { - + switch backend { + case "sqlite3": driver, err := sqlite3.WithInstance(db, &sqlite3.Config{}) if err != nil { - log.Fatal(err) + return err } d, err := iofs.New(migrationFiles, "migrations/sqlite3") if err != nil { - log.Fatal(err) + return err } m, err = migrate.NewWithInstance("iofs", d, "sqlite3", driver) if err != nil { - log.Fatal(err) + return err } - } else if backend == "mysql" { + case "mysql": driver, err := mysql.WithInstance(db, &mysql.Config{}) if err != nil { - log.Fatal(err) + return err } d, err := iofs.New(migrationFiles, "migrations/mysql") if err != nil { - log.Fatal(err) + return err } m, err = migrate.NewWithInstance("iofs", d, "mysql", driver) if err != nil { - log.Fatal(err) + return err } } @@ -61,25 +60,26 @@ func checkDBVersion(backend string, db *sql.DB) { if err == migrate.ErrNilVersion { log.Warn("Legacy database without version or missing database file!") } else { - log.Fatal(err) + return err } } if v < Version { - log.Warnf("Unsupported database version %d, need %d.\nPlease backup your database file and run cc-backend --migrate-db", v, Version) - os.Exit(0) + return fmt.Errorf("unsupported database version %d, need %d.\nPlease backup your database file and run cc-backend --migrate-db", v, Version) } if v > Version { - log.Warnf("Unsupported database version %d, need %d.\nPlease refer to documentation how to downgrade db with external migrate tool!", v, Version) - os.Exit(0) + return fmt.Errorf("unsupported database version %d, need %d.\nPlease refer to documentation how to downgrade db with external migrate tool", v, Version) } + + return nil } -func MigrateDB(backend string, db string) { +func MigrateDB(backend string, db string) error { var m *migrate.Migrate - if backend == "sqlite3" { + switch backend { + case "sqlite3": d, err := iofs.New(migrationFiles, "migrations/sqlite3") if err != nil { log.Fatal(err) @@ -87,17 +87,17 @@ func MigrateDB(backend string, db string) { m, err = migrate.NewWithSourceInstance("iofs", d, fmt.Sprintf("sqlite3://%s?_foreign_keys=on", db)) if err != nil { - log.Fatal(err) + return err } - } else if backend == "mysql" { + case "mysql": d, err := iofs.New(migrationFiles, "migrations/mysql") if err != nil { - log.Fatal(err) + return err } m, err = migrate.NewWithSourceInstance("iofs", d, fmt.Sprintf("mysql://%s?multiStatements=true", db)) if err != nil { - log.Fatal(err) + return err } } @@ -105,9 +105,10 @@ func MigrateDB(backend string, db string) { if err == migrate.ErrNoChange { log.Info("DB already up to date!") } else { - log.Fatal(err) + return err } } m.Close() + return nil } diff --git a/internal/repository/migrations/mysql/01_init-schema.up.sql b/internal/repository/migrations/mysql/01_init-schema.up.sql index 4ae85a9..d3369fc 100644 --- a/internal/repository/migrations/mysql/01_init-schema.up.sql +++ b/internal/repository/migrations/mysql/01_init-schema.up.sql @@ -31,13 +31,15 @@ CREATE TABLE IF NOT EXISTS job ( net_bw_avg REAL NOT NULL DEFAULT 0.0, net_data_vol_total REAL NOT NULL DEFAULT 0.0, file_bw_avg REAL NOT NULL DEFAULT 0.0, - file_data_vol_total REAL NOT NULL DEFAULT 0.0); + file_data_vol_total REAL NOT NULL DEFAULT 0.0, + UNIQUE (job_id, cluster, start_time) + ); CREATE TABLE IF NOT EXISTS tag ( id INTEGER PRIMARY KEY, tag_type VARCHAR(255) NOT NULL, tag_name VARCHAR(255) NOT NULL, - CONSTRAINT be_unique UNIQUE (tag_type, tag_name)); + UNIQUE (tag_type, tag_name)); CREATE TABLE IF NOT EXISTS jobtag ( job_id INTEGER, diff --git a/internal/repository/migrations/sqlite3/01_init-schema.up.sql b/internal/repository/migrations/sqlite3/01_init-schema.up.sql index 01761f9..eab2d8d 100644 --- a/internal/repository/migrations/sqlite3/01_init-schema.up.sql +++ b/internal/repository/migrations/sqlite3/01_init-schema.up.sql @@ -7,19 +7,19 @@ CREATE TABLE IF NOT EXISTS job ( user VARCHAR(255) NOT NULL, project VARCHAR(255) NOT NULL, - partition VARCHAR(255) NOT NULL, - array_job_id BIGINT NOT NULL, - duration INT NOT NULL DEFAULT 0, - walltime INT NOT NULL DEFAULT 0, - job_state VARCHAR(255) NOT NULL + partition VARCHAR(255), + array_job_id BIGINT, + duration INT NOT NULL, + walltime INT NOT NULL, + job_state VARCHAR(255) NOT NULL CHECK(job_state IN ('running', 'completed', 'failed', 'cancelled', 'stopped', 'timeout', 'preempted', 'out_of_memory')), meta_data TEXT, -- JSON resources TEXT NOT NULL, -- JSON num_nodes INT NOT NULL, - num_hwthreads INT NOT NULL, - num_acc INT NOT NULL, + num_hwthreads INT, + num_acc INT, smt TINYINT NOT NULL DEFAULT 1 CHECK(smt IN (0, 1 )), exclusive TINYINT NOT NULL DEFAULT 1 CHECK(exclusive IN (0, 1, 2)), monitoring_status TINYINT NOT NULL DEFAULT 1 CHECK(monitoring_status IN (0, 1, 2, 3)), @@ -31,13 +31,15 @@ CREATE TABLE IF NOT EXISTS job ( net_bw_avg REAL NOT NULL DEFAULT 0.0, net_data_vol_total REAL NOT NULL DEFAULT 0.0, file_bw_avg REAL NOT NULL DEFAULT 0.0, - file_data_vol_total REAL NOT NULL DEFAULT 0.0); + file_data_vol_total REAL NOT NULL DEFAULT 0.0, + UNIQUE (job_id, cluster, start_time) + ); CREATE TABLE IF NOT EXISTS tag ( id INTEGER PRIMARY KEY, tag_type VARCHAR(255) NOT NULL, tag_name VARCHAR(255) NOT NULL, - CONSTRAINT be_unique UNIQUE (tag_type, tag_name)); + UNIQUE (tag_type, tag_name)); CREATE TABLE IF NOT EXISTS jobtag ( job_id INTEGER, diff --git a/internal/repository/query.go b/internal/repository/query.go index 50585b3..1fbced7 100644 --- a/internal/repository/query.go +++ b/internal/repository/query.go @@ -34,11 +34,13 @@ func (r *JobRepository) QueryJobs( if order != nil { field := toSnakeCase(order.Field) - if order.Order == model.SortDirectionEnumAsc { + + switch order.Order { + case model.SortDirectionEnumAsc: query = query.OrderBy(fmt.Sprintf("job.%s ASC", field)) - } else if order.Order == model.SortDirectionEnumDesc { + case model.SortDirectionEnumDesc: query = query.OrderBy(fmt.Sprintf("job.%s DESC", field)) - } else { + default: return nil, errors.New("REPOSITORY/QUERY > invalid sorting order") } } @@ -159,7 +161,7 @@ func SecurityCheck(ctx context.Context, query sq.SelectBuilder) (queryOut sq.Sel return query.Where("job.user = ?", user.Username), nil } else { // Unauthorized : Error var qnil sq.SelectBuilder - return qnil, errors.New(fmt.Sprintf("User '%s' with unknown roles! [%#v]\n", user.Username, user.Roles)) + return qnil, fmt.Errorf("user '%s' with unknown roles [%#v]", user.Username, user.Roles) } } diff --git a/internal/repository/testdata/test.db b/internal/repository/testdata/test.db new file mode 100644 index 0000000..e2dd7de Binary files /dev/null and b/internal/repository/testdata/test.db differ diff --git a/internal/repository/user_test.go b/internal/repository/user_test.go index ba1ec38..f5987ac 100644 --- a/internal/repository/user_test.go +++ b/internal/repository/user_test.go @@ -11,12 +11,10 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/auth" "github.com/ClusterCockpit/cc-backend/internal/config" + "github.com/ClusterCockpit/cc-backend/pkg/log" + _ "github.com/mattn/go-sqlite3" ) -func init() { - Connect("sqlite3", "../../test/test.db") -} - func setupUserTest(t *testing.T) *UserCfgRepo { const testconfig = `{ "addr": "0.0.0.0:8080", @@ -34,6 +32,15 @@ func setupUserTest(t *testing.T) *UserCfgRepo { "startTime": { "from": "2022-01-01T00:00:00Z", "to": null } } } ] }` + + log.Init("info", true) + dbfilepath := "testdata/test.db" + err := MigrateDB("sqlite3", dbfilepath) + if err != nil { + t.Fatal(err) + } + Connect("sqlite3", dbfilepath) + tmpdir := t.TempDir() cfgFilePath := filepath.Join(tmpdir, "config.json") if err := os.WriteFile(cfgFilePath, []byte(testconfig), 0666); err != nil { @@ -43,9 +50,10 @@ func setupUserTest(t *testing.T) *UserCfgRepo { config.Init(cfgFilePath) return GetUserCfgRepo() } + func TestGetUIConfig(t *testing.T) { r := setupUserTest(t) - u := auth.User{Username: "jan"} + u := auth.User{Username: "demo"} cfg, err := r.GetUIConfig(&u) if err != nil { @@ -53,10 +61,9 @@ func TestGetUIConfig(t *testing.T) { } tmp := cfg["plot_list_selectedMetrics"] - metrics := tmp.([]interface{}) - - str := metrics[2].(string) - if str != "mem_bw" { + metrics := tmp.([]string) + str := metrics[2] + if str != "mem_used" { t.Errorf("wrong config\ngot: %s \nwant: mem_bw", str) } } diff --git a/internal/runtimeEnv/setup.go b/internal/runtimeEnv/setup.go index a98bf39..5407a0e 100644 --- a/internal/runtimeEnv/setup.go +++ b/internal/runtimeEnv/setup.go @@ -24,7 +24,7 @@ import ( func LoadEnv(file string) error { f, err := os.Open(file) if err != nil { - log.Error("Error while opening file") + log.Error("Error while opening .env file") return err } diff --git a/internal/util/compress.go b/internal/util/compress.go new file mode 100644 index 0000000..0930f7e --- /dev/null +++ b/internal/util/compress.go @@ -0,0 +1,77 @@ +// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +package util + +import ( + "compress/gzip" + "io" + "os" + + "github.com/ClusterCockpit/cc-backend/pkg/log" +) + +func CompressFile(fileIn string, fileOut string) error { + originalFile, err := os.Open(fileIn) + if err != nil { + log.Errorf("CompressFile() error: %v", err) + return err + } + defer originalFile.Close() + + gzippedFile, err := os.Create(fileOut) + + if err != nil { + log.Errorf("CompressFile() error: %v", err) + return err + } + defer gzippedFile.Close() + + gzipWriter := gzip.NewWriter(gzippedFile) + defer gzipWriter.Close() + + _, err = io.Copy(gzipWriter, originalFile) + if err != nil { + log.Errorf("CompressFile() error: %v", err) + return err + } + gzipWriter.Flush() + if err := os.Remove(fileIn); err != nil { + log.Errorf("CompressFile() error: %v", err) + return err + } + + return nil +} + +func UncompressFile(fileIn string, fileOut string) error { + gzippedFile, err := os.Open(fileIn) + if err != nil { + log.Errorf("UncompressFile() error: %v", err) + return err + } + defer gzippedFile.Close() + + gzipReader, _ := gzip.NewReader(gzippedFile) + defer gzipReader.Close() + + uncompressedFile, err := os.Create(fileOut) + if err != nil { + log.Errorf("UncompressFile() error: %v", err) + return err + } + defer uncompressedFile.Close() + + _, err = io.Copy(uncompressedFile, gzipReader) + if err != nil { + log.Errorf("UncompressFile() error: %v", err) + return err + } + if err := os.Remove(fileIn); err != nil { + log.Errorf("UncompressFile() error: %v", err) + return err + } + + return nil +} diff --git a/internal/util/copy.go b/internal/util/copy.go new file mode 100644 index 0000000..3527e1e --- /dev/null +++ b/internal/util/copy.go @@ -0,0 +1,107 @@ +// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +package util + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" +) + +func CopyFile(src, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return + } + defer in.Close() + + out, err := os.Create(dst) + if err != nil { + return + } + defer func() { + if e := out.Close(); e != nil { + err = e + } + }() + + _, err = io.Copy(out, in) + if err != nil { + return + } + + err = out.Sync() + if err != nil { + return + } + + si, err := os.Stat(src) + if err != nil { + return + } + err = os.Chmod(dst, si.Mode()) + if err != nil { + return + } + + return +} + +func CopyDir(src string, dst string) (err error) { + src = filepath.Clean(src) + dst = filepath.Clean(dst) + + si, err := os.Stat(src) + if err != nil { + return err + } + if !si.IsDir() { + return fmt.Errorf("source is not a directory") + } + + _, err = os.Stat(dst) + if err != nil && !os.IsNotExist(err) { + return + } + if err == nil { + return fmt.Errorf("destination already exists") + } + + err = os.MkdirAll(dst, si.Mode()) + if err != nil { + return + } + + entries, err := ioutil.ReadDir(src) + if err != nil { + return + } + + for _, entry := range entries { + srcPath := filepath.Join(src, entry.Name()) + dstPath := filepath.Join(dst, entry.Name()) + + if entry.IsDir() { + err = CopyDir(srcPath, dstPath) + if err != nil { + return + } + } else { + // Skip symlinks. + if entry.Mode()&os.ModeSymlink != 0 { + continue + } + + err = CopyFile(srcPath, dstPath) + if err != nil { + return + } + } + } + + return +} diff --git a/internal/util/diskUsage.go b/internal/util/diskUsage.go new file mode 100644 index 0000000..8c70201 --- /dev/null +++ b/internal/util/diskUsage.go @@ -0,0 +1,34 @@ +// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +package util + +import ( + "os" + + "github.com/ClusterCockpit/cc-backend/pkg/log" +) + +func DiskUsage(dirpath string) float64 { + var size int64 + + dir, err := os.Open(dirpath) + if err != nil { + log.Errorf("DiskUsage() error: %v", err) + return 0 + } + defer dir.Close() + + files, err := dir.Readdir(-1) + if err != nil { + log.Errorf("DiskUsage() error: %v", err) + return 0 + } + + for _, file := range files { + size += file.Size() + } + + return float64(size) * 1e-6 +} diff --git a/internal/util/fstat.go b/internal/util/fstat.go new file mode 100644 index 0000000..3361e39 --- /dev/null +++ b/internal/util/fstat.go @@ -0,0 +1,34 @@ +// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +package util + +import ( + "errors" + "os" + + "github.com/ClusterCockpit/cc-backend/pkg/log" +) + +func CheckFileExists(filePath string) bool { + _, err := os.Stat(filePath) + return !errors.Is(err, os.ErrNotExist) +} + +func GetFilesize(filePath string) int64 { + fileInfo, err := os.Stat(filePath) + if err != nil { + log.Errorf("Error on Stat %s: %v", filePath, err) + } + return fileInfo.Size() +} + +func GetFilecount(path string) int { + files, err := os.ReadDir(path) + if err != nil { + log.Errorf("Error on ReadDir %s: %v", path, err) + } + + return len(files) +} diff --git a/internal/util/statistics.go b/internal/util/statistics.go new file mode 100644 index 0000000..ca84dac --- /dev/null +++ b/internal/util/statistics.go @@ -0,0 +1,21 @@ +// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +package util + +import "golang.org/x/exp/constraints" + +func Min[T constraints.Ordered](a, b T) T { + if a < b { + return a + } + return b +} + +func Max[T constraints.Ordered](a, b T) T { + if a > b { + return a + } + return b +} diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go index 6b0671b..17211c9 100644 --- a/pkg/archive/archive.go +++ b/pkg/archive/archive.go @@ -18,6 +18,10 @@ const Version uint64 = 1 type ArchiveBackend interface { Init(rawConfig json.RawMessage) (uint64, error) + Info() + + Exists(job *schema.Job) bool + LoadJobMeta(job *schema.Job) (*schema.JobMeta, error) LoadJobData(job *schema.Job) (schema.JobData, error) @@ -30,6 +34,14 @@ type ArchiveBackend interface { GetClusters() []string + CleanUp(jobs []*schema.Job) + + Move(jobs []*schema.Job, path string) + + Clean(before int64, after int64) + + Compress(jobs []*schema.Job) + Iter(loadMetricData bool) <-chan JobContainer } @@ -44,21 +56,23 @@ var useArchive bool func Init(rawConfig json.RawMessage, disableArchive bool) error { useArchive = !disableArchive - var kind struct { + + var cfg struct { Kind string `json:"kind"` } - if err := json.Unmarshal(rawConfig, &kind); err != nil { + + if err := json.Unmarshal(rawConfig, &cfg); err != nil { log.Warn("Error while unmarshaling raw config json") return err } - switch kind.Kind { + switch cfg.Kind { case "file": ar = &FsArchive{} // case "s3": // ar = &S3Archive{} default: - return fmt.Errorf("ARCHIVE/ARCHIVE > unkown archive backend '%s''", kind.Kind) + return fmt.Errorf("ARCHIVE/ARCHIVE > unkown archive backend '%s''", cfg.Kind) } version, err := ar.Init(rawConfig) @@ -67,6 +81,7 @@ func Init(rawConfig json.RawMessage, disableArchive bool) error { return err } log.Infof("Load archive version %d", version) + return initClusterConfig() } diff --git a/pkg/archive/archive_test.go b/pkg/archive/archive_test.go new file mode 100644 index 0000000..b41a033 --- /dev/null +++ b/pkg/archive/archive_test.go @@ -0,0 +1,69 @@ +// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +package archive_test + +import ( + "encoding/json" + "fmt" + "path/filepath" + "testing" + "time" + + "github.com/ClusterCockpit/cc-backend/internal/util" + "github.com/ClusterCockpit/cc-backend/pkg/archive" + "github.com/ClusterCockpit/cc-backend/pkg/schema" +) + +var jobs []*schema.Job + +func setup(t *testing.T) archive.ArchiveBackend { + tmpdir := t.TempDir() + jobarchive := filepath.Join(tmpdir, "job-archive") + util.CopyDir("./testdata/archive/", jobarchive) + archiveCfg := fmt.Sprintf("{\"kind\": \"file\",\"path\": \"%s\"}", jobarchive) + + if err := archive.Init(json.RawMessage(archiveCfg), false); err != nil { + t.Fatal(err) + } + + jobs = make([]*schema.Job, 2) + jobs[0] = &schema.Job{} + jobs[0].JobID = 1403244 + jobs[0].Cluster = "emmy" + jobs[0].StartTime = time.Unix(1608923076, 0) + + jobs[1] = &schema.Job{} + jobs[0].JobID = 1404397 + jobs[0].Cluster = "emmy" + jobs[0].StartTime = time.Unix(1609300556, 0) + + return archive.GetHandle() +} + +func TestCleanUp(t *testing.T) { + a := setup(t) + if !a.Exists(jobs[0]) { + t.Error("Job does not exist") + } + + a.CleanUp(jobs) + + if a.Exists(jobs[0]) || a.Exists(jobs[1]) { + t.Error("Jobs still exist") + } +} + +// func TestCompress(t *testing.T) { +// a := setup(t) +// if !a.Exists(jobs[0]) { +// t.Error("Job does not exist") +// } +// +// a.Compress(jobs) +// +// if a.Exists(jobs[0]) || a.Exists(jobs[1]) { +// t.Error("Jobs still exist") +// } +// } diff --git a/pkg/archive/fsBackend.go b/pkg/archive/fsBackend.go index 71d21ea..3825821 100644 --- a/pkg/archive/fsBackend.go +++ b/pkg/archive/fsBackend.go @@ -11,14 +11,17 @@ import ( "encoding/json" "errors" "fmt" + "math" "os" "path" "path/filepath" "strconv" "strings" + "text/tabwriter" "time" "github.com/ClusterCockpit/cc-backend/internal/config" + "github.com/ClusterCockpit/cc-backend/internal/util" "github.com/ClusterCockpit/cc-backend/pkg/log" "github.com/ClusterCockpit/cc-backend/pkg/schema" "github.com/santhosh-tekuri/jsonschema/v5" @@ -33,9 +36,17 @@ type FsArchive struct { clusters []string } -func checkFileExists(filePath string) bool { - _, err := os.Stat(filePath) - return !errors.Is(err, os.ErrNotExist) +func getDirectory( + job *schema.Job, + rootPath string, +) string { + lvl1, lvl2 := fmt.Sprintf("%d", job.JobID/1000), fmt.Sprintf("%03d", job.JobID%1000) + + return filepath.Join( + rootPath, + job.Cluster, + lvl1, lvl2, + strconv.FormatInt(job.StartTime.Unix(), 10)) } func getPath( @@ -43,12 +54,8 @@ func getPath( rootPath string, file string) string { - lvl1, lvl2 := fmt.Sprintf("%d", job.JobID/1000), fmt.Sprintf("%03d", job.JobID%1000) return filepath.Join( - rootPath, - job.Cluster, - lvl1, lvl2, - strconv.FormatInt(job.StartTime.Unix(), 10), file) + getDirectory(job, rootPath), file) } func loadJobMeta(filename string) (*schema.JobMeta, error) { @@ -74,6 +81,7 @@ func loadJobData(filename string, isCompressed bool) (schema.JobData, error) { log.Errorf("fsBackend LoadJobData()- %v", err) return nil, err } + defer f.Close() if isCompressed { r, err := gzip.NewReader(f) @@ -91,7 +99,6 @@ func loadJobData(filename string, isCompressed bool) (schema.JobData, error) { return DecodeJobData(r, filename) } else { - defer f.Close() if config.Keys.Validate { if err := schema.Validate(schema.Data, bufio.NewReader(f)); err != nil { return schema.JobData{}, fmt.Errorf("validate job data: %v", err) @@ -147,10 +154,205 @@ func (fsa *FsArchive) Init(rawConfig json.RawMessage) (uint64, error) { return version, nil } +type clusterInfo struct { + numJobs int + dateFirst int64 + dateLast int64 + diskSize float64 +} + +func (fsa *FsArchive) Info() { + fmt.Printf("Job archive %s\n", fsa.path) + clusters, err := os.ReadDir(fsa.path) + if err != nil { + log.Fatalf("Reading clusters failed: %s", err.Error()) + } + + ci := make(map[string]*clusterInfo) + + for _, cluster := range clusters { + if !cluster.IsDir() { + continue + } + + cc := cluster.Name() + ci[cc] = &clusterInfo{dateFirst: time.Now().Unix()} + lvl1Dirs, err := os.ReadDir(filepath.Join(fsa.path, cluster.Name())) + if err != nil { + log.Fatalf("Reading jobs failed @ lvl1 dirs: %s", err.Error()) + } + + for _, lvl1Dir := range lvl1Dirs { + if !lvl1Dir.IsDir() { + continue + } + lvl2Dirs, err := os.ReadDir(filepath.Join(fsa.path, cluster.Name(), lvl1Dir.Name())) + if err != nil { + log.Fatalf("Reading jobs failed @ lvl2 dirs: %s", err.Error()) + } + + for _, lvl2Dir := range lvl2Dirs { + dirpath := filepath.Join(fsa.path, cluster.Name(), lvl1Dir.Name(), lvl2Dir.Name()) + startTimeDirs, err := os.ReadDir(dirpath) + if err != nil { + log.Fatalf("Reading jobs failed @ starttime dirs: %s", err.Error()) + } + + for _, startTimeDir := range startTimeDirs { + if startTimeDir.IsDir() { + ci[cc].numJobs++ + startTime, err := strconv.ParseInt(startTimeDir.Name(), 10, 64) + if err != nil { + log.Fatalf("Cannot parse starttime: %s", err.Error()) + } + ci[cc].dateFirst = util.Min(ci[cc].dateFirst, startTime) + ci[cc].dateLast = util.Max(ci[cc].dateLast, startTime) + ci[cc].diskSize += util.DiskUsage(filepath.Join(dirpath, startTimeDir.Name())) + } + } + } + } + } + + cit := clusterInfo{dateFirst: time.Now().Unix()} + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', tabwriter.Debug) + fmt.Fprintln(w, "cluster\t#jobs\tfrom\tto\tdu (MB)") + for cluster, clusterInfo := range ci { + fmt.Fprintf(w, "%s\t%d\t%s\t%s\t%.2f\n", cluster, + clusterInfo.numJobs, + time.Unix(clusterInfo.dateFirst, 0), + time.Unix(clusterInfo.dateLast, 0), + clusterInfo.diskSize) + + cit.numJobs += clusterInfo.numJobs + cit.dateFirst = util.Min(cit.dateFirst, clusterInfo.dateFirst) + cit.dateLast = util.Max(cit.dateLast, clusterInfo.dateLast) + cit.diskSize += clusterInfo.diskSize + } + + fmt.Fprintf(w, "TOTAL\t%d\t%s\t%s\t%.2f\n", + cit.numJobs, time.Unix(cit.dateFirst, 0), time.Unix(cit.dateLast, 0), cit.diskSize) + w.Flush() +} + +func (fsa *FsArchive) Exists(job *schema.Job) bool { + dir := getDirectory(job, fsa.path) + _, err := os.Stat(dir) + return !errors.Is(err, os.ErrNotExist) +} + +func (fsa *FsArchive) Clean(before int64, after int64) { + + if after == 0 { + after = math.MaxInt64 + } + + clusters, err := os.ReadDir(fsa.path) + if err != nil { + log.Fatalf("Reading clusters failed: %s", err.Error()) + } + + for _, cluster := range clusters { + if !cluster.IsDir() { + continue + } + + lvl1Dirs, err := os.ReadDir(filepath.Join(fsa.path, cluster.Name())) + if err != nil { + log.Fatalf("Reading jobs failed @ lvl1 dirs: %s", err.Error()) + } + + for _, lvl1Dir := range lvl1Dirs { + if !lvl1Dir.IsDir() { + continue + } + lvl2Dirs, err := os.ReadDir(filepath.Join(fsa.path, cluster.Name(), lvl1Dir.Name())) + if err != nil { + log.Fatalf("Reading jobs failed @ lvl2 dirs: %s", err.Error()) + } + + for _, lvl2Dir := range lvl2Dirs { + dirpath := filepath.Join(fsa.path, cluster.Name(), lvl1Dir.Name(), lvl2Dir.Name()) + startTimeDirs, err := os.ReadDir(dirpath) + if err != nil { + log.Fatalf("Reading jobs failed @ starttime dirs: %s", err.Error()) + } + + for _, startTimeDir := range startTimeDirs { + if startTimeDir.IsDir() { + startTime, err := strconv.ParseInt(startTimeDir.Name(), 10, 64) + if err != nil { + log.Fatalf("Cannot parse starttime: %s", err.Error()) + } + + if startTime < before || startTime > after { + if err := os.RemoveAll(filepath.Join(dirpath, startTimeDir.Name())); err != nil { + log.Errorf("JobArchive Cleanup() error: %v", err) + } + } + } + } + if util.GetFilecount(dirpath) == 0 { + if err := os.Remove(dirpath); err != nil { + log.Errorf("JobArchive Clean() error: %v", err) + } + } + } + } + } +} + +func (fsa *FsArchive) Move(jobs []*schema.Job, path string) { + for _, job := range jobs { + source := getDirectory(job, fsa.path) + target := getDirectory(job, path) + + if err := os.MkdirAll(filepath.Clean(filepath.Join(target, "..")), 0777); err != nil { + log.Errorf("JobArchive Move MkDir error: %v", err) + } + if err := os.Rename(source, target); err != nil { + log.Errorf("JobArchive Move() error: %v", err) + } + + parent := filepath.Clean(filepath.Join(source, "..")) + if util.GetFilecount(parent) == 0 { + if err := os.Remove(parent); err != nil { + log.Errorf("JobArchive Move() error: %v", err) + } + } + } +} + +func (fsa *FsArchive) CleanUp(jobs []*schema.Job) { + for _, job := range jobs { + dir := getDirectory(job, fsa.path) + if err := os.RemoveAll(dir); err != nil { + log.Errorf("JobArchive Cleanup() error: %v", err) + } + + parent := filepath.Clean(filepath.Join(dir, "..")) + if util.GetFilecount(parent) == 0 { + if err := os.Remove(parent); err != nil { + log.Errorf("JobArchive Cleanup() error: %v", err) + } + } + } +} + +func (fsa *FsArchive) Compress(jobs []*schema.Job) { + for _, job := range jobs { + fileIn := getPath(job, fsa.path, "data.json") + if !util.CheckFileExists(fileIn) && util.GetFilesize(fileIn) > 2000 { + util.CompressFile(fileIn, getPath(job, fsa.path, "data.json.gz")) + } + } +} + func (fsa *FsArchive) LoadJobData(job *schema.Job) (schema.JobData, error) { var isCompressed bool = true filename := getPath(job, fsa.path, "data.json.gz") - if !checkFileExists(filename) { + + if !util.CheckFileExists(filename) { filename = getPath(job, fsa.path, "data.json") isCompressed = false } @@ -159,7 +361,6 @@ func (fsa *FsArchive) LoadJobData(job *schema.Job) (schema.JobData, error) { } func (fsa *FsArchive) LoadJobMeta(job *schema.Job) (*schema.JobMeta, error) { - filename := getPath(job, fsa.path, "meta.json") return loadJobMeta(filename) } @@ -226,7 +427,7 @@ func (fsa *FsArchive) Iter(loadMetricData bool) <-chan JobContainer { var isCompressed bool = true filename := filepath.Join(dirpath, startTimeDir.Name(), "data.json.gz") - if !checkFileExists(filename) { + if !util.CheckFileExists(filename) { filename = filepath.Join(dirpath, startTimeDir.Name(), "data.json") isCompressed = false } diff --git a/pkg/archive/fsBackend_test.go b/pkg/archive/fsBackend_test.go index 6e3cfc0..c5d869d 100644 --- a/pkg/archive/fsBackend_test.go +++ b/pkg/archive/fsBackend_test.go @@ -7,20 +7,17 @@ package archive import ( "encoding/json" "fmt" + "path/filepath" "testing" "time" - "github.com/ClusterCockpit/cc-backend/pkg/log" + "github.com/ClusterCockpit/cc-backend/internal/util" "github.com/ClusterCockpit/cc-backend/pkg/schema" ) -func init() { - log.Init("info", true) -} - func TestInitEmptyPath(t *testing.T) { var fsa FsArchive - _, err := fsa.Init(json.RawMessage("{\"kind\":\"../../test/archive\"}")) + _, err := fsa.Init(json.RawMessage("{\"kind\":\"testdata/archive\"}")) if err == nil { t.Fatal(err) } @@ -28,14 +25,14 @@ func TestInitEmptyPath(t *testing.T) { func TestInitNoJson(t *testing.T) { var fsa FsArchive - _, err := fsa.Init(json.RawMessage("\"path\":\"../../test/archive\"}")) + _, err := fsa.Init(json.RawMessage("\"path\":\"testdata/archive\"}")) if err == nil { t.Fatal(err) } } func TestInitNotExists(t *testing.T) { var fsa FsArchive - _, err := fsa.Init(json.RawMessage("{\"path\":\"../../test/job-archive\"}")) + _, err := fsa.Init(json.RawMessage("{\"path\":\"testdata/job-archive\"}")) if err == nil { t.Fatal(err) } @@ -43,11 +40,11 @@ func TestInitNotExists(t *testing.T) { func TestInit(t *testing.T) { var fsa FsArchive - version, err := fsa.Init(json.RawMessage("{\"path\":\"../../test/archive\"}")) + version, err := fsa.Init(json.RawMessage("{\"path\":\"testdata/archive\"}")) if err != nil { t.Fatal(err) } - if fsa.path != "../../test/archive" { + if fsa.path != "testdata/archive" { t.Fail() } if version != 1 { @@ -60,12 +57,12 @@ func TestInit(t *testing.T) { func TestLoadJobMetaInternal(t *testing.T) { var fsa FsArchive - _, err := fsa.Init(json.RawMessage("{\"path\":\"../../test/archive\"}")) + _, err := fsa.Init(json.RawMessage("{\"path\":\"testdata/archive\"}")) if err != nil { t.Fatal(err) } - job, err := loadJobMeta("../../test/archive/emmy/1404/397/1609300556/meta.json") + job, err := loadJobMeta("testdata/archive/emmy/1404/397/1609300556/meta.json") if err != nil { t.Fatal(err) } @@ -83,7 +80,7 @@ func TestLoadJobMetaInternal(t *testing.T) { func TestLoadJobMeta(t *testing.T) { var fsa FsArchive - _, err := fsa.Init(json.RawMessage("{\"path\":\"../../test/archive\"}")) + _, err := fsa.Init(json.RawMessage("{\"path\":\"testdata/archive\"}")) if err != nil { t.Fatal(err) } @@ -111,7 +108,7 @@ func TestLoadJobMeta(t *testing.T) { func TestLoadJobData(t *testing.T) { var fsa FsArchive - _, err := fsa.Init(json.RawMessage("{\"path\":\"../../test/archive\"}")) + _, err := fsa.Init(json.RawMessage("{\"path\": \"testdata/archive\"}")) if err != nil { t.Fatal(err) } @@ -126,8 +123,8 @@ func TestLoadJobData(t *testing.T) { t.Fatal(err) } - for name, scopes := range data { - fmt.Printf("Metric name: %s\n", name) + for _, scopes := range data { + // fmt.Printf("Metric name: %s\n", name) if _, exists := scopes[schema.MetricScopeNode]; !exists { t.Fail() @@ -135,9 +132,54 @@ func TestLoadJobData(t *testing.T) { } } +func BenchmarkLoadJobData(b *testing.B) { + + tmpdir := b.TempDir() + jobarchive := filepath.Join(tmpdir, "job-archive") + util.CopyDir("./testdata/archive/", jobarchive) + archiveCfg := fmt.Sprintf("{\"path\": \"%s\"}", jobarchive) + + var fsa FsArchive + fsa.Init(json.RawMessage(archiveCfg)) + + jobIn := schema.Job{BaseJob: schema.JobDefaults} + jobIn.StartTime = time.Unix(1608923076, 0) + jobIn.JobID = 1403244 + jobIn.Cluster = "emmy" + + util.UncompressFile(filepath.Join(jobarchive, "emmy/1403/244/1608923076/data.json.gz"), + filepath.Join(jobarchive, "emmy/1403/244/1608923076/data.json")) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + fsa.LoadJobData(&jobIn) + } +} + +func BenchmarkLoadJobDataCompressed(b *testing.B) { + + tmpdir := b.TempDir() + jobarchive := filepath.Join(tmpdir, "job-archive") + util.CopyDir("./testdata/archive/", jobarchive) + archiveCfg := fmt.Sprintf("{\"path\": \"%s\"}", jobarchive) + + var fsa FsArchive + fsa.Init(json.RawMessage(archiveCfg)) + + jobIn := schema.Job{BaseJob: schema.JobDefaults} + jobIn.StartTime = time.Unix(1608923076, 0) + jobIn.JobID = 1403244 + jobIn.Cluster = "emmy" + + b.ResetTimer() + for i := 0; i < b.N; i++ { + fsa.LoadJobData(&jobIn) + } +} + func TestLoadCluster(t *testing.T) { var fsa FsArchive - _, err := fsa.Init(json.RawMessage("{\"path\":\"../../test/archive\"}")) + _, err := fsa.Init(json.RawMessage("{\"path\":\"testdata/archive\"}")) if err != nil { t.Fatal(err) } @@ -154,7 +196,7 @@ func TestLoadCluster(t *testing.T) { func TestIter(t *testing.T) { var fsa FsArchive - _, err := fsa.Init(json.RawMessage("{\"path\":\"../../test/archive\"}")) + _, err := fsa.Init(json.RawMessage("{\"path\":\"testdata/archive\"}")) if err != nil { t.Fatal(err) } diff --git a/test/archive/emmy/1403/244/1608923076/data.json.gz b/pkg/archive/testdata/archive/emmy/1403/244/1608923076/data.json.gz similarity index 100% rename from test/archive/emmy/1403/244/1608923076/data.json.gz rename to pkg/archive/testdata/archive/emmy/1403/244/1608923076/data.json.gz diff --git a/test/archive/emmy/1403/244/1608923076/meta.json b/pkg/archive/testdata/archive/emmy/1403/244/1608923076/meta.json similarity index 100% rename from test/archive/emmy/1403/244/1608923076/meta.json rename to pkg/archive/testdata/archive/emmy/1403/244/1608923076/meta.json diff --git a/test/archive/emmy/1404/397/1609300556/data.json.gz b/pkg/archive/testdata/archive/emmy/1404/397/1609300556/data.json.gz similarity index 100% rename from test/archive/emmy/1404/397/1609300556/data.json.gz rename to pkg/archive/testdata/archive/emmy/1404/397/1609300556/data.json.gz diff --git a/test/archive/emmy/1404/397/1609300556/meta.json b/pkg/archive/testdata/archive/emmy/1404/397/1609300556/meta.json similarity index 100% rename from test/archive/emmy/1404/397/1609300556/meta.json rename to pkg/archive/testdata/archive/emmy/1404/397/1609300556/meta.json diff --git a/test/archive/emmy/cluster.json b/pkg/archive/testdata/archive/emmy/cluster.json similarity index 100% rename from test/archive/emmy/cluster.json rename to pkg/archive/testdata/archive/emmy/cluster.json diff --git a/test/archive/version.txt b/pkg/archive/testdata/archive/version.txt similarity index 100% rename from test/archive/version.txt rename to pkg/archive/testdata/archive/version.txt diff --git a/pkg/log/log.go b/pkg/log/log.go index 5fa7cd3..8240194 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -34,11 +34,11 @@ var ( ) var ( - DebugLog *log.Logger - InfoLog *log.Logger - WarnLog *log.Logger - ErrLog *log.Logger - CritLog *log.Logger + DebugLog *log.Logger = log.New(DebugWriter, DebugPrefix, log.LstdFlags) + InfoLog *log.Logger = log.New(InfoWriter, InfoPrefix, log.LstdFlags|log.Lshortfile) + WarnLog *log.Logger = log.New(WarnWriter, WarnPrefix, log.LstdFlags|log.Lshortfile) + ErrLog *log.Logger = log.New(ErrWriter, ErrPrefix, log.LstdFlags|log.Llongfile) + CritLog *log.Logger = log.New(CritWriter, CritPrefix, log.LstdFlags|log.Llongfile) ) /* CONFIG */ @@ -70,12 +70,6 @@ func Init(lvl string, logdate bool) { WarnLog = log.New(WarnWriter, WarnPrefix, log.Lshortfile) ErrLog = log.New(ErrWriter, ErrPrefix, log.Llongfile) CritLog = log.New(CritWriter, CritPrefix, log.Llongfile) - } else { - DebugLog = log.New(DebugWriter, DebugPrefix, log.LstdFlags) - InfoLog = log.New(InfoWriter, InfoPrefix, log.LstdFlags|log.Lshortfile) - WarnLog = log.New(WarnWriter, WarnPrefix, log.LstdFlags|log.Lshortfile) - ErrLog = log.New(ErrWriter, ErrPrefix, log.LstdFlags|log.Llongfile) - CritLog = log.New(CritWriter, CritPrefix, log.LstdFlags|log.Llongfile) } } diff --git a/pkg/schema/config.go b/pkg/schema/config.go index fa7ba9f..9a88ea2 100644 --- a/pkg/schema/config.go +++ b/pkg/schema/config.go @@ -57,6 +57,13 @@ type ClusterConfig struct { MetricDataRepository json.RawMessage `json:"metricDataRepository"` } +type Retention struct { + Age int `json:"age"` + IncludeDB bool `json:"includeDB"` + Policy string `json:"policy"` + Location string `json:"location"` +} + // Format of the configuration (file). See below for the defaults. type ProgramConfig struct { // Address where the http (or https) server will listen on (for example: 'localhost:80'). diff --git a/pkg/schema/job.go b/pkg/schema/job.go index 702f992..3f75551 100644 --- a/pkg/schema/job.go +++ b/pkg/schema/job.go @@ -11,8 +11,9 @@ import ( "time" ) -// Non-Swaggered Comment: BaseJob -// Non-Swaggered Comment: Common subset of Job and JobMeta. Use one of those, not this type directly. +// BaseJob is the common part of the job metadata structs +// +// Common subset of Job and JobMeta. Use one of those, not this type directly. type BaseJob struct { // The unique identifier of a job @@ -21,17 +22,17 @@ type BaseJob struct { Project string `json:"project" db:"project" example:"abcd200"` // The unique identifier of a project Cluster string `json:"cluster" db:"cluster" example:"fritz"` // The unique identifier of a cluster SubCluster string `json:"subCluster" db:"subcluster" example:"main"` // The unique identifier of a sub cluster - Partition *string `json:"partition,omitempty" db:"partition" example:"main"` // The Slurm partition to which the job was submitted - ArrayJobId *int64 `json:"arrayJobId,omitempty" db:"array_job_id" example:"123000"` // The unique identifier of an array job + Partition string `json:"partition,omitempty" db:"partition" example:"main"` // The Slurm partition to which the job was submitted + ArrayJobId int64 `json:"arrayJobId,omitempty" db:"array_job_id" example:"123000"` // The unique identifier of an array job NumNodes int32 `json:"numNodes" db:"num_nodes" example:"2" minimum:"1"` // Number of nodes used (Min > 0) - NumHWThreads *int32 `json:"numHwthreads,omitempty" db:"num_hwthreads" example:"20" minimum:"1"` // Number of HWThreads used (Min > 0) - NumAcc *int32 `json:"numAcc,omitempty" db:"num_acc" example:"2" minimum:"1"` // Number of accelerators used (Min > 0) + NumHWThreads int32 `json:"numHwthreads,omitempty" db:"num_hwthreads" example:"20" minimum:"1"` // Number of HWThreads used (Min > 0) + NumAcc int32 `json:"numAcc,omitempty" db:"num_acc" example:"2" minimum:"1"` // Number of accelerators used (Min > 0) Exclusive int32 `json:"exclusive" db:"exclusive" example:"1" minimum:"0" maximum:"2"` // Specifies how nodes are shared: 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple jobs of same user MonitoringStatus int32 `json:"monitoringStatus,omitempty" db:"monitoring_status" example:"1" minimum:"0" maximum:"3"` // State of monitoring system during job run: 0 - Disabled, 1 - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull - SMT *int32 `json:"smt,omitempty" db:"smt" example:"4"` // SMT threads used by job + SMT int32 `json:"smt,omitempty" db:"smt" example:"4"` // SMT threads used by job State JobState `json:"jobState" db:"job_state" example:"completed" enums:"completed,failed,cancelled,stopped,timeout,out_of_memory"` // Final state of job Duration int32 `json:"duration" db:"duration" example:"43200" minimum:"1"` // Duration of job in seconds (Min > 0) - Walltime *int64 `json:"walltime,omitempty" db:"walltime" example:"86400" minimum:"1"` // Requested walltime of job in seconds (Min > 0) + Walltime int64 `json:"walltime,omitempty" db:"walltime" example:"86400" minimum:"1"` // Requested walltime of job in seconds (Min > 0) Tags []*Tag `json:"tags,omitempty"` // List of tags RawResources []byte `json:"-" db:"resources"` // Resources used by job [As Bytes] Resources []*Resource `json:"resources"` // Resources used by job @@ -40,9 +41,10 @@ type BaseJob struct { ConcurrentJobs JobLinkResultList `json:"concurrentJobs"` } -// Non-Swaggered Comment: Job -// Non-Swaggered Comment: This type is used as the GraphQL interface and using sqlx as a table row. - +// Job struct type +// +// This type is used as the GraphQL interface and using sqlx as a table row. +// // Job model // @Description Information of a HPC job. type Job struct { @@ -61,6 +63,17 @@ type Job struct { FileDataVolTotal float64 `json:"-" db:"file_data_vol_total"` // FileDataVolTotal as Float64 } +// JobMeta struct type +// +// When reading from the database or sending data via GraphQL, the start time +// can be in the much more convenient time.Time type. In the `meta.json` +// files, the start time is encoded as a unix epoch timestamp. This is why +// there is this struct, which contains all fields from the regular job +// struct, but "overwrites" the StartTime field with one of type int64. ID +// *int64 `json:"id,omitempty"` >> never used in the job-archive, only +// available via REST-API +// + type JobLink struct { ID int64 `json:"id"` JobID int64 `json:"jobId"` @@ -71,13 +84,6 @@ type JobLinkResultList struct { Count int `json:"count"` } -// Non-Swaggered Comment: JobMeta -// Non-Swaggered Comment: When reading from the database or sending data via GraphQL, the start time can be in the much more -// Non-Swaggered Comment: convenient time.Time type. In the `meta.json` files, the start time is encoded as a unix epoch timestamp. -// Non-Swaggered Comment: This is why there is this struct, which contains all fields from the regular job struct, but "overwrites" -// Non-Swaggered Comment: the StartTime field with one of type int64. -// Non-Swaggered Comment: ID *int64 `json:"id,omitempty"` >> never used in the job-archive, only available via REST-API - // JobMeta model // @Description Meta data information of a HPC job. type JobMeta struct { @@ -85,7 +91,7 @@ type JobMeta struct { ID *int64 `json:"id,omitempty"` BaseJob StartTime int64 `json:"startTime" db:"start_time" example:"1649723812" minimum:"1"` // Start epoch time stamp in seconds (Min > 0) - Statistics map[string]JobStatistics `json:"statistics,omitempty"` // Metric statistics of job + Statistics map[string]JobStatistics `json:"statistics"` // Metric statistics of job } const ( @@ -101,8 +107,8 @@ var JobDefaults BaseJob = BaseJob{ } type Unit struct { - Base string `json:"base"` - Prefix *string `json:"prefix,omitempty"` + Base string `json:"base"` + Prefix string `json:"prefix,omitempty"` } // JobStatistics model diff --git a/pkg/schema/schemas/config.schema.json b/pkg/schema/schemas/config.schema.json index c05750c..6518281 100644 --- a/pkg/schema/schemas/config.schema.json +++ b/pkg/schema/schemas/config.schema.json @@ -3,7 +3,7 @@ "$id": "embedfs://config.schema.json", "title": "cc-backend configuration file schema", "type": "object", - "properties":{ + "properties": { "addr": { "description": "Address where the http (or https) server will listen on (for example: 'localhost:80').", "type": "string" @@ -41,8 +41,59 @@ "type": "string" }, "job-archive": { - "description": "Path to the job-archive.", - "type": "string" + "description": "Configuration keys for job-archive", + "type": "object", + "properties": { + "kind": { + "description": "Backend type for job-archive", + "type": "string", + "enum": [ + "file", + "s3" + ] + }, + "path": { + "description": "Path to job archive for file backend", + "type": "string" + }, + "compression": { + "description": "Setup automatic compression for jobs older than number of days", + "type": "integer" + }, + "retention": { + "description": "Configuration keys for retention", + "type": "object", + "properties": { + "policy": { + "description": "Retention policy", + "type": "string", + "enum": [ + "none", + "delete", + "move" + ] + }, + "includeDB": { + "description": "Also remove jobs from database", + "type": "boolean" + }, + "age": { + "description": "Act on jobs with startTime older than age (in days)", + "type": "integer" + }, + "location": { + "description": "The target directory for retention. Only applicable for retention move.", + "type": "string" + } + }, + "required": [ + "policy" + ] + } + }, + "required": [ + "kind" + ] }, "disable-archive": { "description": "Keep all metric data in the metric data repositories, do not write to the job-archive.", diff --git a/pkg/units/README.md b/pkg/units/README.md deleted file mode 100644 index 862239c..0000000 --- a/pkg/units/README.md +++ /dev/null @@ -1,175 +0,0 @@ -# cc-units - A unit system for ClusterCockpit - -When working with metrics, the problem comes up that they may use different unit name but have the same unit in fact. - There are a lot of real world examples like 'kB' and 'Kbyte'. In [cc-metric-collector](https://github.com/ClusterCockpit/cc-metric-collector), the collectors read data from different sources which may use different units or the programmer specifies a unit for a metric by hand. The cc-units system is not comparable with the SI unit system. If you are looking for a package for the SI units, see [here](https://pkg.go.dev/github.com/gurre/si). - -In order to enable unit comparison and conversion, the ccUnits package provides some helpers: -```go -NewUnit(unit string) Unit // create a new unit from some string like 'GHz', 'Mbyte' or 'kevents/s' -func GetUnitUnitFactor(in Unit, out Unit) (func(value float64) float64, error) // Get conversion function between two units -func GetPrefixFactor(in Prefix, out Prefix) func(value float64) float64 // Get conversion function between two prefixes -func GetUnitPrefixFactor(in Unit, out Prefix) (func(value float64) float64, Unit) // Get conversion function for prefix changes and the new unit for further use - -type Unit interface { - Valid() bool - String() string - Short() string - AddUnitDenominator(div Measure) -} -``` - -In order to get the "normalized" string unit back or test for validity, you can use: -```go -u := NewUnit("MB") -fmt.Println(u.Valid()) // true -fmt.Printf("Long string %q", u.String()) // MegaBytes -fmt.Printf("Short string %q", u.Short()) // MBytes -v := NewUnit("foo") -fmt.Println(v.Valid()) // false -``` - -If you have two units or other components and need the conversion function: -```go -// Get conversion functions for 'kB' to 'MBytes' -u1 := NewUnit("kB") -u2 := NewUnit("MBytes") -convFunc, err := GetUnitUnitFactor(u1, u2) // Returns an error if the units have different measures -if err == nil { - v2 := convFunc(v1) - fmt.Printf("%f %s\n", v2, u2.Short()) -} -// Get conversion function for 'kB' -> 'G' prefix. -// Returns the function and the new unit 'GBytes' -p1 := NewPrefix("G") -convFunc, u_p1 := GetUnitPrefixFactor(u1, p1) -// or -// convFunc, u_p1 := GetUnitPrefixStringFactor(u1, "G") -if convFunc != nil { - v2 := convFunc(v1) - fmt.Printf("%f %s\n", v2, u_p1.Short()) -} -// Get conversion function for two prefixes: 'G' -> 'T' -p2 := NewPrefix("T") -convFunc = GetPrefixPrefixFactor(p1, p2) -if convFunc != nil { - v2 := convFunc(v1) - fmt.Printf("%f %s -> %f %s\n", v1, p1.Prefix(), v2, p2.Prefix()) -} - - -``` - -(In the ClusterCockpit ecosystem the separation between values and units if useful since they are commonly not stored as a single entity but the value is a field in the CCMetric while unit is a tag or a meta information). - -If you have a metric and want the derivation to a bandwidth or events per second, you can use the original unit: - -```go -in_unit, err := metric.GetMeta("unit") -if err == nil { - value, ok := metric.GetField("value") - if ok { - out_unit = NewUnit(in_unit) - out_unit.AddUnitDenominator("seconds") - seconds := timeDiff.Seconds() - y, err := lp.New(metric.Name()+"_bw", - metric.Tags(), - metric.Meta(), - map[string]interface{"value": value/seconds}, - metric.Time()) - if err == nil { - y.AddMeta("unit", out_unit.Short()) - } - } -} -``` - -## Special unit detection - -Some used measures like Bytes and Flops are non-dividable. Consequently there prefixes like Milli, Micro and Nano are not useful. This is quite handy since a unit `mb` for `MBytes` is not uncommon but would by default be parsed as "MilliBytes". - -Special parsing rules for the following measures: iff `prefix==Milli`, use `prefix==Mega` - - `Bytes` - - `Flops` - - `Packets` - - `Events` - - `Cycles` - - `Requests` - -This means the prefixes `Micro` (like `ubytes`) and `Nano` like (`nflops/sec`) are not allowed and return an invalid unit. But you can specify `mflops` and `mb`. - -Prefixes for `%` or `percent` are ignored. - -## Supported prefixes - -```go -const ( - Base Prefix = 1 - Exa = 1e18 - Peta = 1e15 - Tera = 1e12 - Giga = 1e9 - Mega = 1e6 - Kilo = 1e3 - Milli = 1e-3 - Micro = 1e-6 - Nano = 1e-9 - Kibi = 1024 - Mebi = 1024 * 1024 - Gibi = 1024 * 1024 * 1024 - Tebi = 1024 * 1024 * 1024 * 1024 -) -``` - -The prefixes are detected using a regular expression `^([kKmMgGtTpP]?[i]?)(.*)` that splits the prefix from the measure. You probably don't need to deal with the prefixes in the code. - -## Supported measures - -```go -const ( - None Measure = iota - Bytes - Flops - Percentage - TemperatureC - TemperatureF - Rotation - Hertz - Time - Watt - Joule - Cycles - Requests - Packets - Events -) -``` - -There a regular expression for each of the measures like `^([bB][yY]?[tT]?[eE]?[sS]?)` for the `Bytes` measure. - - -## New units - -If the selected units are not suitable for your metric, feel free to send a PR. - -### New prefix - -For a new prefix, add it to the big `const` in `ccUnitPrefix.go` and adjust the prefix-unit-splitting regular expression. Afterwards, you have to add cases to the three functions `String()`, `Prefix()` and `NewPrefix()`. `NewPrefix()` contains the parser (`k` or `K` -> `Kilo`). The other one are used for output. `String()` outputs a longer version of the prefix (`Kilo`), while `Prefix()` returns only the short notation (`K`). - -### New measure - -Adding new prefixes is probably rare but adding a new measure is a more common task. At first, add it to the big `const` in `ccUnitMeasure.go`. Moreover, create a regular expression matching the measure (and pre-compile it like the others). Add the expression matching to `NewMeasure()`. The `String()` and `Short()` functions return descriptive strings for the measure in long form (like `Hertz`) and short form (like `Hz`). - -If there are special conversation rules between measures and you want to convert one measure to another, like temperatures in Celsius to Fahrenheit, a special case in `GetUnitPrefixFactor()` is required. - -### Special parsing rules - -The two parsers for prefix and measure are called under the hood by `NewUnit()` and there might some special rules apply. Like in the above section about 'special unit detection', special rules for your new measure might be required. Currently there are two special cases: - -- Measures that are non-dividable like Flops, Bytes, Events, ... cannot use `Milli`, `Micro` and `Nano`. The prefix `m` is forced to `M` for these measures -- If the prefix is `p`/`P` (`Peta`) or `e`/`E` (`Exa`) and the measure is not detectable, it retries detection with the prefix. So first round it tries, for example, prefix `p` and measure `ackets` which fails, so it retries the detection with measure `packets` and `` prefix (resolves to `Base` prefix). - -## Limitations - -The `ccUnits` package is a simple implemtation of a unit system and comes with some limitations: - -- The unit denominator (like `s` in `Mbyte/s`) can only have the `Base` prefix, you cannot specify `Byte/ms` for "Bytes per milli second". diff --git a/pkg/units/unitMeasure.go b/pkg/units/unitMeasure.go deleted file mode 100644 index 227e3c1..0000000 --- a/pkg/units/unitMeasure.go +++ /dev/null @@ -1,134 +0,0 @@ -package units - -import "regexp" - -type Measure int - -const ( - InvalidMeasure Measure = iota - Bytes - Flops - Percentage - TemperatureC - TemperatureF - Rotation - Frequency - Time - Watt - Joule - Cycles - Requests - Packets - Events -) - -type MeasureData struct { - Long string - Short string - Regex string -} - -// Different names and regex used for input and output -var InvalidMeasureLong string = "Invalid" -var InvalidMeasureShort string = "inval" -var MeasuresMap map[Measure]MeasureData = map[Measure]MeasureData{ - Bytes: { - Long: "byte", - Short: "B", - Regex: "^([bB][yY]?[tT]?[eE]?[sS]?)", - }, - Flops: { - Long: "Flops", - Short: "F", - Regex: "^([fF][lL]?[oO]?[pP]?[sS]?)", - }, - Percentage: { - Long: "Percent", - Short: "%", - Regex: "^(%|[pP]ercent)", - }, - TemperatureC: { - Long: "DegreeC", - Short: "degC", - Regex: "^(deg[Cc]|°[cC])", - }, - TemperatureF: { - Long: "DegreeF", - Short: "degF", - Regex: "^(deg[fF]|°[fF])", - }, - Rotation: { - Long: "RPM", - Short: "RPM", - Regex: "^([rR][pP][mM])", - }, - Frequency: { - Long: "Hertz", - Short: "Hz", - Regex: "^([hH][eE]?[rR]?[tT]?[zZ])", - }, - Time: { - Long: "Seconds", - Short: "s", - Regex: "^([sS][eE]?[cC]?[oO]?[nN]?[dD]?[sS]?)", - }, - Cycles: { - Long: "Cycles", - Short: "cyc", - Regex: "^([cC][yY][cC]?[lL]?[eE]?[sS]?)", - }, - Watt: { - Long: "Watts", - Short: "W", - Regex: "^([wW][aA]?[tT]?[tT]?[sS]?)", - }, - Joule: { - Long: "Joules", - Short: "J", - Regex: "^([jJ][oO]?[uU]?[lL]?[eE]?[sS]?)", - }, - Requests: { - Long: "Requests", - Short: "requests", - Regex: "^([rR][eE][qQ][uU]?[eE]?[sS]?[tT]?[sS]?)", - }, - Packets: { - Long: "Packets", - Short: "packets", - Regex: "^([pP][aA]?[cC]?[kK][eE]?[tT][sS]?)", - }, - Events: { - Long: "Events", - Short: "events", - Regex: "^([eE][vV]?[eE]?[nN][tT][sS]?)", - }, -} - -// String returns the long string for the measure like 'Percent' or 'Seconds' -func (m *Measure) String() string { - if data, ok := MeasuresMap[*m]; ok { - return data.Long - } - return InvalidMeasureLong -} - -// Short returns the short string for the measure like 'B' (Bytes), 's' (Time) or 'W' (Watt). Is is recommened to use Short() over String(). -func (m *Measure) Short() string { - if data, ok := MeasuresMap[*m]; ok { - return data.Short - } - return InvalidMeasureShort -} - -// NewMeasure creates a new measure out of a string representing a measure like 'Bytes', 'Flops' and 'precent'. -// It uses regular expressions for matching. -func NewMeasure(unit string) Measure { - for m, data := range MeasuresMap { - regex := regexp.MustCompile(data.Regex) - match := regex.FindStringSubmatch(unit) - if match != nil { - return m - } - } - return InvalidMeasure -} diff --git a/pkg/units/unitPrefix.go b/pkg/units/unitPrefix.go deleted file mode 100644 index 014fcc7..0000000 --- a/pkg/units/unitPrefix.go +++ /dev/null @@ -1,192 +0,0 @@ -package units - -import ( - "math" - "regexp" -) - -type Prefix float64 - -const ( - InvalidPrefix Prefix = iota - Base = 1 - Yotta = 1e24 - Zetta = 1e21 - Exa = 1e18 - Peta = 1e15 - Tera = 1e12 - Giga = 1e9 - Mega = 1e6 - Kilo = 1e3 - Milli = 1e-3 - Micro = 1e-6 - Nano = 1e-9 - Kibi = 1024 - Mebi = 1024 * 1024 - Gibi = 1024 * 1024 * 1024 - Tebi = 1024 * 1024 * 1024 * 1024 - Pebi = 1024 * 1024 * 1024 * 1024 * 1024 - Exbi = 1024 * 1024 * 1024 * 1024 * 1024 * 1024 - Zebi = 1024 * 1024 * 1024 * 1024 * 1024 * 1024 * 1024 - Yobi = 1024 * 1024 * 1024 * 1024 * 1024 * 1024 * 1024 * 1024 -) -const PrefixUnitSplitRegexStr = `^([kKmMgGtTpPeEzZyY]?[i]?)(.*)` - -var prefixUnitSplitRegex = regexp.MustCompile(PrefixUnitSplitRegexStr) - -type PrefixData struct { - Long string - Short string - Regex string -} - -// Different names and regex used for input and output -var InvalidPrefixLong string = "Invalid" -var InvalidPrefixShort string = "inval" -var PrefixDataMap map[Prefix]PrefixData = map[Prefix]PrefixData{ - Base: { - Long: "", - Short: "", - Regex: "^$", - }, - Kilo: { - Long: "Kilo", - Short: "K", - Regex: "^[kK]$", - }, - Mega: { - Long: "Mega", - Short: "M", - Regex: "^[M]$", - }, - Giga: { - Long: "Giga", - Short: "G", - Regex: "^[gG]$", - }, - Tera: { - Long: "Tera", - Short: "T", - Regex: "^[tT]$", - }, - Peta: { - Long: "Peta", - Short: "P", - Regex: "^[pP]$", - }, - Exa: { - Long: "Exa", - Short: "E", - Regex: "^[eE]$", - }, - Zetta: { - Long: "Zetta", - Short: "Z", - Regex: "^[zZ]$", - }, - Yotta: { - Long: "Yotta", - Short: "Y", - Regex: "^[yY]$", - }, - Milli: { - Long: "Milli", - Short: "m", - Regex: "^[m]$", - }, - Micro: { - Long: "Micro", - Short: "u", - Regex: "^[u]$", - }, - Nano: { - Long: "Nano", - Short: "n", - Regex: "^[n]$", - }, - Kibi: { - Long: "Kibi", - Short: "Ki", - Regex: "^[kK][i]$", - }, - Mebi: { - Long: "Mebi", - Short: "Mi", - Regex: "^[M][i]$", - }, - Gibi: { - Long: "Gibi", - Short: "Gi", - Regex: "^[gG][i]$", - }, - Tebi: { - Long: "Tebi", - Short: "Ti", - Regex: "^[tT][i]$", - }, - Pebi: { - Long: "Pebi", - Short: "Pi", - Regex: "^[pP][i]$", - }, - Exbi: { - Long: "Exbi", - Short: "Ei", - Regex: "^[eE][i]$", - }, - Zebi: { - Long: "Zebi", - Short: "Zi", - Regex: "^[zZ][i]$", - }, - Yobi: { - Long: "Yobi", - Short: "Yi", - Regex: "^[yY][i]$", - }, -} - -// String returns the long string for the prefix like 'Kilo' or 'Mega' -func (p *Prefix) String() string { - if data, ok := PrefixDataMap[*p]; ok { - return data.Long - } - return InvalidMeasureLong -} - -// Prefix returns the short string for the prefix like 'K', 'M' or 'G'. Is is recommened to use Prefix() over String(). -func (p *Prefix) Prefix() string { - if data, ok := PrefixDataMap[*p]; ok { - return data.Short - } - return InvalidMeasureShort -} - -// NewPrefix creates a new prefix out of a string representing a unit like 'k', 'K', 'M' or 'G'. -func NewPrefix(prefix string) Prefix { - for p, data := range PrefixDataMap { - regex := regexp.MustCompile(data.Regex) - match := regex.FindStringSubmatch(prefix) - if match != nil { - return p - } - } - return InvalidPrefix -} - -func getExponent(p float64) int { - count := 0 - - for p > 1.0 { - p = p / 1000.0 - count++ - } - - return count * 3 -} - -func NewPrefixFromFactor(op Prefix, e int) Prefix { - f := float64(op) - exp := math.Pow10(getExponent(f) - e) - return Prefix(exp) -} diff --git a/pkg/units/units.go b/pkg/units/units.go deleted file mode 100644 index 5598dfe..0000000 --- a/pkg/units/units.go +++ /dev/null @@ -1,339 +0,0 @@ -// Unit system for cluster monitoring metrics like bytes, flops and events -package units - -import ( - "fmt" - "math" - "strings" - - "github.com/ClusterCockpit/cc-backend/pkg/schema" -) - -type unit struct { - prefix Prefix - measure Measure - divMeasure Measure -} - -type Unit interface { - Valid() bool - String() string - Short() string - AddUnitDenominator(div Measure) - getPrefix() Prefix - getMeasure() Measure - getUnitDenominator() Measure - setPrefix(p Prefix) -} - -var INVALID_UNIT = NewUnit("foobar") - -// Valid checks whether a unit is a valid unit. -// A unit is valid if it has at least a prefix and a measure. -// The unit denominator is optional. -func (u *unit) Valid() bool { - return u.prefix != InvalidPrefix && u.measure != InvalidMeasure -} - -// String returns the long string for the unit like 'KiloHertz' or 'MegaBytes' -func (u *unit) String() string { - if u.divMeasure != InvalidMeasure { - return fmt.Sprintf("%s%s/%s", u.prefix.String(), u.measure.String(), u.divMeasure.String()) - } else { - return fmt.Sprintf("%s%s", u.prefix.String(), u.measure.String()) - } -} - -// Short returns the short string for the unit like 'kHz' or 'MByte'. Is is recommened to use Short() over String(). -func (u *unit) Short() string { - if u.divMeasure != InvalidMeasure { - return fmt.Sprintf("%s%s/%s", u.prefix.Prefix(), u.measure.Short(), u.divMeasure.Short()) - } else { - return fmt.Sprintf("%s%s", u.prefix.Prefix(), u.measure.Short()) - } -} - -// AddUnitDenominator adds a unit denominator to an exising unit. Can be used if you want to derive e.g. data volume to bandwidths. -// The data volume is in a Byte unit like 'kByte' and by dividing it by the runtime in seconds, we get the bandwidth. We can use the -// data volume unit and add 'Second' as unit denominator -func (u *unit) AddUnitDenominator(div Measure) { - u.divMeasure = div -} - -func (u *unit) getPrefix() Prefix { - return u.prefix -} - -func (u *unit) setPrefix(p Prefix) { - u.prefix = p -} - -func (u *unit) getMeasure() Measure { - return u.measure -} - -func (u *unit) getUnitDenominator() Measure { - return u.divMeasure -} - -func ConvertValue(v *float64, from string, to string) { - uf := NewUnit(from) - ut := NewUnit(to) - factor := float64(uf.getPrefix()) / float64(ut.getPrefix()) - *v = math.Ceil(*v * factor) -} - -func ConvertSeries(s []float64, from string, to string) { - uf := NewUnit(from) - ut := NewUnit(to) - factor := float64(uf.getPrefix()) / float64(ut.getPrefix()) - - for i := 0; i < len(s); i++ { - s[i] = math.Ceil(s[i] * factor) - } -} - -func getNormalizationFactor(v float64) (float64, int) { - count := 0 - scale := -3 - - if v > 1000.0 { - for v > 1000.0 { - v *= 1e-3 - count++ - } - } else { - for v < 1.0 { - v *= 1e3 - count++ - } - scale = 3 - } - return math.Pow10(count * scale), count * scale -} - -func NormalizeValue(v *float64, us string, nu *string) { - u := NewUnit(us) - f, e := getNormalizationFactor((*v)) - *v = math.Ceil(*v * f) - u.setPrefix(NewPrefixFromFactor(u.getPrefix(), e)) - *nu = u.Short() -} - -func NormalizeSeries(s []float64, avg float64, us string, nu *string) { - u := NewUnit(us) - f, e := getNormalizationFactor(avg) - - for i := 0; i < len(s); i++ { - s[i] *= f - s[i] = math.Ceil(s[i]) - } - u.setPrefix(NewPrefixFromFactor(u.getPrefix(), e)) - fmt.Printf("Prefix: %e \n", u.getPrefix()) - *nu = u.Short() -} - -func ConvertUnitString(us string) schema.Unit { - var nu schema.Unit - - if us == "CPI" || - us == "IPC" || - us == "load" || - us == "" { - nu.Base = us - return nu - } - u := NewUnit(us) - p := u.getPrefix() - if p.Prefix() != "" { - prefix := p.Prefix() - nu.Prefix = &prefix - } - m := u.getMeasure() - d := u.getUnitDenominator() - if d.Short() != "inval" { - nu.Base = fmt.Sprintf("%s/%s", m.Short(), d.Short()) - } else { - nu.Base = m.Short() - } - - return nu -} - -// GetPrefixPrefixFactor creates the default conversion function between two prefixes. -// It returns a conversation function for the value. -func GetPrefixPrefixFactor(in Prefix, out Prefix) func(value interface{}) interface{} { - var factor = 1.0 - var in_prefix = float64(in) - var out_prefix = float64(out) - factor = in_prefix / out_prefix - conv := func(value interface{}) interface{} { - switch v := value.(type) { - case float64: - return v * factor - case float32: - return float32(float64(v) * factor) - case int: - return int(float64(v) * factor) - case int32: - return int32(float64(v) * factor) - case int64: - return int64(float64(v) * factor) - case uint: - return uint(float64(v) * factor) - case uint32: - return uint32(float64(v) * factor) - case uint64: - return uint64(float64(v) * factor) - } - return value - } - return conv -} - -// This is the conversion function between temperatures in Celsius to Fahrenheit -func convertTempC2TempF(value interface{}) interface{} { - switch v := value.(type) { - case float64: - return (v * 1.8) + 32 - case float32: - return (v * 1.8) + 32 - case int: - return int((float64(v) * 1.8) + 32) - case int32: - return int32((float64(v) * 1.8) + 32) - case int64: - return int64((float64(v) * 1.8) + 32) - case uint: - return uint((float64(v) * 1.8) + 32) - case uint32: - return uint32((float64(v) * 1.8) + 32) - case uint64: - return uint64((float64(v) * 1.8) + 32) - } - return value -} - -// This is the conversion function between temperatures in Fahrenheit to Celsius -func convertTempF2TempC(value interface{}) interface{} { - switch v := value.(type) { - case float64: - return (v - 32) / 1.8 - case float32: - return (v - 32) / 1.8 - case int: - return int(((float64(v) - 32) / 1.8)) - case int32: - return int32(((float64(v) - 32) / 1.8)) - case int64: - return int64(((float64(v) - 32) / 1.8)) - case uint: - return uint(((float64(v) - 32) / 1.8)) - case uint32: - return uint32(((float64(v) - 32) / 1.8)) - case uint64: - return uint64(((float64(v) - 32) / 1.8)) - } - return value -} - -// GetPrefixStringPrefixStringFactor is a wrapper for GetPrefixPrefixFactor with string inputs instead -// of prefixes. It also returns a conversation function for the value. -func GetPrefixStringPrefixStringFactor(in string, out string) func(value interface{}) interface{} { - var i Prefix = NewPrefix(in) - var o Prefix = NewPrefix(out) - return GetPrefixPrefixFactor(i, o) -} - -// GetUnitPrefixFactor gets the conversion function and resulting unit for a unit and a prefix. This is -// the most common case where you have some input unit and want to convert it to the same unit but with -// a different prefix. The returned unit represents the value after conversation. -func GetUnitPrefixFactor(in Unit, out Prefix) (func(value interface{}) interface{}, Unit) { - outUnit := NewUnit(in.Short()) - if outUnit.Valid() { - outUnit.setPrefix(out) - conv := GetPrefixPrefixFactor(in.getPrefix(), out) - return conv, outUnit - } - return nil, INVALID_UNIT -} - -// GetUnitPrefixStringFactor gets the conversion function and resulting unit for a unit and a prefix as string. -// It is a wrapper for GetUnitPrefixFactor -func GetUnitPrefixStringFactor(in Unit, out string) (func(value interface{}) interface{}, Unit) { - var o Prefix = NewPrefix(out) - return GetUnitPrefixFactor(in, o) -} - -// GetUnitStringPrefixStringFactor gets the conversion function and resulting unit for a unit and a prefix when both are only string representations. -// This is just a wrapper for GetUnitPrefixFactor with the given input unit and the desired output prefix. -func GetUnitStringPrefixStringFactor(in string, out string) (func(value interface{}) interface{}, Unit) { - var i = NewUnit(in) - return GetUnitPrefixStringFactor(i, out) -} - -// GetUnitUnitFactor gets the conversion function and (maybe) error for unit to unit conversion. -// It is basically a wrapper for GetPrefixPrefixFactor with some special cases for temperature -// conversion between Fahrenheit and Celsius. -func GetUnitUnitFactor(in Unit, out Unit) (func(value interface{}) interface{}, error) { - if in.getMeasure() == TemperatureC && out.getMeasure() == TemperatureF { - return convertTempC2TempF, nil - } else if in.getMeasure() == TemperatureF && out.getMeasure() == TemperatureC { - return convertTempF2TempC, nil - } else if in.getMeasure() != out.getMeasure() || in.getUnitDenominator() != out.getUnitDenominator() { - return func(value interface{}) interface{} { return 1.0 }, fmt.Errorf("UNITS/UNITS > invalid measures in in and out Unit") - } - return GetPrefixPrefixFactor(in.getPrefix(), out.getPrefix()), nil -} - -// NewUnit creates a new unit out of a string representing a unit like 'Mbyte/s' or 'GHz'. -// It uses regular expressions to detect the prefix, unit and (maybe) unit denominator. -func NewUnit(unitStr string) Unit { - u := &unit{ - prefix: InvalidPrefix, - measure: InvalidMeasure, - divMeasure: InvalidMeasure, - } - matches := prefixUnitSplitRegex.FindStringSubmatch(unitStr) - if len(matches) > 2 { - pre := NewPrefix(matches[1]) - measures := strings.Split(matches[2], "/") - m := NewMeasure(measures[0]) - // Special case for prefix 'p' or 'P' (Peta) and measures starting with 'p' or 'P' - // like 'packets' or 'percent'. Same for 'e' or 'E' (Exa) for measures starting with - // 'e' or 'E' like 'events' - if m == InvalidMeasure { - switch pre { - case Peta, Exa: - t := NewMeasure(matches[1] + measures[0]) - if t != InvalidMeasure { - m = t - pre = Base - } - } - } - div := InvalidMeasure - if len(measures) > 1 { - div = NewMeasure(measures[1]) - } - - switch m { - // Special case for 'm' as prefix for Bytes and some others as thers is no unit like MilliBytes - case Bytes, Flops, Packets, Events, Cycles, Requests: - if pre == Milli { - pre = Mega - } - // Special case for percentage. No/ignore prefix - case Percentage: - pre = Base - } - if pre != InvalidPrefix && m != InvalidMeasure { - u.prefix = pre - u.measure = m - if div != InvalidMeasure { - u.divMeasure = div - } - } - } - return u -} diff --git a/pkg/units/units_test.go b/pkg/units/units_test.go deleted file mode 100644 index 078a6a0..0000000 --- a/pkg/units/units_test.go +++ /dev/null @@ -1,307 +0,0 @@ -package units - -import ( - "fmt" - "reflect" - "regexp" - "testing" -) - -func TestUnitsExact(t *testing.T) { - testCases := []struct { - in string - want Unit - }{ - {"b", NewUnit("Bytes")}, - {"B", NewUnit("Bytes")}, - {"byte", NewUnit("Bytes")}, - {"bytes", NewUnit("Bytes")}, - {"BYtes", NewUnit("Bytes")}, - {"Mb", NewUnit("MBytes")}, - {"MB", NewUnit("MBytes")}, - {"Mbyte", NewUnit("MBytes")}, - {"Mbytes", NewUnit("MBytes")}, - {"MbYtes", NewUnit("MBytes")}, - {"Gb", NewUnit("GBytes")}, - {"GB", NewUnit("GBytes")}, - {"Hz", NewUnit("Hertz")}, - {"MHz", NewUnit("MHertz")}, - {"GHz", NewUnit("GHertz")}, - {"pkts", NewUnit("Packets")}, - {"packets", NewUnit("Packets")}, - {"packet", NewUnit("Packets")}, - {"flop", NewUnit("Flops")}, - {"flops", NewUnit("Flops")}, - {"floPS", NewUnit("Flops")}, - {"Mflop", NewUnit("MFlops")}, - {"Gflop", NewUnit("GFlops")}, - {"gflop", NewUnit("GFlops")}, - {"%", NewUnit("Percent")}, - {"percent", NewUnit("Percent")}, - {"degc", NewUnit("degC")}, - {"degC", NewUnit("degC")}, - {"degf", NewUnit("degF")}, - {"°f", NewUnit("degF")}, - {"events", NewUnit("events")}, - {"event", NewUnit("events")}, - {"EveNts", NewUnit("events")}, - {"reqs", NewUnit("requests")}, - {"reQuEsTs", NewUnit("requests")}, - {"Requests", NewUnit("requests")}, - {"cyc", NewUnit("cycles")}, - {"cy", NewUnit("cycles")}, - {"Cycles", NewUnit("cycles")}, - {"J", NewUnit("Joules")}, - {"Joule", NewUnit("Joules")}, - {"joule", NewUnit("Joules")}, - {"W", NewUnit("Watt")}, - {"Watts", NewUnit("Watt")}, - {"watt", NewUnit("Watt")}, - {"s", NewUnit("seconds")}, - {"sec", NewUnit("seconds")}, - {"secs", NewUnit("seconds")}, - {"RPM", NewUnit("rpm")}, - {"rPm", NewUnit("rpm")}, - {"watt/byte", NewUnit("W/B")}, - {"watts/bytes", NewUnit("W/B")}, - {"flop/byte", NewUnit("flops/Bytes")}, - {"F/B", NewUnit("flops/Bytes")}, - } - compareUnitExact := func(in, out Unit) bool { - if in.getMeasure() == out.getMeasure() && in.getUnitDenominator() == out.getUnitDenominator() && in.getPrefix() == out.getPrefix() { - return true - } - return false - } - for _, c := range testCases { - u := NewUnit(c.in) - if (!u.Valid()) || (!compareUnitExact(u, c.want)) { - t.Errorf("func NewUnit(%q) == %q, want %q", c.in, u.String(), c.want.String()) - } else { - t.Logf("NewUnit(%q) == %q", c.in, u.String()) - } - } -} - -func TestUnitUnitConversion(t *testing.T) { - testCases := []struct { - in string - want Unit - prefixFactor float64 - }{ - {"kb", NewUnit("Bytes"), 1000}, - {"Mb", NewUnit("Bytes"), 1000000}, - {"Mb/s", NewUnit("Bytes/s"), 1000000}, - {"Flops/s", NewUnit("MFlops/s"), 1e-6}, - {"Flops/s", NewUnit("GFlops/s"), 1e-9}, - {"MHz", NewUnit("Hertz"), 1e6}, - {"kb", NewUnit("Kib"), 1000.0 / 1024}, - {"Mib", NewUnit("MBytes"), (1024 * 1024.0) / (1e6)}, - {"mb", NewUnit("MBytes"), 1.0}, - } - compareUnitWithPrefix := func(in, out Unit, factor float64) bool { - if in.getMeasure() == out.getMeasure() && in.getUnitDenominator() == out.getUnitDenominator() { - if f := GetPrefixPrefixFactor(in.getPrefix(), out.getPrefix()); f(1.0) == factor { - return true - } else { - fmt.Println(f(1.0)) - } - } - return false - } - for _, c := range testCases { - u := NewUnit(c.in) - if (!u.Valid()) || (!compareUnitWithPrefix(u, c.want, c.prefixFactor)) { - t.Errorf("GetPrefixPrefixFactor(%q, %q) invalid, want %q with factor %g", c.in, u.String(), c.want.String(), c.prefixFactor) - } else { - t.Logf("GetPrefixPrefixFactor(%q, %q) = %g", c.in, c.want.String(), c.prefixFactor) - } - } -} - -func TestUnitPrefixConversion(t *testing.T) { - testCases := []struct { - in string - want string - prefixFactor float64 - wantUnit Unit - }{ - {"KBytes", "", 1000, NewUnit("Bytes")}, - {"MBytes", "", 1e6, NewUnit("Bytes")}, - {"MBytes", "G", 1e-3, NewUnit("GBytes")}, - {"mb", "M", 1, NewUnit("MBytes")}, - } - compareUnitPrefix := func(in Unit, out Prefix, factor float64, outUnit Unit) bool { - if in.Valid() { - conv, unit := GetUnitPrefixFactor(in, out) - value := conv(1.0) - if value == factor && unit.String() == outUnit.String() { - return true - } - } - return false - } - for _, c := range testCases { - u := NewUnit(c.in) - p := NewPrefix(c.want) - if (!u.Valid()) || (!compareUnitPrefix(u, p, c.prefixFactor, c.wantUnit)) { - t.Errorf("GetUnitPrefixFactor(%q, %q) invalid, want %q with factor %g", c.in, p.Prefix(), c.wantUnit.String(), c.prefixFactor) - } else { - t.Logf("GetUnitPrefixFactor(%q, %q) = %g", c.in, c.wantUnit.String(), c.prefixFactor) - } - } -} - -func TestPrefixPrefixConversion(t *testing.T) { - testCases := []struct { - in string - want string - prefixFactor float64 - }{ - {"K", "", 1000}, - {"M", "", 1e6}, - {"M", "G", 1e-3}, - {"", "M", 1e-6}, - {"", "m", 1e3}, - {"m", "n", 1e6}, - //{"", "n", 1e9}, //does not work because of IEEE rounding problems - } - for _, c := range testCases { - i := NewPrefix(c.in) - o := NewPrefix(c.want) - if i != InvalidPrefix && o != InvalidPrefix { - conv := GetPrefixPrefixFactor(i, o) - value := conv(1.0) - if value != c.prefixFactor { - t.Errorf("GetPrefixPrefixFactor(%q, %q) invalid, want %q with factor %g but got %g", c.in, c.want, o.Prefix(), c.prefixFactor, value) - } else { - t.Logf("GetPrefixPrefixFactor(%q, %q) = %g", c.in, c.want, c.prefixFactor) - } - } - } -} - -func TestMeasureRegex(t *testing.T) { - for _, data := range MeasuresMap { - _, err := regexp.Compile(data.Regex) - if err != nil { - t.Errorf("failed to compile regex '%s': %s", data.Regex, err.Error()) - } - t.Logf("succussfully compiled regex '%s' for measure %s", data.Regex, data.Long) - } -} - -func TestPrefixRegex(t *testing.T) { - for _, data := range PrefixDataMap { - _, err := regexp.Compile(data.Regex) - if err != nil { - t.Errorf("failed to compile regex '%s': %s", data.Regex, err.Error()) - } - t.Logf("succussfully compiled regex '%s' for prefix %s", data.Regex, data.Long) - } -} - -func TestConvertValue(t *testing.T) { - v := float64(103456) - ConvertValue(&v, "MB/s", "GB/s") - - if v != 104.00 { - t.Errorf("Failed ConvertValue: Want 103.456, Got %f", v) - } -} - -func TestConvertValueUp(t *testing.T) { - v := float64(10.3456) - ConvertValue(&v, "GB/s", "MB/s") - - if v != 10346.00 { - t.Errorf("Failed ConvertValue: Want 10346.00, Got %f", v) - } -} -func TestConvertSeries(t *testing.T) { - s := []float64{2890031237, 23998994567, 389734042344, 390349424345} - r := []float64{3, 24, 390, 391} - ConvertSeries(s, "F/s", "GF/s") - - if !reflect.DeepEqual(s, r) { - t.Errorf("Failed ConvertValue: Want 3, 24, 390, 391, Got %v", s) - } -} - -func TestNormalizeValue(t *testing.T) { - var s string - v := float64(103456) - - NormalizeValue(&v, "MB/s", &s) - - if v != 104.00 { - t.Errorf("Failed ConvertValue: Want 104.00, Got %f", v) - } - if s != "GB/s" { - t.Errorf("Failed Prefix or unit: Want GB/s, Got %s", s) - } -} - -func TestNormalizeValueNoPrefix(t *testing.T) { - var s string - v := float64(103458596) - - NormalizeValue(&v, "F/s", &s) - - if v != 104.00 { - t.Errorf("Failed ConvertValue: Want 104.00, Got %f", v) - } - if s != "MF/s" { - t.Errorf("Failed Prefix or unit: Want MF/s, Got %s", s) - } -} - -func TestNormalizeValueKeep(t *testing.T) { - var s string - v := float64(345) - - NormalizeValue(&v, "MB/s", &s) - - if v != 345.00 { - t.Errorf("Failed ConvertValue: Want 104.00, Got %f", v) - } - if s != "MB/s" { - t.Errorf("Failed Prefix or unit: Want GB/s, Got %s", s) - } -} - -func TestNormalizeValueDown(t *testing.T) { - var s string - v := float64(0.0004578) - - NormalizeValue(&v, "GB/s", &s) - - if v != 458.00 { - t.Errorf("Failed ConvertValue: Want 458.00, Got %f", v) - } - if s != "KB/s" { - t.Errorf("Failed Prefix or unit: Want KB/s, Got %s", s) - } -} - -func TestNormalizeSeries(t *testing.T) { - var us string - s := []float64{2890031237, 23998994567, 389734042344, 390349424345} - r := []float64{3, 24, 390, 391} - - total := 0.0 - for _, number := range s { - total += number - } - avg := total / float64(len(s)) - - fmt.Printf("AVG: %e\n", avg) - NormalizeSeries(s, avg, "KB/s", &us) - - if !reflect.DeepEqual(s, r) { - t.Errorf("Failed ConvertValue: Want 3, 24, 390, 391, Got %v", s) - } - if us != "TB/s" { - t.Errorf("Failed Prefix or unit: Want TB/s, Got %s", us) - } -} diff --git a/test/data.json b/test/data.json deleted file mode 100644 index 1fce05f..0000000 --- a/test/data.json +++ /dev/null @@ -1,496 +0,0 @@ -{ - "cpu_used": { - "core": { - "unit": { - "base": "" - }, - "timestep": 30, - "series": [ - { - "hostname": "taurusi6489", - "id": "0", - "statistics": { - "min": 0.09090909090909093, - "avg": 0.9173553719008265, - "max": 1.0000000000000002 - }, - "data": [ - 0.09090909090909093, - 0.9999999999999999, - 1.0, - 1.0000000000000002, - 1.0, - 1.0000000000000002, - 0.9999999999999999, - 1.0, - 1.0, - 1.0, - 1.0 - ] - }, - { - "hostname": "taurusi6489", - "id": "1", - "statistics": { - "min": 0.03694102397926118, - "avg": 0.045968409230268584, - "max": 0.08809840425531917 - }, - "data": [ - 0.08809840425531917, - 0.05710659898477157, - 0.04034861200774694, - 0.037962362102530824, - 0.03976721629485936, - 0.04163976759199483, - 0.03694102397926118, - 0.03821243523316062, - 0.03851132686084142, - 0.044752092723760455, - 0.04231266149870802 - ] - }, - { - "hostname": "taurusi6490", - "id": "10", - "statistics": { - "min": 0.10505319148936171, - "avg": 0.9186411992263056, - "max": 1.0000000000000002 - }, - "data": [ - 0.10505319148936171, - 1.0000000000000002, - 1.0, - 1.0, - 1.0, - 0.9999999999999999, - 1.0, - 0.9999999999999999, - 1.0, - 1.0, - 1.0 - ] - }, - { - "hostname": "taurusi6490", - "id": "11", - "statistics": { - "min": 0.05286048845767815, - "avg": 0.07053823838706144, - "max": 0.075148113501715 - }, - "data": [ - 0.05286048845767815, - 0.06936597614563718, - 0.07254534083802376, - 0.075148113501715, - 0.06909547738693468, - 0.07372696032489846, - 0.07077983088005012, - 0.07082419304293325, - 0.07424812030075188, - 0.07285803627267043, - 0.07446808510638298 - ] - } - ], - "statisticsSeries": null - } - }, - "ipc": { - "core": { - "unit": { - "base": "IPC" - }, - "timestep": 60, - "series": [ - { - "hostname": "taurusi6489", - "id": "0", - "statistics": { - "min": 1.3808406263195592, - "avg": 1.3960848578375105, - "max": 1.4485575599350569 - }, - "data": [ - 1.4485575599350569, - 1.3808406263195592, - 1.3830284413690626, - 1.3836692663348698, - 1.3843283952290035 - ] - }, - { - "hostname": "taurusi6489", - "id": "1", - "statistics": { - "min": 0.30469640475234366, - "avg": 0.8816944294664065, - "max": 1.797623522191001 - }, - "data": [ - 1.797623522191001, - 0.954395633726228, - 1.0019972349956185, - 0.30469640475234366, - 0.3497593516668412 - ] - }, - { - "hostname": "taurusi6490", - "id": "10", - "statistics": { - "min": 1.3791232173760588, - "avg": 1.3850247295506815, - "max": 1.386710405495511 - }, - "data": [ - 1.3791232173760588, - 1.38619977419787, - 1.386397917938246, - 1.3866923327457215, - 1.386710405495511 - ] - }, - { - "hostname": "taurusi6490", - "id": "11", - "statistics": { - "min": 0.6424094604392216, - "avg": 0.9544442638400293, - "max": 1.2706704244636826 - }, - "data": [ - 1.2706704244636826, - 0.6424094604392216, - 0.9249973908234796, - 0.6940110823242276, - 1.2401329611495353 - ] - } - ], - "statisticsSeries": null - } - }, - "flops_any": { - "core": { - "unit": { - "base": "F/s" - }, - "timestep": 60, - "series": [ - { - "hostname": "taurusi6489", - "id": "0", - "statistics": { - "min": 0.0, - "avg": 184.2699002412084, - "max": 921.3495012060421 - }, - "data": [ - 921.3495012060421, - 0.0, - 0.0, - 0.0, - 0.0 - ] - }, - { - "hostname": "taurusi6489", - "id": "1", - "statistics": { - "min": 0.13559227208748068, - "avg": 273.2997868356056, - "max": 1355.9227390817396 - }, - "data": [ - 1355.9227390817396, - 8.94908797747172, - 0.6779613312519499, - 0.13559227208748068, - 0.8135535154771758 - ] - }, - { - "hostname": "taurusi6490", - "id": "10", - "statistics": { - "min": 0.0, - "avg": 1678.8419461262179, - "max": 4346.591400350933 - }, - "data": [ - 4346.591400350933, - 0.0, - 578.4248288199713, - 0.0, - 3469.193501460185 - ] - }, - { - "hostname": "taurusi6490", - "id": "11", - "statistics": { - "min": 45.28689133054866, - "avg": 609.6644949204072, - "max": 2582.7080822873186 - }, - "data": [ - 2582.7080822873186, - 45.28689133054866, - 48.67663233623293, - 47.591911855555026, - 324.0589567923803 - ] - } - ], - "statisticsSeries": null - } - }, - "mem_bw": { - "socket": { - "unit": { - "base": "B/s" - }, - "timestep": 60, - "series": [ - { - "hostname": "taurusi6489", - "id": "0", - "statistics": { - "min": 653671812.1661415, - "avg": 1637585527.5854635, - "max": 2614718291.9554267 - }, - "data": [ - 653671812.1661415, - 2614718291.9554267, - 1732453371.7073724, - 1612865229.8704093, - 1574218932.2279677 - ] - }, - { - "hostname": "taurusi6490", - "id": "0", - "statistics": { - "min": 1520190251.61048, - "avg": 1572477682.3850098, - "max": 1688960732.2760606 - }, - "data": [ - 1688960732.2760606, - 1580140679.8216474, - 1520190251.61048, - 1541841829.6250021, - 1531254918.591859 - ] - } - ], - "statisticsSeries": null - } - }, - "file_bw": { - "node": { - "unit": { - "base": "B/s" - }, - "timestep": 30, - "series": [ - { - "hostname": "taurusi6489", - "statistics": { - "min": 0.0, - "avg": 190352.6328851857, - "max": 2093878.361723524 - }, - "data": [ - 0.0, - 0.0, - 0.0, - 0.6000135186380174, - 0.0, - 0.0, - 2093878.361723524, - 0.0, - 0.0, - 0.0, - 0.0 - ] - }, - { - "hostname": "taurusi6490", - "statistics": { - "min": 0.0, - "avg": 1050832.4509396513, - "max": 11559156.360352296 - }, - "data": [ - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 11559156.360352296, - 0.0, - 0.5999838690326298, - 0.0, - 0.0 - ] - } - ], - "statisticsSeries": null - } - }, - "net_bw": { - "node": { - "unit": { - "base": "B/s" - }, - "timestep": 30, - "series": [ - { - "hostname": "taurusi6489", - "statistics": { - "min": 126779.89655880642, - "avg": 653834.5091507058, - "max": 1285639.5107541133 - }, - "data": [ - 1158202.7403032137, - 126779.89655880642, - 419017.91939583793, - 345766.3974972795, - 645419.3296982117, - 644667.7333333333, - 1285639.5107541133, - 643481.2108874657, - 640025.3562553325, - 643241.4875354709, - 639938.0184386979 - ] - }, - { - "hostname": "taurusi6490", - "statistics": { - "min": 640156.9862985397, - "avg": 872367.6551257868, - "max": 1916309.7075416835 - }, - "data": [ - 1774843.146788355, - 643218.3646426039, - 641681.1031071587, - 644690.1512268113, - 647183.5650609672, - 644439.3303402043, - 1916309.7075416835, - 643748.3241006166, - 757189.8273227927, - 642583.6999539217, - 640156.9862985397 - ] - } - ], - "statisticsSeries": null - } - }, - "mem_used": { - "node": { - "unit": { - "base": "B" - }, - "timestep": 30, - "series": [ - { - "hostname": "taurusi6489", - "statistics": { - "min": 2779066368.0, - "avg": 9282117259.636364, - "max": 10202595328.0 - }, - "data": [ - 2779066368.0, - 8518217728.0, - 9852760064.0, - 9979805696.0, - 10039619584.0, - 10087104512.0, - 10136084480.0, - 10202595328.0, - 10154196992.0, - 10177409024.0, - 10176430080.0 - ] - }, - { - "hostname": "taurusi6490", - "statistics": { - "min": 9993277440.0, - "avg": 10013080110.545454, - "max": 10039676928.0 - }, - "data": [ - 10001317888.0, - 10013028352.0, - 10006728704.0, - 10039676928.0, - 10035838976.0, - 10033356800.0, - 10006577152.0, - 10005659648.0, - 9993277440.0, - 9993564160.0, - 10014855168.0 - ] - } - ], - "statisticsSeries": null - } - }, - "cpu_power": { - "socket": { - "unit": { - "base": "W" - }, - "timestep": 60, - "series": [ - { - "hostname": "taurusi6489", - "id": "0", - "statistics": { - "min": 35.50647456742635, - "avg": 72.08313211552377, - "max": 83.33799371150049 - }, - "data": [ - 35.50647456742635, - 75.65022009482759, - 83.33799371150049, - 83.00405043233219, - 82.9169217715322 - ] - }, - { - "hostname": "taurusi6490", - "id": "0", - "statistics": { - "min": 83.8466923147859, - "avg": 85.18572681122097, - "max": 85.83909286117324 - }, - "data": [ - 83.8466923147859, - 85.58816979864088, - 85.31266819129794, - 85.83909286117324, - 85.34201089020692 - ] - } - ], - "statisticsSeries": null - } - } -} \ No newline at end of file diff --git a/test/meta.json b/test/meta.json deleted file mode 100644 index 90e2fe5..0000000 --- a/test/meta.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "jobId": 20639587, - "user": "s3804552", - "project": "p_speichersysteme", - "cluster": "taurus", - "subCluster": "haswell", - "partition": "haswell64", - "numNodes": 2, - "numHwthreads": 4, - "exclusive": 0, - "startTime": 1635856524, - "jobState": "completed", - "duration": 310, - "walltime": 3600, - "smt": 0, - "resources": [ - { - "hostname": "taurusi6489", - "hwthreads": [ - 0, - 1 - ] - }, - { - "hostname": "taurusi6490", - "hwthreads": [ - 10, - 11 - ] - } - ], - "statistics": { - "cpu_used": { - "min": 0.03694102397926118, - "avg": 0.48812580468611544, - "max": 1.0000000000000002, - "unit": { - "base": "" - } - }, - "ipc": { - "min": 0.30469640475234366, - "avg": 1.154312070173657, - "max": 1.797623522191001, - "unit": { - "base": "IPC" - } - }, - "flops_any": { - "min": 0.0, - "avg": 686.5190320308598, - "max": 4346.591400350933, - "unit": { - "base": "F/s" - } - }, - "mem_bw": { - "min": 653671812.1661415, - "avg": 1605031604.9852366, - "max": 2614718291.9554267, - "unit": { - "base": "B/s" - } - }, - "file_bw": { - "min": 0.0, - "avg": 620592.5419124186, - "max": 11559156.360352296, - "unit": { - "base": "B/s" - } - }, - "net_bw": { - "min": 126779.89655880642, - "avg": 763101.082138246, - "max": 1916309.7075416835, - "unit": { - "base": "B/s" - } - }, - "mem_used": { - "min": 2779066368.0, - "avg": 9647598685.09091, - "max": 10202595328.0, - "unit": { - "base": "B" - } - }, - "cpu_power": { - "min": 35.50647456742635, - "avg": 78.63442946337237, - "max": 85.83909286117324, - "unit": { - "base": "W" - } - } - } -} diff --git a/test/test.db b/test/test.db deleted file mode 100644 index bf9e2ce..0000000 Binary files a/test/test.db and /dev/null differ diff --git a/tools/archive-manager/main.go b/tools/archive-manager/main.go index 34d66ac..988bb78 100644 --- a/tools/archive-manager/main.go +++ b/tools/archive-manager/main.go @@ -8,33 +8,65 @@ import ( "encoding/json" "flag" "fmt" + "os" + "time" "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/pkg/archive" "github.com/ClusterCockpit/cc-backend/pkg/log" ) +func parseDate(in string) int64 { + const shortForm = "2006-Jan-02" + loc, _ := time.LoadLocation("Local") + if in != "" { + t, err := time.ParseInLocation(shortForm, in, loc) + if err != nil { + fmt.Printf("date parse error %v", err) + os.Exit(0) + } + return t.Unix() + } + + return 0 +} + func main() { - var srcPath, flagConfigFile, flagLogLevel string - var flagLogDateTime bool + var srcPath, flagConfigFile, flagLogLevel, flagRemoveCluster, flagRemoveAfter, flagRemoveBefore string + var flagLogDateTime, flagValidate bool flag.StringVar(&srcPath, "s", "./var/job-archive", "Specify the source job archive path. Default is ./var/job-archive") flag.BoolVar(&flagLogDateTime, "logdate", false, "Set this flag to add date and time to log messages") flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug,info,warn (default),err,fatal,crit]`") flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`") + flag.StringVar(&flagRemoveCluster, "remove-cluster", "", "Remove cluster from archive and database") + flag.StringVar(&flagRemoveBefore, "remove-before", "", "Remove all jobs with start time before date (Format: 2006-Jan-04)") + flag.StringVar(&flagRemoveAfter, "remove-after", "", "Remove all jobs with start time after date (Format: 2006-Jan-04)") + flag.BoolVar(&flagValidate, "validate", false, "Set this flag to validate a job archive against the json schema") flag.Parse() + archiveCfg := fmt.Sprintf("{\"kind\": \"file\",\"path\": \"%s\"}", srcPath) log.Init(flagLogLevel, flagLogDateTime) config.Init(flagConfigFile) - config.Keys.Validate = true if err := archive.Init(json.RawMessage(archiveCfg), false); err != nil { log.Fatal(err) } ar := archive.GetHandle() - for job := range ar.Iter(true) { - log.Printf("Validate %s - %d\n", job.Meta.Cluster, job.Meta.JobID) + if flagValidate { + config.Keys.Validate = true + for job := range ar.Iter(true) { + log.Printf("Validate %s - %d\n", job.Meta.Cluster, job.Meta.JobID) + } + os.Exit(0) } + + if flagRemoveBefore != "" || flagRemoveAfter != "" { + ar.Clean(parseDate(flagRemoveBefore), parseDate(flagRemoveAfter)) + os.Exit(0) + } + + ar.Info() } diff --git a/tools/archive-migration/job.go b/tools/archive-migration/job.go index 883ea81..cd54d6c 100644 --- a/tools/archive-migration/job.go +++ b/tools/archive-migration/job.go @@ -23,17 +23,17 @@ type BaseJob struct { Project string `json:"project" db:"project" example:"abcd200"` // The unique identifier of a project Cluster string `json:"cluster" db:"cluster" example:"fritz"` // The unique identifier of a cluster SubCluster string `json:"subCluster" db:"subcluster" example:"main"` // The unique identifier of a sub cluster - Partition *string `json:"partition" db:"partition" example:"main"` // The Slurm partition to which the job was submitted - ArrayJobId *int64 `json:"arrayJobId" db:"array_job_id" example:"123000"` // The unique identifier of an array job + Partition string `json:"partition" db:"partition" example:"main"` // The Slurm partition to which the job was submitted + ArrayJobId int64 `json:"arrayJobId" db:"array_job_id" example:"123000"` // The unique identifier of an array job NumNodes int32 `json:"numNodes" db:"num_nodes" example:"2" minimum:"1"` // Number of nodes used (Min > 0) - NumHWThreads *int32 `json:"numHwthreads" db:"num_hwthreads" example:"20" minimum:"1"` // Number of HWThreads used (Min > 0) - NumAcc *int32 `json:"numAcc" db:"num_acc" example:"2" minimum:"1"` // Number of accelerators used (Min > 0) + NumHWThreads int32 `json:"numHwthreads" db:"num_hwthreads" example:"20" minimum:"1"` // Number of HWThreads used (Min > 0) + NumAcc int32 `json:"numAcc" db:"num_acc" example:"2" minimum:"1"` // Number of accelerators used (Min > 0) Exclusive int32 `json:"exclusive" db:"exclusive" example:"1" minimum:"0" maximum:"2"` // Specifies how nodes are shared: 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple jobs of same user MonitoringStatus int32 `json:"monitoringStatus" db:"monitoring_status" example:"1" minimum:"0" maximum:"3"` // State of monitoring system during job run: 0 - Disabled, 1 - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull - SMT *int32 `json:"smt" db:"smt" example:"4"` // SMT threads used by job + SMT int32 `json:"smt" db:"smt" example:"4"` // SMT threads used by job State JobState `json:"jobState" db:"job_state" example:"completed" enums:"completed,failed,cancelled,stopped,timeout,out_of_memory"` // Final state of job Duration int32 `json:"duration" db:"duration" example:"43200" minimum:"1"` // Duration of job in seconds (Min > 0) - Walltime *int64 `json:"walltime" db:"walltime" example:"86400" minimum:"1"` // Requested walltime of job in seconds (Min > 0) + Walltime int64 `json:"walltime" db:"walltime" example:"86400" minimum:"1"` // Requested walltime of job in seconds (Min > 0) Tags []*schema.Tag `json:"tags"` // List of tags RawResources []byte `json:"-" db:"resources"` // Resources used by job [As Bytes] Resources []*Resource `json:"resources"` // Resources used by job diff --git a/tools/archive-migration/main.go b/tools/archive-migration/main.go index 3e21ffa..dce44de 100644 --- a/tools/archive-migration/main.go +++ b/tools/archive-migration/main.go @@ -17,7 +17,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/pkg/log" "github.com/ClusterCockpit/cc-backend/pkg/schema" - "github.com/ClusterCockpit/cc-backend/pkg/units" + ccunits "github.com/ClusterCockpit/cc-units" ) const Version = 1 @@ -35,6 +35,33 @@ func loadJobData(filename string) (*JobData, error) { return DecodeJobData(bufio.NewReader(f)) } +func ConvertUnitString(us string) schema.Unit { + var nu schema.Unit + + if us == "CPI" || + us == "IPC" || + us == "load" || + us == "" { + nu.Base = us + return nu + } + u := ccunits.NewUnit(us) + p := u.GetPrefix() + if p.Prefix() != "" { + prefix := p.Prefix() + nu.Prefix = prefix + } + m := u.GetMeasure() + d := u.GetUnitDenominator() + if d.Short() != "inval" { + nu.Base = fmt.Sprintf("%s/%s", m.Short(), d.Short()) + } else { + nu.Base = m.Short() + } + + return nu +} + func deepCopyJobMeta(j *JobMeta) schema.JobMeta { var jn schema.JobMeta @@ -78,7 +105,7 @@ func deepCopyJobMeta(j *JobMeta) schema.JobMeta { sn.Avg = v.Avg sn.Max = v.Max sn.Min = v.Min - tmpUnit := units.ConvertUnitString(v.Unit) + tmpUnit := ConvertUnitString(v.Unit) if tmpUnit.Base == "inval" { sn.Unit = schema.Unit{Base: ""} } else { @@ -113,7 +140,7 @@ func deepCopyJobData(d *JobData, cluster string, subCluster string) *schema.JobD for mk, mv := range v { // fmt.Printf("Scope %s\n", mk) var mn schema.JobMetric - tmpUnit := units.ConvertUnitString(mv.Unit) + tmpUnit := ConvertUnitString(mv.Unit) if tmpUnit.Base == "inval" { mn.Unit = schema.Unit{Base: ""} } else { @@ -174,16 +201,14 @@ func deepCopyClusterConfig(co *Cluster) schema.Cluster { scn.SocketsPerNode = sco.SocketsPerNode scn.CoresPerSocket = sco.CoresPerSocket scn.ThreadsPerCore = sco.ThreadsPerCore - var prefix = new(string) - *prefix = "G" scn.FlopRateScalar = schema.MetricValue{ - Unit: schema.Unit{Base: "F/s", Prefix: prefix}, + Unit: schema.Unit{Base: "F/s", Prefix: "G"}, Value: float64(sco.FlopRateScalar)} scn.FlopRateSimd = schema.MetricValue{ - Unit: schema.Unit{Base: "F/s", Prefix: prefix}, + Unit: schema.Unit{Base: "F/s", Prefix: "G"}, Value: float64(sco.FlopRateSimd)} scn.MemoryBandwidth = schema.MetricValue{ - Unit: schema.Unit{Base: "B/s", Prefix: prefix}, + Unit: schema.Unit{Base: "B/s", Prefix: "G"}, Value: float64(sco.MemoryBandwidth)} scn.Topology = *sco.Topology cn.SubClusters = append(cn.SubClusters, &scn) @@ -194,13 +219,13 @@ func deepCopyClusterConfig(co *Cluster) schema.Cluster { mcn.Name = mco.Name mcn.Scope = mco.Scope if mco.Aggregation == "" { - fmt.Println("Property aggregation missing! Please review file!") + fmt.Println("cluster.json - Property aggregation missing! Please review file!") mcn.Aggregation = "sum" } else { mcn.Aggregation = mco.Aggregation } mcn.Timestep = mco.Timestep - tmpUnit := units.ConvertUnitString(mco.Unit) + tmpUnit := ConvertUnitString(mco.Unit) if tmpUnit.Base == "inval" { mcn.Unit = schema.Unit{Base: ""} } else { @@ -227,8 +252,8 @@ func main() { flag.BoolVar(&flagLogDateTime, "logdate", false, "Set this flag to add date and time to log messages") flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug,info,warn (default),err,fatal,crit]`") flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`") - flag.StringVar(&srcPath, "s", "./var/job-archive", "Specify the source job archive path. Default is ./var/job-archive") - flag.StringVar(&dstPath, "d", "./var/job-archive-new", "Specify the destination job archive path. Default is ./var/job-archive-new") + flag.StringVar(&srcPath, "s", "./var/job-archive", "Specify the source job archive path") + flag.StringVar(&dstPath, "d", "./var/job-archive-new", "Specify the destination job archive path") flag.Parse() if _, err := os.Stat(filepath.Join(srcPath, "version.txt")); !errors.Is(err, os.ErrNotExist) { diff --git a/web/frontend/package-lock.json b/web/frontend/package-lock.json new file mode 100644 index 0000000..b00380d --- /dev/null +++ b/web/frontend/package-lock.json @@ -0,0 +1,712 @@ +{ + "name": "cc-frontend", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "cc-frontend", + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "@rollup/plugin-replace": "^5.0.2", + "@urql/svelte": "^4.0.1", + "graphql": "^16.6.0", + "sveltestrap": "^5.10.0", + "uplot": "^1.6.24", + "wonka": "^6.3.2" + }, + "devDependencies": { + "@rollup/plugin-commonjs": "^24.1.0", + "@rollup/plugin-node-resolve": "^15.0.2", + "@rollup/plugin-terser": "^0.4.1", + "rollup": "^3.21.0", + "rollup-plugin-css-only": "^4.3.0", + "rollup-plugin-svelte": "^7.1.4", + "svelte": "^3.58.0" + } + }, + "node_modules/@0no-co/graphql.web": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@0no-co/graphql.web/-/graphql.web-1.0.1.tgz", + "integrity": "sha512-6Yaxyv6rOwRkLIvFaL0NrLDgfNqC/Ng9QOPmTmlqW4mORXMEKmh5NYGkIvvt5Yw8fZesnMAqkj8cIqTj8f40cQ==", + "peerDependencies": { + "graphql": "^14.0.0 || ^15.0.0 || ^16.0.0" + }, + "peerDependenciesMeta": { + "graphql": { + "optional": true + } + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz", + "integrity": "sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A==", + "dev": true, + "dependencies": { + "@jridgewell/set-array": "^1.0.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.9" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", + "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/set-array": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz", + "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/source-map": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.2.tgz", + "integrity": "sha512-m7O9o2uR8k2ObDysZYzdfhb08VuEml5oWGiosa1VdaPZ/A6QyPkAJuwN0Q1lhULOf6B7MtQmHENS743hWtCrgw==", + "dev": true, + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.0", + "@jridgewell/trace-mapping": "^0.3.9" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.4.14", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", + "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.14", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.14.tgz", + "integrity": "sha512-bJWEfQ9lPTvm3SneWwRFVLzrh6nhjwqw7TUFFBEMzwvg7t7PCDenf2lDwqo4NQXzdpgBXyFgDWnQA+2vkruksQ==", + "dev": true, + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "node_modules/@popperjs/core": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.0.tgz", + "integrity": "sha512-zrsUxjLOKAzdewIDRWy9nsV1GQsKBCWaGwsZQlCgr6/q+vjyZhFgqedLfFBuI9anTPEUT4APq9Mu0SZBTzIcGQ==", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/popperjs" + } + }, + "node_modules/@rollup/plugin-commonjs": { + "version": "24.1.0", + "resolved": "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-24.1.0.tgz", + "integrity": "sha512-eSL45hjhCWI0jCCXcNtLVqM5N1JlBGvlFfY0m6oOYnLCJ6N0qEXoZql4sY2MOUArzhH4SA/qBpTxvvZp2Sc+DQ==", + "dev": true, + "dependencies": { + "@rollup/pluginutils": "^5.0.1", + "commondir": "^1.0.1", + "estree-walker": "^2.0.2", + "glob": "^8.0.3", + "is-reference": "1.2.1", + "magic-string": "^0.27.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "rollup": "^2.68.0||^3.0.0" + }, + "peerDependenciesMeta": { + "rollup": { + "optional": true + } + } + }, + "node_modules/@rollup/plugin-node-resolve": { + "version": "15.0.2", + "resolved": "https://registry.npmjs.org/@rollup/plugin-node-resolve/-/plugin-node-resolve-15.0.2.tgz", + "integrity": "sha512-Y35fRGUjC3FaurG722uhUuG8YHOJRJQbI6/CkbRkdPotSpDj9NtIN85z1zrcyDcCQIW4qp5mgG72U+gJ0TAFEg==", + "dev": true, + "dependencies": { + "@rollup/pluginutils": "^5.0.1", + "@types/resolve": "1.20.2", + "deepmerge": "^4.2.2", + "is-builtin-module": "^3.2.1", + "is-module": "^1.0.0", + "resolve": "^1.22.1" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "rollup": "^2.78.0||^3.0.0" + }, + "peerDependenciesMeta": { + "rollup": { + "optional": true + } + } + }, + "node_modules/@rollup/plugin-replace": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@rollup/plugin-replace/-/plugin-replace-5.0.2.tgz", + "integrity": "sha512-M9YXNekv/C/iHHK+cvORzfRYfPbq0RDD8r0G+bMiTXjNGKulPnCT9O3Ss46WfhI6ZOCgApOP7xAdmCQJ+U2LAA==", + "dependencies": { + "@rollup/pluginutils": "^5.0.1", + "magic-string": "^0.27.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "rollup": "^1.20.0||^2.0.0||^3.0.0" + }, + "peerDependenciesMeta": { + "rollup": { + "optional": true + } + } + }, + "node_modules/@rollup/plugin-terser": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@rollup/plugin-terser/-/plugin-terser-0.4.1.tgz", + "integrity": "sha512-aKS32sw5a7hy+fEXVy+5T95aDIwjpGHCTv833HXVtyKMDoVS7pBr5K3L9hEQoNqbJFjfANPrNpIXlTQ7is00eA==", + "dev": true, + "dependencies": { + "serialize-javascript": "^6.0.0", + "smob": "^0.0.6", + "terser": "^5.15.1" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "rollup": "^2.x || ^3.x" + }, + "peerDependenciesMeta": { + "rollup": { + "optional": true + } + } + }, + "node_modules/@rollup/plugin-terser/node_modules/serialize-javascript": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.1.tgz", + "integrity": "sha512-owoXEFjWRllis8/M1Q+Cw5k8ZH40e3zhp/ovX+Xr/vi1qj6QesbyXXViFbpNvWvPNAD62SutwEXavefrLJWj7w==", + "dev": true, + "dependencies": { + "randombytes": "^2.1.0" + } + }, + "node_modules/@rollup/pluginutils": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.0.2.tgz", + "integrity": "sha512-pTd9rIsP92h+B6wWwFbW8RkZv4hiR/xKsqre4SIuAOaOEQRxi0lqLke9k2/7WegC85GgUs9pjmOjCUi3In4vwA==", + "dependencies": { + "@types/estree": "^1.0.0", + "estree-walker": "^2.0.2", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "rollup": "^1.20.0||^2.0.0||^3.0.0" + }, + "peerDependenciesMeta": { + "rollup": { + "optional": true + } + } + }, + "node_modules/@types/estree": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.1.tgz", + "integrity": "sha512-LG4opVs2ANWZ1TJoKc937iMmNstM/d0ae1vNbnBvBhqCSezgVUOzcLCqbI5elV8Vy6WKwKjaqR+zO9VKirBBCA==" + }, + "node_modules/@types/resolve": { + "version": "1.20.2", + "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-1.20.2.tgz", + "integrity": "sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q==", + "dev": true + }, + "node_modules/@urql/core": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/@urql/core/-/core-4.0.7.tgz", + "integrity": "sha512-UtZ9oSbSFODXzFydgLCXpAQz26KGT1d6uEfcylKphiRWNXSWZi8k7vhJXNceNm/Dn0MiZ+kaaJHKcnGY1jvHRQ==", + "dependencies": { + "@0no-co/graphql.web": "^1.0.1", + "wonka": "^6.3.2" + } + }, + "node_modules/@urql/svelte": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@urql/svelte/-/svelte-4.0.1.tgz", + "integrity": "sha512-WbsVjuK7IUNlJlvXAgevjQunoso0T+AngFlb0zafDvay6HN47Zc3CSVbAlP8KjETjERUMJLuiqknmPFFm2GEFQ==", + "dependencies": { + "@urql/core": "^4.0.0", + "wonka": "^6.3.2" + }, + "peerDependencies": { + "svelte": "^3.0.0" + } + }, + "node_modules/acorn": { + "version": "8.8.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.0.tgz", + "integrity": "sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w==", + "dev": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true + }, + "node_modules/builtin-modules": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.3.0.tgz", + "integrity": "sha512-zhaCDicdLuWN5UbN5IMnFqNMhNfo919sH85y2/ea+5Yg9TsTkeZxpL+JLbp6cgYFS4sRLp3YV4S6yDuqVWHYOw==", + "dev": true, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "dev": true + }, + "node_modules/commondir": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", + "integrity": "sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs= sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==", + "dev": true + }, + "node_modules/deepmerge": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.2.2.tgz", + "integrity": "sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/estree-walker": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", + "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==" + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "dev": true + }, + "node_modules/glob": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^5.0.1", + "once": "^1.3.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/graphql": { + "version": "16.6.0", + "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.6.0.tgz", + "integrity": "sha512-KPIBPDlW7NxrbT/eh4qPXz5FiFdL5UbaA0XUNz2Rp3Z3hqBSkbj0GVjwFDztsWVauZUWsbKHgMg++sk8UX0bkw==", + "engines": { + "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" + } + }, + "node_modules/has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.1" + }, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dev": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "node_modules/is-builtin-module": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-3.2.1.tgz", + "integrity": "sha512-BSLE3HnV2syZ0FK0iMA/yUGplUeMmNz4AW5fnTunbCIqZi4vG3WjJT9FHMy5D69xmAYBHXQhJdALdpwVxV501A==", + "dev": true, + "dependencies": { + "builtin-modules": "^3.3.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-core-module": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.12.0.tgz", + "integrity": "sha512-RECHCBCd/viahWmwj6enj19sKbHfJrddi/6cBDsNTKbNq0f7VeaUkBo60BqzvPqo/W54ChS62Z5qyun7cfOMqQ==", + "dev": true, + "dependencies": { + "has": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-module": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-module/-/is-module-1.0.0.tgz", + "integrity": "sha1-Mlj7afeMFNW4FdZkM2tM/7ZEFZE= sha512-51ypPSPCoTEIN9dy5Oy+h4pShgJmPCygKfyRCISBI+JoWT/2oJvK8QPxmwv7b/p239jXrm9M1mlQbyKJ5A152g==", + "dev": true + }, + "node_modules/is-reference": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/is-reference/-/is-reference-1.2.1.tgz", + "integrity": "sha512-U82MsXXiFIrjCK4otLT+o2NA2Cd2g5MLoOVXUZjIOhLurrRxpEXzI8O0KZHr3IjLvlAH1kTPYSuqer5T9ZVBKQ==", + "dev": true, + "dependencies": { + "@types/estree": "*" + } + }, + "node_modules/magic-string": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.27.0.tgz", + "integrity": "sha512-8UnnX2PeRAPZuN12svgR9j7M1uWMovg/CEnIwIG0LFkXSJJe4PdfUGiTGl8V9bsBHFUtfVINcSyYxd7q+kx9fA==", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.4.13" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "dev": true, + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "dev": true, + "dependencies": { + "safe-buffer": "^5.1.0" + } + }, + "node_modules/resolve": { + "version": "1.22.2", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.2.tgz", + "integrity": "sha512-Sb+mjNHOULsBv818T40qSPeRiuWLyaGMa5ewydRLFimneixmVy2zdivRl+AF6jaYPC8ERxGDmFSiqui6SfPd+g==", + "dev": true, + "dependencies": { + "is-core-module": "^2.11.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve.exports": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.2.tgz", + "integrity": "sha512-X2UW6Nw3n/aMgDVy+0rSqgHlv39WZAlZrXCdnbyEiKm17DSqHX4MmQMaST3FbeWR5FTuRcUwYAziZajji0Y7mg==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/rollup": { + "version": "3.21.0", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-3.21.0.tgz", + "integrity": "sha512-ANPhVcyeHvYdQMUyCbczy33nbLzI7RzrBje4uvNiTDJGIMtlKoOStmympwr9OtS1LZxiDmE2wvxHyVhoLtf1KQ==", + "devOptional": true, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=14.18.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/rollup-plugin-css-only": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/rollup-plugin-css-only/-/rollup-plugin-css-only-4.3.0.tgz", + "integrity": "sha512-BsiCqJJQzZh2lQiHY5irejRoJ3I1EUFHEi5PjVqsr+EmOh54YrWVwd3YZEXnQJ2+fzlhif0YM/Kf0GuH90GAdQ==", + "dev": true, + "dependencies": { + "@rollup/pluginutils": "5" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "rollup": "<4" + } + }, + "node_modules/rollup-plugin-svelte": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/rollup-plugin-svelte/-/rollup-plugin-svelte-7.1.4.tgz", + "integrity": "sha512-Jm0FCydR7k8bBGe7wimXAes8x2zEK10Ew3f3lEZwYor/Zya3X0AZVeSAPRH7yiXB9hWQVzJu597EUeNwGDTdjQ==", + "dev": true, + "dependencies": { + "@rollup/pluginutils": "^4.1.0", + "resolve.exports": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "rollup": ">=2.0.0", + "svelte": ">=3.5.0" + } + }, + "node_modules/rollup-plugin-svelte/node_modules/@rollup/pluginutils": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-4.2.1.tgz", + "integrity": "sha512-iKnFXr7NkdZAIHiIWE+BX5ULi/ucVFYWD6TbAV+rZctiRTY2PL6tsIKhoIOaoskiWAkgu+VsbXgUVDNLHf+InQ==", + "dev": true, + "dependencies": { + "estree-walker": "^2.0.1", + "picomatch": "^2.2.2" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/smob": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/smob/-/smob-0.0.6.tgz", + "integrity": "sha512-V21+XeNni+tTyiST1MHsa84AQhT1aFZipzPpOFAVB8DkHzwJyjjAmt9bgwnuZiZWnIbMo2duE29wybxv/7HWUw==", + "dev": true + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "dev": true, + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "dev": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/svelte": { + "version": "3.58.0", + "resolved": "https://registry.npmjs.org/svelte/-/svelte-3.58.0.tgz", + "integrity": "sha512-brIBNNB76mXFmU/Kerm4wFnkskBbluBDCjx/8TcpYRb298Yh2dztS2kQ6bhtjMcvUhd5ynClfwpz5h2gnzdQ1A==", + "engines": { + "node": ">= 8" + } + }, + "node_modules/sveltestrap": { + "version": "5.10.0", + "resolved": "https://registry.npmjs.org/sveltestrap/-/sveltestrap-5.10.0.tgz", + "integrity": "sha512-k6Ob+6G2AMYvBidXHBKM9W28fJqFHbmosqCe/NC8pv6TV7K+v47Yw+zmnLWkjqCzzmjkSLkL48SrHZrlWc9mYQ==", + "dependencies": { + "@popperjs/core": "^2.9.2" + }, + "peerDependencies": { + "svelte": "^3.29.0" + } + }, + "node_modules/terser": { + "version": "5.17.1", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.17.1.tgz", + "integrity": "sha512-hVl35zClmpisy6oaoKALOpS0rDYLxRFLHhRuDlEGTKey9qHjS1w9GMORjuwIMt70Wan4lwsLYyWDVnWgF+KUEw==", + "dev": true, + "dependencies": { + "@jridgewell/source-map": "^0.3.2", + "acorn": "^8.5.0", + "commander": "^2.20.0", + "source-map-support": "~0.5.20" + }, + "bin": { + "terser": "bin/terser" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/uplot": { + "version": "1.6.24", + "resolved": "https://registry.npmjs.org/uplot/-/uplot-1.6.24.tgz", + "integrity": "sha512-WpH2BsrFrqxkMu+4XBvc0eCDsRBhzoq9crttYeSI0bfxpzR5YoSVzZXOKFVWcVC7sp/aDXrdDPbDZGCtck2PVg==" + }, + "node_modules/wonka": { + "version": "6.3.2", + "resolved": "https://registry.npmjs.org/wonka/-/wonka-6.3.2.tgz", + "integrity": "sha512-2xXbQ1LnwNS7egVm1HPhW2FyKrekolzhpM3mCwXdQr55gO+tAiY76rhb32OL9kKsW8taj++iP7C6hxlVzbnvrw==" + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true + } + } +} diff --git a/web/frontend/package.json b/web/frontend/package.json index 174fa5f..0b8baaa 100644 --- a/web/frontend/package.json +++ b/web/frontend/package.json @@ -7,20 +7,20 @@ "dev": "rollup -c -w" }, "devDependencies": { - "@rollup/plugin-commonjs": "^17.0.0", - "@rollup/plugin-node-resolve": "^11.0.0", - "rollup": "^2.3.4", - "rollup-plugin-css-only": "^3.1.0", - "rollup-plugin-svelte": "^7.0.0", - "rollup-plugin-terser": "^7.0.0", - "svelte": "^3.49.0" + "@rollup/plugin-commonjs": "^24.1.0", + "@rollup/plugin-node-resolve": "^15.0.2", + "@rollup/plugin-terser": "^0.4.1", + "rollup": "^3.21.0", + "rollup-plugin-css-only": "^4.3.0", + "rollup-plugin-svelte": "^7.1.4", + "svelte": "^3.58.0" }, "dependencies": { - "@rollup/plugin-replace": "^2.4.1", - "@urql/svelte": "^1.3.0", - "graphql": "^15.6.0", - "sveltestrap": "^5.6.1", - "uplot": "^1.6.7", - "wonka": "^4.0.15" + "@rollup/plugin-replace": "^5.0.2", + "@urql/svelte": "^4.0.1", + "graphql": "^16.6.0", + "sveltestrap": "^5.10.0", + "uplot": "^1.6.24", + "wonka": "^6.3.2" } } diff --git a/web/frontend/rollup.config.js b/web/frontend/rollup.config.js deleted file mode 100644 index 2737c8a..0000000 --- a/web/frontend/rollup.config.js +++ /dev/null @@ -1,71 +0,0 @@ -import svelte from 'rollup-plugin-svelte'; -import replace from "@rollup/plugin-replace"; -import commonjs from '@rollup/plugin-commonjs'; -import resolve from '@rollup/plugin-node-resolve'; -import { terser } from 'rollup-plugin-terser'; -import css from 'rollup-plugin-css-only'; - -// const production = !process.env.ROLLUP_WATCH; -const production = true - -const plugins = [ - svelte({ - compilerOptions: { - // enable run-time checks when not in production - dev: !production - } - }), - - // If you have external dependencies installed from - // npm, you'll most likely need these plugins. In - // some cases you'll need additional configuration - - // consult the documentation for details: - // https://github.com/rollup/plugins/tree/master/packages/commonjs - resolve({ - browser: true, - dedupe: ['svelte'] - }), - commonjs(), - - // If we're building for production (npm run build - // instead of npm run dev), minify - production && terser(), - - replace({ - "process.env.NODE_ENV": JSON.stringify("development"), - preventAssignment: true - }) -]; - -const entrypoint = (name, path) => ({ - input: path, - output: { - sourcemap: false, - format: 'iife', - name: 'app', - file: `public/build/${name}.js` - }, - plugins: [ - ...plugins, - - // we'll extract any component CSS out into - // a separate file - better for performance - css({ output: `${name}.css` }), - ], - watch: { - clearScreen: false - } -}); - -export default [ - entrypoint('header', 'src/header.entrypoint.js'), - entrypoint('jobs', 'src/jobs.entrypoint.js'), - entrypoint('user', 'src/user.entrypoint.js'), - entrypoint('list', 'src/list.entrypoint.js'), - entrypoint('job', 'src/job.entrypoint.js'), - entrypoint('systems', 'src/systems.entrypoint.js'), - entrypoint('node', 'src/node.entrypoint.js'), - entrypoint('analysis', 'src/analysis.entrypoint.js'), - entrypoint('status', 'src/status.entrypoint.js'), - entrypoint('config', 'src/config.entrypoint.js') -]; diff --git a/web/frontend/rollup.config.mjs b/web/frontend/rollup.config.mjs new file mode 100644 index 0000000..8336287 --- /dev/null +++ b/web/frontend/rollup.config.mjs @@ -0,0 +1,71 @@ +import svelte from 'rollup-plugin-svelte'; +import replace from "@rollup/plugin-replace"; +import commonjs from '@rollup/plugin-commonjs'; +import resolve from '@rollup/plugin-node-resolve'; +import terser from '@rollup/plugin-terser'; +import css from 'rollup-plugin-css-only'; + +const production = !process.env.ROLLUP_WATCH; +// const production = false + +const plugins = [ + svelte({ + compilerOptions: { + // enable run-time checks when not in production + dev: !production + } + }), + + // If you have external dependencies installed from + // npm, you'll most likely need these plugins. In + // some cases you'll need additional configuration - + // consult the documentation for details: + // https://github.com/rollup/plugins/tree/master/packages/commonjs + resolve({ + browser: true, + dedupe: ['svelte'] + }), + commonjs(), + + // If we're building for production (npm run build + // instead of npm run dev), minify + production && terser(), + + replace({ + "process.env.NODE_ENV": JSON.stringify("development"), + preventAssignment: true + }) +]; + +const entrypoint = (name, path) => ({ + input: path, + output: { + sourcemap: false, + format: 'iife', + name: 'app', + file: `public/build/${name}.js` + }, + plugins: [ + ...plugins, + + // we'll extract any component CSS out into + // a separate file - better for performance + css({ output: `${name}.css` }), + ], + watch: { + clearScreen: false + } +}); + +export default [ + entrypoint('header', 'src/header.entrypoint.js'), + entrypoint('jobs', 'src/jobs.entrypoint.js'), + entrypoint('user', 'src/user.entrypoint.js'), + entrypoint('list', 'src/list.entrypoint.js'), + entrypoint('job', 'src/job.entrypoint.js'), + entrypoint('systems', 'src/systems.entrypoint.js'), + entrypoint('node', 'src/node.entrypoint.js'), + entrypoint('analysis', 'src/analysis.entrypoint.js'), + entrypoint('status', 'src/status.entrypoint.js'), + entrypoint('config', 'src/config.entrypoint.js') +]; diff --git a/web/frontend/src/Analysis.root.svelte b/web/frontend/src/Analysis.root.svelte index e6dba50..59492fc 100644 --- a/web/frontend/src/Analysis.root.svelte +++ b/web/frontend/src/Analysis.root.svelte @@ -1,7 +1,7 @@ @@ -116,11 +120,7 @@ disableClusterSelection={true} startTimeQuickSelect={true} on:update={({ detail }) => { - $statsQuery.context.pause = false - $statsQuery.variables = { filter: detail.filters } - $footprintsQuery.context.pause = false - $footprintsQuery.variables = { metrics, filter: detail.filters } - $rooflineQuery.variables = { ...$rooflineQuery.variables, filter: detail.filters } + filters = detail.filters; }} /> diff --git a/web/frontend/src/Job.root.svelte b/web/frontend/src/Job.root.svelte index c713c33..33a38b1 100644 --- a/web/frontend/src/Job.root.svelte +++ b/web/frontend/src/Job.root.svelte @@ -134,8 +134,8 @@ cluster={clusters .find(c => c.name == $initq.data.job.cluster).subClusters .find(sc => sc.name == $initq.data.job.subCluster)} - flopsAny={$jobMetrics.data.jobMetrics.find(m => m.name == 'flops_any' && m.scope == 'node').metric} - memBw={$jobMetrics.data.jobMetrics.find(m => m.name == 'mem_bw' && m.scope == 'node').metric} /> + flopsAny={$jobMetrics.data.jobMetrics.find(m => m.name == 'flops_any' && m.scope == 'node')} + memBw={$jobMetrics.data.jobMetrics.find(m => m.name == 'mem_bw' && m.scope == 'node')} /> {:else} diff --git a/web/frontend/src/Jobs.root.svelte b/web/frontend/src/Jobs.root.svelte index f78c1eb..b96576b 100644 --- a/web/frontend/src/Jobs.root.svelte +++ b/web/frontend/src/Jobs.root.svelte @@ -72,7 +72,7 @@ filters.update(detail)}/> - jobList.update()} /> + jobList.refresh()} />
diff --git a/web/frontend/src/List.root.svelte b/web/frontend/src/List.root.svelte index 5eb36ae..371c743 100644 --- a/web/frontend/src/List.root.svelte +++ b/web/frontend/src/List.root.svelte @@ -2,64 +2,78 @@ @component List of users or projects --> @@ -68,59 +82,86 @@ - + { - $stats.variables = { filter: detail.filters } - $stats.context.pause = false - $stats.reexecute() - }} /> + filters = detail.filters; + }} + /> - {#if type == 'USER'} + {#if type == "USER"} {/if} @@ -129,26 +170,36 @@ {#if $stats.fetching} - + {:else if $stats.error} - + {:else if $stats.data} {#each sort($stats.data.rows, sorting, nameFilter) as row (row.id)} - {#if type == 'USER'} - + {#if type == "USER"} + {/if} @@ -156,7 +207,9 @@ {:else} - + {/each} {/if} diff --git a/web/frontend/src/MetricSelection.svelte b/web/frontend/src/MetricSelection.svelte index 3397afd..59fe263 100644 --- a/web/frontend/src/MetricSelection.svelte +++ b/web/frontend/src/MetricSelection.svelte @@ -10,7 +10,7 @@ diff --git a/web/frontend/src/Node.root.svelte b/web/frontend/src/Node.root.svelte index ecfab70..25e3b77 100644 --- a/web/frontend/src/Node.root.svelte +++ b/web/frontend/src/Node.root.svelte @@ -1,7 +1,7 @@ diff --git a/web/frontend/src/PlotSelection.svelte b/web/frontend/src/PlotSelection.svelte index 0205c27..449de64 100644 --- a/web/frontend/src/PlotSelection.svelte +++ b/web/frontend/src/PlotSelection.svelte @@ -1,17 +1,22 @@ diff --git a/web/frontend/src/Status.root.svelte b/web/frontend/src/Status.root.svelte index 60dcb51..519b95e 100644 --- a/web/frontend/src/Status.root.svelte +++ b/web/frontend/src/Status.root.svelte @@ -4,16 +4,19 @@ import Histogram from './plots/Histogram.svelte' import { Row, Col, Spinner, Card, CardHeader, CardTitle, CardBody, Table, Progress, Icon } from 'sveltestrap' import { init } from './utils.js' - import { operationStore, query } from '@urql/svelte' + import { queryStore, gql, getContextClient } from '@urql/svelte' const { query: initq } = init() export let cluster let plotWidths = [], colWidth1 = 0, colWidth2 - let from = new Date(Date.now() - 5 * 60 * 1000), to = new Date(Date.now()) - const mainQuery = operationStore(`query($cluster: String!, $filter: [JobFilter!]!, $metrics: [String!], $from: Time!, $to: Time!) { + + const client = getContextClient(); + $: mainQuery = queryStore({ + client: client, + query: gql`query($cluster: String!, $filter: [JobFilter!]!, $metrics: [String!], $from: Time!, $to: Time!) { nodeMetrics(cluster: $cluster, metrics: $metrics, from: $from, to: $to) { host subCluster @@ -36,12 +39,11 @@ allocatedNodes(cluster: $cluster) { name, count } topUsers: jobsCount(filter: $filter, groupBy: USER, weight: NODE_COUNT, limit: 10) { name, count } topProjects: jobsCount(filter: $filter, groupBy: PROJECT, weight: NODE_COUNT, limit: 10) { name, count } - }`, { - cluster: cluster, - metrics: ['flops_any', 'mem_bw'], - from: from.toISOString(), - to: to.toISOString(), + }`, + variables: { + cluster: cluster, metrics: ['flops_any', 'mem_bw'], from: from.toISOString(), to: to.toISOString(), filter: [{ state: ['running'] }, { cluster: { eq: cluster } }] + } }) const sumUp = (data, subcluster, metric) => data.reduce((sum, node) => node.subCluster == subcluster @@ -60,7 +62,6 @@ } } - query(mainQuery) @@ -80,13 +81,8 @@ { - console.log('reload...') - from = new Date(Date.now() - 5 * 60 * 1000) to = new Date(Date.now()) - - $mainQuery.variables = { ...$mainQuery.variables, from: from, to: to } - $mainQuery.reexecute({ requestPolicy: 'network-only' }) }} /> diff --git a/web/frontend/src/Systems.root.svelte b/web/frontend/src/Systems.root.svelte index 93fa2f4..ed24928 100644 --- a/web/frontend/src/Systems.root.svelte +++ b/web/frontend/src/Systems.root.svelte @@ -1,7 +1,7 @@ diff --git a/web/frontend/src/TagManagement.svelte b/web/frontend/src/TagManagement.svelte index 747b092..6ab4752 100644 --- a/web/frontend/src/TagManagement.svelte +++ b/web/frontend/src/TagManagement.svelte @@ -1,6 +1,6 @@ @@ -154,8 +172,7 @@
{#if newTagType && newTagName && isNewTag(newTagType, newTagName)} diff --git a/web/frontend/src/User.root.svelte b/web/frontend/src/User.root.svelte index 5021017..7aa6317 100644 --- a/web/frontend/src/User.root.svelte +++ b/web/frontend/src/User.root.svelte @@ -2,7 +2,7 @@ import { onMount, getContext } from 'svelte' import { init } from './utils.js' import { Table, Row, Col, Button, Icon, Card, Spinner } from 'sveltestrap' - import { operationStore, query } from '@urql/svelte' + import { queryStore, gql, getContextClient } from '@urql/svelte' import Filters from './filters/Filters.svelte' import JobList from './joblist/JobList.svelte' import Sorting from './joblist/SortSelection.svelte' @@ -25,31 +25,24 @@ let w1, w2, histogramHeight = 250 let selectedCluster = filterPresets?.cluster ? filterPresets.cluster : null - const stats = operationStore(` - query($filter: [JobFilter!]!) { - jobsStatistics(filter: $filter) { - totalJobs - shortJobs - totalWalltime - totalCoreHours - histDuration { count, value } - histNumNodes { count, value } - } - } - `, { - filter: [] - }, { - pause: true + const client = getContextClient(); + $: stats = queryStore({ + client: client, + query: gql` + query($filters: [JobFilter!]!) { + jobsStatistics(filter: $filters) { + totalJobs + shortJobs + totalWalltime + totalCoreHours + histDuration { count, value } + histNumNodes { count, value } + }}`, + variables: { filters } }) - // filters[filters.findIndex(filter => filter.cluster != null)] ? - // filters[filters.findIndex(filter => filter.cluster != null)].cluster.eq : - // null - // Cluster filter has to be alwas @ first index, above will throw error $: selectedCluster = filters[0]?.cluster ? filters[0].cluster.eq : null - query(stats) - onMount(() => filters.update()) @@ -84,15 +77,12 @@ bind:this={filters} on:update={({ detail }) => { let jobFilters = [...detail.filters, { user: { eq: user.username } }] - $stats.variables = { filter: jobFilters } - $stats.context.pause = false - $stats.reexecute() filters = jobFilters jobList.update(jobFilters) }} />
- jobList.update()} /> + jobList.refresh()} />
diff --git a/web/frontend/src/joblist/JobList.svelte b/web/frontend/src/joblist/JobList.svelte index 0bd5bc5..68e1a76 100644 --- a/web/frontend/src/joblist/JobList.svelte +++ b/web/frontend/src/joblist/JobList.svelte @@ -9,83 +9,144 @@ - update(filters?: [JobFilter]) --> @@ -93,20 +154,43 @@
- {({ USER: 'Username', PROJECT: 'Project Name' })[type]} - Name - Total Jobs - Total Walltime - Total Core Hours -
{$stats.error.message}{$stats.error.message}
- {#if type == 'USER'} - {scrambleNames ? scramble(row.id) : row.id} - {:else if type == 'PROJECT'} - {row.id} + {#if type == "USER"} + {scrambleNames ? scramble(row.id) : row.id} + {:else if type == "PROJECT"} + {row.id} {:else} {row.id} {/if} {row?.name ? row.name : ''}{row?.name ? row.name : ""}{row.totalJobs} {row.totalWalltime}
No {type.toLowerCase()}s/jobs foundNo {type.toLowerCase()}s/jobs found
- {#each metrics as metric (metric)} - {/each} @@ -115,28 +199,27 @@ {#if $jobs.error} - {:else if $jobs.fetching || !$jobs.data} - {:else if $jobs.data && $initialized} {#each $jobs.data.jobs.items as job (job)} - + {:else} - - - + + + {/each} {/if} @@ -145,24 +228,21 @@ { if (detail.itemsPerPage != itemsPerPage) { - itemsPerPage = detail.itemsPerPage - updateConfiguration({ - name: "plot_list_jobsPerPage", - value: itemsPerPage.toString() - }).then(res => { - if (res.error) - console.error(res.error); - }) + updateConfiguration( + detail.itemsPerPage.toString(), + detail.page + ) + } else { + paging = { itemsPerPage: detail.itemsPerPage, page: detail.page } } - - paging = { itemsPerPage: detail.itemsPerPage, page: detail.page } - }} /> + }} +/>
+ Job Info + {metric} {#if $initialized} ({clusters - .map(cluster => cluster.metricConfig.find(m => m.name == metric)) - .filter(m => m != null) - .map(m => (m.unit?.prefix?m.unit?.prefix:'') + (m.unit?.base?m.unit?.base:'')) // Build unitStr - .reduce((arr, unitStr) => arr.includes(unitStr) ? arr : [...arr, unitStr], []) // w/o this, output would be [unitStr, unitStr] - .join(', ') - }) + .map((cluster) => + cluster.metricConfig.find( + (m) => m.name == metric + ) + ) + .filter((m) => m != null) + .map( + (m) => + (m.unit?.prefix + ? m.unit?.prefix + : "") + + (m.unit?.base ? m.unit?.base : "") + ) // Build unitStr + .reduce( + (arr, unitStr) => + arr.includes(unitStr) + ? arr + : [...arr, unitStr], + [] + ) // w/o this, output would be [unitStr, unitStr] + .join(", ")}) {/if}
-

{$jobs.error.message}

+
+

{$jobs.error.message}

+
- No jobs found -
+ No jobs found +