diff --git a/.gitignore b/.gitignore index 2f7c206..e23a17b 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ var/job.db-wal dist/ *.db +internal/repository/testdata/job.db-shm +internal/repository/testdata/job.db-wal diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 19d29cf..3edcb7d 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -34,19 +34,6 @@ builds: main: ./tools/archive-manager tags: - static_build - - env: - - CGO_ENABLED=0 - goos: - - linux - goarch: - - amd64 - goamd64: - - v3 - id: "archive-migration" - binary: archive-migration - main: ./tools/archive-migration - tags: - - static_build - env: - CGO_ENABLED=0 goos: @@ -70,7 +57,7 @@ archives: {{- else }}{{ .Arch }}{{ end }} {{- if .Arm }}v{{ .Arm }}{{ end }} checksum: - name_template: 'checksums.txt' + name_template: "checksums.txt" snapshot: name_template: "{{ incpatch .Version }}-next" changelog: @@ -100,7 +87,7 @@ changelog: release: draft: false footer: | - Supports job archive version 1 and database version 6. + Supports job archive version 2 and database version 8. Please check out the [Release Notes](https://github.com/ClusterCockpit/cc-backend/blob/master/ReleaseNotes.md) for further details on breaking changes. # vim: set ts=2 sw=2 tw=0 fo=cnqoj diff --git a/Makefile b/Makefile index b673e79..52f0d39 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ TARGET = ./cc-backend VAR = ./var CFG = config.json .env FRONTEND = ./web/frontend -VERSION = 1.4.0 +VERSION = 1.4.2 GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development') CURRENT_TIME = $(shell date +"%Y-%m-%d:T%H:%M:%S") LD_FLAGS = '-s -X main.date=${CURRENT_TIME} -X main.version=${VERSION} -X main.commit=${GIT_HASH}' diff --git a/README.md b/README.md index 5ce9125..ce093d2 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ cd ./cc-backend ./startDemo.sh ``` -You can also try the demo using the lates release binary. +You can also try the demo using the latest release binary. Create a folder and put the release binary `cc-backend` into this folder. Execute the following steps: @@ -88,7 +88,9 @@ Analysis, Systems and Status views). There is a Makefile to automate the build of cc-backend. The Makefile supports the following targets: -* `make`: Initialize `var` directory and build svelte frontend and backend binary. Note that there is no proper prerequesite handling. Any change of frontend source files will result in a complete rebuild. +* `make`: Initialize `var` directory and build svelte frontend and backend +binary. Note that there is no proper prerequisite handling. Any change of +frontend source files will result in a complete rebuild. * `make clean`: Clean go build cache and remove binary. * `make test`: Run the tests that are also run in the GitHub workflow setup. @@ -147,8 +149,6 @@ contains Go packages that can be used by other projects. Additional command line helper tools. * [`archive-manager`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/archive-manager) Commands for getting infos about and existing job archive. - * [`archive-migration`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/archive-migration) - Tool to migrate from previous to current job archive version. * [`convert-pem-pubkey`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/convert-pem-pubkey) Tool to convert external pubkey for use in `cc-backend`. * [`gen-keypair`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/gen-keypair) diff --git a/ReleaseNotes.md b/ReleaseNotes.md index 547a1f4..2659964 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -1,11 +1,46 @@ -# `cc-backend` version 1.3.1 +# `cc-backend` version 1.4.2 -Supports job archive version 1 and database version 7. +Supports job archive version 2 and database version 8. -This is a bugfix release of `cc-backend`, the API backend and frontend +This is a small bug fix release of `cc-backend`, the API backend and frontend implementation of ClusterCockpit. For release specific notes visit the [ClusterCockpit Documentation](https://clusterockpit.org/docs/release/). ## Breaking changes -None +- You need to perform a database migration. Depending on your database size the + migration might require several hours! +- You need to adapt the `cluster.json` configuration files in the job-archive, + add new required attributes to the metric list and after that edit + `./job-archive/version.txt` to version 2. Only metrics that have the footprint + attribute set can be filtered and show up in the footprint UI and polar plot. +- Continuous scrolling is default now in all job lists. You can change this back + to paging globally, also every user can configure to use paging or continuous + scrolling individually. +- Tags have a scope now. Existing tags will get global scope in the database + migration. + +## New features + +- Tags have a scope now. Tags created by a basic user are only visible by that + user. Tags created by an admin/support role can be configured to be visible by + all users (global scope) or only be admin/support role. +- Re-sampling support for running (requires a recent `cc-metric-store`) and + archived jobs. This greatly speeds up loading of large or very long jobs. You + need to add the new configuration key `enable-resampling` to the `config.json` + file. +- For finished jobs a total job energy is shown in the job view. +- Continuous scrolling in job lists is default now. +- All database queries (especially for sqlite) were optimized resulting in + dramatically faster load times. +- A performance and energy footprint can be freely configured on a per + subcluster base. One can filter for footprint statistics for running and + finished jobs. + +## Known issues + +- Currently energy footprint metrics of type energy are ignored for calculating + total energy. +- Resampling for running jobs only works with cc-metric-store +- With energy footprint metrics of type power the unit is ignored and it is + assumed the metric has the unit Watt. diff --git a/api/swagger.json b/api/swagger.json index 7f5eaf7..3b59b5e 100644 --- a/api/swagger.json +++ b/api/swagger.json @@ -595,88 +595,6 @@ } } }, - "/jobs/stop_job/{id}": { - "post": { - "security": [ - { - "ApiKeyAuth": [] - } - ], - "description": "Job to stop is specified by database ID. Only stopTime and final state are required in request body.\nReturns full job resource information according to 'JobMeta' scheme.", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Job add and modify" - ], - "summary": "Marks job as completed and triggers archiving", - "parameters": [ - { - "type": "integer", - "description": "Database ID of Job", - "name": "id", - "in": "path", - "required": true - }, - { - "description": "stopTime and final state in request body", - "name": "request", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/api.StopJobApiRequest" - } - } - ], - "responses": { - "200": { - "description": "Job resource", - "schema": { - "$ref": "#/definitions/schema.JobMeta" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/api.ErrorResponse" - } - }, - "401": { - "description": "Unauthorized", - "schema": { - "$ref": "#/definitions/api.ErrorResponse" - } - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/api.ErrorResponse" - } - }, - "404": { - "description": "Resource not found", - "schema": { - "$ref": "#/definitions/api.ErrorResponse" - } - }, - "422": { - "description": "Unprocessable Entity: finding job failed: sql: no rows in result set", - "schema": { - "$ref": "#/definitions/api.ErrorResponse" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/api.ErrorResponse" - } - } - } - } - }, "/jobs/tag_job/{id}": { "post": { "security": [ @@ -684,7 +602,7 @@ "ApiKeyAuth": [] } ], - "description": "Adds tag(s) to a job specified by DB ID. Name and Type of Tag(s) can be chosen freely.\nIf tagged job is already finished: Tag will be written directly to respective archive files.", + "description": "Adds tag(s) to a job specified by DB ID. Name and Type of Tag(s) can be chosen freely.\nTag Scope for frontend visibility will default to \"global\" if none entered, other options: \"admin\" or specific username.\nIf tagged job is already finished: Tag will be written directly to respective archive files.", "consumes": [ "application/json" ], @@ -1277,6 +1195,11 @@ "type": "string", "example": "Testjob" }, + "scope": { + "description": "Tag Scope for Frontend Display", + "type": "string", + "example": "global" + }, "type": { "description": "Tag Type", "type": "string", @@ -1404,9 +1327,8 @@ "api.StartJobApiResponse": { "type": "object", "properties": { - "id": { - "description": "Database ID of new job", - "type": "integer" + "msg": { + "type": "string" } } }, @@ -1418,17 +1340,14 @@ ], "properties": { "cluster": { - "description": "Cluster of job", "type": "string", "example": "fritz" }, "jobId": { - "description": "Cluster Job ID of job", "type": "integer", "example": 123000 }, "jobState": { - "description": "Final job state", "allOf": [ { "$ref": "#/definitions/schema.JobState" @@ -1437,12 +1356,10 @@ "example": "completed" }, "startTime": { - "description": "Start Time of job as epoch", "type": "integer", "example": 1649723812 }, "stopTime": { - "description": "Stop Time of job as epoch", "type": "integer", "example": 1649763839 } @@ -1487,12 +1404,10 @@ "type": "object", "properties": { "arrayJobId": { - "description": "The unique identifier of an array job", "type": "integer", "example": 123000 }, "cluster": { - "description": "The unique identifier of a cluster", "type": "string", "example": "fritz" }, @@ -1500,33 +1415,39 @@ "$ref": "#/definitions/schema.JobLinkResultList" }, "duration": { - "description": "Duration of job in seconds (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 43200 }, + "energy": { + "type": "number" + }, + "energyFootprint": { + "type": "object", + "additionalProperties": { + "type": "number" + } + }, "exclusive": { - "description": "Specifies how nodes are shared: 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple jobs of same user", "type": "integer", "maximum": 2, "minimum": 0, "example": 1 }, - "flopsAnyAvg": { - "description": "FlopsAnyAvg as Float64", - "type": "number" + "footprint": { + "type": "object", + "additionalProperties": { + "type": "number" + } }, "id": { - "description": "The unique identifier of a job in the database", "type": "integer" }, "jobId": { - "description": "The unique identifier of a job", "type": "integer", "example": 123000 }, "jobState": { - "description": "Final state of job", "enum": [ "completed", "failed", @@ -1542,95 +1463,69 @@ ], "example": "completed" }, - "loadAvg": { - "description": "LoadAvg as Float64", - "type": "number" - }, - "memBwAvg": { - "description": "MemBwAvg as Float64", - "type": "number" - }, - "memUsedMax": { - "description": "MemUsedMax as Float64", - "type": "number" - }, "metaData": { - "description": "Additional information about the job", "type": "object", "additionalProperties": { "type": "string" } }, "monitoringStatus": { - "description": "State of monitoring system during job run: 0 - Disabled, 1 - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull", "type": "integer", "maximum": 3, "minimum": 0, "example": 1 }, "numAcc": { - "description": "Number of accelerators used (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 2 }, "numHwthreads": { - "description": "NumCores int32 `json:\"numCores\" db:\"num_cores\" example:\"20\" minimum:\"1\"` // Number of HWThreads used (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 20 }, "numNodes": { - "description": "Number of nodes used (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 2 }, "partition": { - "description": "The Slurm partition to which the job was submitted", "type": "string", "example": "main" }, "project": { - "description": "The unique identifier of a project", "type": "string", "example": "abcd200" }, "resources": { - "description": "Resources used by job", "type": "array", "items": { "$ref": "#/definitions/schema.Resource" } }, "smt": { - "description": "SMT threads used by job", "type": "integer", "example": 4 }, "startTime": { - "description": "Start time as 'time.Time' data type", "type": "string" }, "subCluster": { - "description": "The unique identifier of a sub cluster", "type": "string", "example": "main" }, "tags": { - "description": "List of tags", "type": "array", "items": { "$ref": "#/definitions/schema.Tag" } }, "user": { - "description": "The unique identifier of a user", "type": "string", "example": "abcd100h" }, "walltime": { - "description": "Requested walltime of job in seconds (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 86400 @@ -1667,12 +1562,10 @@ "type": "object", "properties": { "arrayJobId": { - "description": "The unique identifier of an array job", "type": "integer", "example": 123000 }, "cluster": { - "description": "The unique identifier of a cluster", "type": "string", "example": "fritz" }, @@ -1680,29 +1573,39 @@ "$ref": "#/definitions/schema.JobLinkResultList" }, "duration": { - "description": "Duration of job in seconds (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 43200 }, + "energy": { + "type": "number" + }, + "energyFootprint": { + "type": "object", + "additionalProperties": { + "type": "number" + } + }, "exclusive": { - "description": "Specifies how nodes are shared: 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple jobs of same user", "type": "integer", "maximum": 2, "minimum": 0, "example": 1 }, + "footprint": { + "type": "object", + "additionalProperties": { + "type": "number" + } + }, "id": { - "description": "The unique identifier of a job in the database", "type": "integer" }, "jobId": { - "description": "The unique identifier of a job", "type": "integer", "example": 123000 }, "jobState": { - "description": "Final state of job", "enum": [ "completed", "failed", @@ -1719,91 +1622,76 @@ "example": "completed" }, "metaData": { - "description": "Additional information about the job", "type": "object", "additionalProperties": { "type": "string" } }, "monitoringStatus": { - "description": "State of monitoring system during job run: 0 - Disabled, 1 - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull", "type": "integer", "maximum": 3, "minimum": 0, "example": 1 }, "numAcc": { - "description": "Number of accelerators used (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 2 }, "numHwthreads": { - "description": "NumCores int32 `json:\"numCores\" db:\"num_cores\" example:\"20\" minimum:\"1\"` // Number of HWThreads used (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 20 }, "numNodes": { - "description": "Number of nodes used (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 2 }, "partition": { - "description": "The Slurm partition to which the job was submitted", "type": "string", "example": "main" }, "project": { - "description": "The unique identifier of a project", "type": "string", "example": "abcd200" }, "resources": { - "description": "Resources used by job", "type": "array", "items": { "$ref": "#/definitions/schema.Resource" } }, "smt": { - "description": "SMT threads used by job", "type": "integer", "example": 4 }, "startTime": { - "description": "Start epoch time stamp in seconds (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 1649723812 }, "statistics": { - "description": "Metric statistics of job", "type": "object", "additionalProperties": { "$ref": "#/definitions/schema.JobStatistics" } }, "subCluster": { - "description": "The unique identifier of a sub cluster", "type": "string", "example": "main" }, "tags": { - "description": "List of tags", "type": "array", "items": { "$ref": "#/definitions/schema.Tag" } }, "user": { - "description": "The unique identifier of a user", "type": "string", "example": "abcd100h" }, "walltime": { - "description": "Requested walltime of job in seconds (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 86400 @@ -1892,6 +1780,15 @@ "caution": { "type": "number" }, + "energy": { + "type": "string" + }, + "footprint": { + "type": "string" + }, + "lowerIsBetter": { + "type": "boolean" + }, "name": { "type": "string" }, @@ -1969,22 +1866,18 @@ "type": "object", "properties": { "accelerators": { - "description": "List of of accelerator device ids", "type": "array", "items": { "type": "string" } }, "configuration": { - "description": "The configuration options of the node", "type": "string" }, "hostname": { - "description": "Name of the host (= node)", "type": "string" }, "hwthreads": { - "description": "List of OS processor ids", "type": "array", "items": { "type": "integer" @@ -2027,6 +1920,12 @@ "type": "number" } }, + "median": { + "type": "array", + "items": { + "type": "number" + } + }, "min": { "type": "array", "items": { @@ -2050,15 +1949,33 @@ "coresPerSocket": { "type": "integer" }, + "energyFootprint": { + "type": "array", + "items": { + "type": "string" + } + }, "flopRateScalar": { "$ref": "#/definitions/schema.MetricValue" }, "flopRateSimd": { "$ref": "#/definitions/schema.MetricValue" }, + "footprint": { + "type": "array", + "items": { + "type": "string" + } + }, "memoryBandwidth": { "$ref": "#/definitions/schema.MetricValue" }, + "metricConfig": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.MetricConfig" + } + }, "name": { "type": "string" }, @@ -2088,6 +2005,15 @@ "caution": { "type": "number" }, + "energy": { + "type": "string" + }, + "footprint": { + "type": "string" + }, + "lowerIsBetter": { + "type": "boolean" + }, "name": { "type": "string" }, @@ -2107,16 +2033,17 @@ "type": "object", "properties": { "id": { - "description": "The unique DB identifier of a tag", "type": "integer" }, "name": { - "description": "Tag Name", "type": "string", "example": "Testjob" }, + "scope": { + "type": "string", + "example": "global" + }, "type": { - "description": "Tag Type", "type": "string", "example": "Debug" } diff --git a/api/swagger.yaml b/api/swagger.yaml index f47ac3f..4e3c47e 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -23,6 +23,10 @@ definitions: description: Tag Name example: Testjob type: string + scope: + description: Tag Scope for Frontend Display + example: global + type: string type: description: Tag Type example: Debug @@ -110,31 +114,25 @@ definitions: type: object api.StartJobApiResponse: properties: - id: - description: Database ID of new job - type: integer + msg: + type: string type: object api.StopJobApiRequest: properties: cluster: - description: Cluster of job example: fritz type: string jobId: - description: Cluster Job ID of job example: 123000 type: integer jobState: allOf: - $ref: '#/definitions/schema.JobState' - description: Final job state example: completed startTime: - description: Start Time of job as epoch example: 1649723812 type: integer stopTime: - description: Stop Time of job as epoch example: 1649763839 type: integer required: @@ -167,42 +165,40 @@ definitions: description: Information of a HPC job. properties: arrayJobId: - description: The unique identifier of an array job example: 123000 type: integer cluster: - description: The unique identifier of a cluster example: fritz type: string concurrentJobs: $ref: '#/definitions/schema.JobLinkResultList' duration: - description: Duration of job in seconds (Min > 0) example: 43200 minimum: 1 type: integer + energy: + type: number + energyFootprint: + additionalProperties: + type: number + type: object exclusive: - description: 'Specifies how nodes are shared: 0 - Shared among multiple jobs - of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple - jobs of same user' example: 1 maximum: 2 minimum: 0 type: integer - flopsAnyAvg: - description: FlopsAnyAvg as Float64 - type: number + footprint: + additionalProperties: + type: number + type: object id: - description: The unique identifier of a job in the database type: integer jobId: - description: The unique identifier of a job example: 123000 type: integer jobState: allOf: - $ref: '#/definitions/schema.JobState' - description: Final state of job enum: - completed - failed @@ -211,79 +207,53 @@ definitions: - timeout - out_of_memory example: completed - loadAvg: - description: LoadAvg as Float64 - type: number - memBwAvg: - description: MemBwAvg as Float64 - type: number - memUsedMax: - description: MemUsedMax as Float64 - type: number metaData: additionalProperties: type: string - description: Additional information about the job type: object monitoringStatus: - description: 'State of monitoring system during job run: 0 - Disabled, 1 - - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull' example: 1 maximum: 3 minimum: 0 type: integer numAcc: - description: Number of accelerators used (Min > 0) example: 2 minimum: 1 type: integer numHwthreads: - description: NumCores int32 `json:"numCores" db:"num_cores" - example:"20" minimum:"1"` // - Number of HWThreads used (Min > 0) example: 20 minimum: 1 type: integer numNodes: - description: Number of nodes used (Min > 0) example: 2 minimum: 1 type: integer partition: - description: The Slurm partition to which the job was submitted example: main type: string project: - description: The unique identifier of a project example: abcd200 type: string resources: - description: Resources used by job items: $ref: '#/definitions/schema.Resource' type: array smt: - description: SMT threads used by job example: 4 type: integer startTime: - description: Start time as 'time.Time' data type type: string subCluster: - description: The unique identifier of a sub cluster example: main type: string tags: - description: List of tags items: $ref: '#/definitions/schema.Tag' type: array user: - description: The unique identifier of a user example: abcd100h type: string walltime: - description: Requested walltime of job in seconds (Min > 0) example: 86400 minimum: 1 type: integer @@ -308,39 +278,40 @@ definitions: description: Meta data information of a HPC job. properties: arrayJobId: - description: The unique identifier of an array job example: 123000 type: integer cluster: - description: The unique identifier of a cluster example: fritz type: string concurrentJobs: $ref: '#/definitions/schema.JobLinkResultList' duration: - description: Duration of job in seconds (Min > 0) example: 43200 minimum: 1 type: integer + energy: + type: number + energyFootprint: + additionalProperties: + type: number + type: object exclusive: - description: 'Specifies how nodes are shared: 0 - Shared among multiple jobs - of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple - jobs of same user' example: 1 maximum: 2 minimum: 0 type: integer + footprint: + additionalProperties: + type: number + type: object id: - description: The unique identifier of a job in the database type: integer jobId: - description: The unique identifier of a job example: 123000 type: integer jobState: allOf: - $ref: '#/definitions/schema.JobState' - description: Final state of job enum: - completed - failed @@ -352,74 +323,56 @@ definitions: metaData: additionalProperties: type: string - description: Additional information about the job type: object monitoringStatus: - description: 'State of monitoring system during job run: 0 - Disabled, 1 - - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull' example: 1 maximum: 3 minimum: 0 type: integer numAcc: - description: Number of accelerators used (Min > 0) example: 2 minimum: 1 type: integer numHwthreads: - description: NumCores int32 `json:"numCores" db:"num_cores" - example:"20" minimum:"1"` // - Number of HWThreads used (Min > 0) example: 20 minimum: 1 type: integer numNodes: - description: Number of nodes used (Min > 0) example: 2 minimum: 1 type: integer partition: - description: The Slurm partition to which the job was submitted example: main type: string project: - description: The unique identifier of a project example: abcd200 type: string resources: - description: Resources used by job items: $ref: '#/definitions/schema.Resource' type: array smt: - description: SMT threads used by job example: 4 type: integer startTime: - description: Start epoch time stamp in seconds (Min > 0) example: 1649723812 minimum: 1 type: integer statistics: additionalProperties: $ref: '#/definitions/schema.JobStatistics' - description: Metric statistics of job type: object subCluster: - description: The unique identifier of a sub cluster example: main type: string tags: - description: List of tags items: $ref: '#/definitions/schema.Tag' type: array user: - description: The unique identifier of a user example: abcd100h type: string walltime: - description: Requested walltime of job in seconds (Min > 0) example: 86400 minimum: 1 type: integer @@ -486,6 +439,12 @@ definitions: type: number caution: type: number + energy: + type: string + footprint: + type: string + lowerIsBetter: + type: boolean name: type: string normal: @@ -541,18 +500,14 @@ definitions: description: A resource used by a job properties: accelerators: - description: List of of accelerator device ids items: type: string type: array configuration: - description: The configuration options of the node type: string hostname: - description: Name of the host (= node) type: string hwthreads: - description: List of OS processor ids items: type: integer type: array @@ -580,6 +535,10 @@ definitions: items: type: number type: array + median: + items: + type: number + type: array min: items: type: number @@ -595,12 +554,24 @@ definitions: properties: coresPerSocket: type: integer + energyFootprint: + items: + type: string + type: array flopRateScalar: $ref: '#/definitions/schema.MetricValue' flopRateSimd: $ref: '#/definitions/schema.MetricValue' + footprint: + items: + type: string + type: array memoryBandwidth: $ref: '#/definitions/schema.MetricValue' + metricConfig: + items: + $ref: '#/definitions/schema.MetricConfig' + type: array name: type: string nodes: @@ -620,6 +591,12 @@ definitions: type: number caution: type: number + energy: + type: string + footprint: + type: string + lowerIsBetter: + type: boolean name: type: string normal: @@ -633,14 +610,14 @@ definitions: description: Defines a tag using name and type. properties: id: - description: The unique DB identifier of a tag type: integer name: - description: Tag Name example: Testjob type: string + scope: + example: global + type: string type: - description: Tag Type example: Debug type: string type: object @@ -1197,68 +1174,13 @@ paths: summary: Marks job as completed and triggers archiving tags: - Job add and modify - /jobs/stop_job/{id}: - post: - consumes: - - application/json - description: |- - Job to stop is specified by database ID. Only stopTime and final state are required in request body. - Returns full job resource information according to 'JobMeta' scheme. - parameters: - - description: Database ID of Job - in: path - name: id - required: true - type: integer - - description: stopTime and final state in request body - in: body - name: request - required: true - schema: - $ref: '#/definitions/api.StopJobApiRequest' - produces: - - application/json - responses: - "200": - description: Job resource - schema: - $ref: '#/definitions/schema.JobMeta' - "400": - description: Bad Request - schema: - $ref: '#/definitions/api.ErrorResponse' - "401": - description: Unauthorized - schema: - $ref: '#/definitions/api.ErrorResponse' - "403": - description: Forbidden - schema: - $ref: '#/definitions/api.ErrorResponse' - "404": - description: Resource not found - schema: - $ref: '#/definitions/api.ErrorResponse' - "422": - description: 'Unprocessable Entity: finding job failed: sql: no rows in - result set' - schema: - $ref: '#/definitions/api.ErrorResponse' - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/api.ErrorResponse' - security: - - ApiKeyAuth: [] - summary: Marks job as completed and triggers archiving - tags: - - Job add and modify /jobs/tag_job/{id}: post: consumes: - application/json description: |- Adds tag(s) to a job specified by DB ID. Name and Type of Tag(s) can be chosen freely. + Tag Scope for frontend visibility will default to "global" if none entered, other options: "admin" or specific username. If tagged job is already finished: Tag will be written directly to respective archive files. parameters: - description: Job Database ID diff --git a/cmd/cc-backend/cli.go b/cmd/cc-backend/cli.go index f828a24..8bc6681 100644 --- a/cmd/cc-backend/cli.go +++ b/cmd/cc-backend/cli.go @@ -14,7 +14,7 @@ var ( func cliInit() { flag.BoolVar(&flagInit, "init", false, "Setup var directory, initialize swlite database file, config.json and .env") flag.BoolVar(&flagReinitDB, "init-db", false, "Go through job-archive and re-initialize the 'job', 'tag', and 'jobtag' tables (all running jobs will be lost!)") - flag.BoolVar(&flagSyncLDAP, "sync-ldap", false, "Sync the 'user' table with ldap") + flag.BoolVar(&flagSyncLDAP, "sync-ldap", false, "Sync the 'hpc_user' table with ldap") flag.BoolVar(&flagServer, "server", false, "Start a server, continues listening on port after initialization and argument handling") flag.BoolVar(&flagGops, "gops", false, "Listen via github.com/google/gops/agent (for debugging)") flag.BoolVar(&flagDev, "dev", false, "Enable development components: GraphQL Playground and Swagger UI") diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index 9f7e673..33bab07 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -13,6 +13,7 @@ import ( "sync" "syscall" + "github.com/ClusterCockpit/cc-backend/internal/archiver" "github.com/ClusterCockpit/cc-backend/internal/auth" "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/importer" @@ -111,7 +112,7 @@ func main() { if flagInit { initEnv() - fmt.Print("Succesfully setup environment!\n") + fmt.Print("Successfully setup environment!\n") fmt.Print("Please review config.json and .env and adjust it to your needs.\n") fmt.Print("Add your job-archive at ./var/job-archive.\n") os.Exit(0) @@ -201,16 +202,10 @@ func main() { return } + archiver.Start(repository.GetJobRepository()) taskManager.Start() serverInit() - // Because this program will want to bind to a privileged port (like 80), the listener must - // be established first, then the user can be changed, and after that, - // the actual http server can be started. - if err := runtimeEnv.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil { - log.Fatalf("error while preparing server start: %s", err.Error()) - } - var wg sync.WaitGroup wg.Add(1) diff --git a/cmd/cc-backend/server.go b/cmd/cc-backend/server.go index bc20fcf..0770e81 100644 --- a/cmd/cc-backend/server.go +++ b/cmd/cc-backend/server.go @@ -27,6 +27,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/graph/generated" "github.com/ClusterCockpit/cc-backend/internal/routerConfig" "github.com/ClusterCockpit/cc-backend/pkg/log" + "github.com/ClusterCockpit/cc-backend/pkg/runtimeEnv" "github.com/ClusterCockpit/cc-backend/web" "github.com/gorilla/handlers" "github.com/gorilla/mux" @@ -108,9 +109,7 @@ func serverInit() { if !config.Keys.DisableAuthentication { router.Handle("/login", authHandle.Login( - // On success: - http.RedirectHandler("/", http.StatusTemporaryRedirect), - + // On success: Handled within Login() // On failure: func(rw http.ResponseWriter, r *http.Request, err error) { rw.Header().Add("Content-Type", "text/html; charset=utf-8") @@ -125,9 +124,7 @@ func serverInit() { })).Methods(http.MethodPost) router.Handle("/jwt-login", authHandle.Login( - // On success: - http.RedirectHandler("/", http.StatusTemporaryRedirect), - + // On success: Handled within Login() // On failure: func(rw http.ResponseWriter, r *http.Request, err error) { rw.Header().Add("Content-Type", "text/html; charset=utf-8") @@ -163,11 +160,12 @@ func serverInit() { func(rw http.ResponseWriter, r *http.Request, err error) { rw.WriteHeader(http.StatusUnauthorized) web.RenderTemplate(rw, "login.tmpl", &web.Page{ - Title: "Authentication failed - ClusterCockpit", - MsgType: "alert-danger", - Message: err.Error(), - Build: buildInfo, - Infos: info, + Title: "Authentication failed - ClusterCockpit", + MsgType: "alert-danger", + Message: err.Error(), + Build: buildInfo, + Infos: info, + Redirect: r.RequestURI, }) }) }) @@ -298,6 +296,13 @@ func serverStart() { } else { fmt.Printf("HTTP server listening at %s...", config.Keys.Addr) } + // + // Because this program will want to bind to a privileged port (like 80), the listener must + // be established first, then the user can be changed, and after that, + // the actual http server can be started. + if err := runtimeEnv.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil { + log.Fatalf("error while preparing server start: %s", err.Error()) + } if err = server.Serve(listener); err != nil && err != http.ErrServerClosed { log.Fatalf("starting server failed: %v", err) diff --git a/configs/config-demo.json b/configs/config-demo.json index 8423758..e8d4570 100644 --- a/configs/config-demo.json +++ b/configs/config-demo.json @@ -1,56 +1,67 @@ { - "addr": "127.0.0.1:8080", - "archive": { - "kind": "file", - "path": "./var/job-archive" - }, - "jwts": { - "max-age": "2000h" - }, - "clusters": [ - { - "name": "fritz", - "metricDataRepository": { - "kind": "cc-metric-store", - "url": "http://localhost:8082", - "token": "" - }, - "filterRanges": { - "numNodes": { - "from": 1, - "to": 64 - }, - "duration": { - "from": 0, - "to": 86400 - }, - "startTime": { - "from": "2022-01-01T00:00:00Z", - "to": null - } - } - }, - { - "name": "alex", - "metricDataRepository": { - "kind": "cc-metric-store", - "url": "http://localhost:8082", - "token": "" - }, - "filterRanges": { - "numNodes": { - "from": 1, - "to": 64 - }, - "duration": { - "from": 0, - "to": 86400 - }, - "startTime": { - "from": "2022-01-01T00:00:00Z", - "to": null - } - } - } + "addr": "127.0.0.1:8080", + "short-running-jobs-duration": 300, + "archive": { + "kind": "file", + "path": "./var/job-archive" + }, + "jwts": { + "max-age": "2000h" + }, + "enable-resampling": { + "trigger": 30, + "resolutions": [ + 600, + 300, + 120, + 60 ] + }, + "emission-constant": 317, + "clusters": [ + { + "name": "fritz", + "metricDataRepository": { + "kind": "cc-metric-store", + "url": "http://localhost:8082", + "token": "" + }, + "filterRanges": { + "numNodes": { + "from": 1, + "to": 64 + }, + "duration": { + "from": 0, + "to": 86400 + }, + "startTime": { + "from": "2022-01-01T00:00:00Z", + "to": null + } + } + }, + { + "name": "alex", + "metricDataRepository": { + "kind": "cc-metric-store", + "url": "http://localhost:8082", + "token": "" + }, + "filterRanges": { + "numNodes": { + "from": 1, + "to": 64 + }, + "duration": { + "from": 0, + "to": 86400 + }, + "startTime": { + "from": "2022-01-01T00:00:00Z", + "to": null + } + } + } + ] } diff --git a/configs/config-mariadb.json b/configs/config-mariadb.json new file mode 100644 index 0000000..e068439 --- /dev/null +++ b/configs/config-mariadb.json @@ -0,0 +1,69 @@ +{ + "addr": "127.0.0.1:8080", + "short-running-jobs-duration": 300, + "archive": { + "kind": "file", + "path": "./var/job-archive" + }, + "jwts": { + "max-age": "2000h" + }, + "db-driver": "mysql", + "db": "clustercockpit:demo@tcp(127.0.0.1:3306)/clustercockpit", + "enable-resampling": { + "trigger": 30, + "resolutions": [ + 600, + 300, + 120, + 60 + ] + }, + "emission-constant": 317, + "clusters": [ + { + "name": "fritz", + "metricDataRepository": { + "kind": "cc-metric-store", + "url": "http://localhost:8082", + "token": "" + }, + "filterRanges": { + "numNodes": { + "from": 1, + "to": 64 + }, + "duration": { + "from": 0, + "to": 86400 + }, + "startTime": { + "from": "2022-01-01T00:00:00Z", + "to": null + } + } + }, + { + "name": "alex", + "metricDataRepository": { + "kind": "cc-metric-store", + "url": "http://localhost:8082", + "token": "" + }, + "filterRanges": { + "numNodes": { + "from": 1, + "to": 64 + }, + "duration": { + "from": 0, + "to": 86400 + }, + "startTime": { + "from": "2022-01-01T00:00:00Z", + "to": null + } + } + } + ] +} diff --git a/configs/generate-subcluster.pl b/configs/generate-subcluster.pl index 171db92..7648358 100755 --- a/configs/generate-subcluster.pl +++ b/configs/generate-subcluster.pl @@ -117,10 +117,12 @@ foreach my $ln (split("\n", $topo)) { my $node; my @sockets; +my @nodeCores; foreach my $socket ( @{$DOMAINS{socket}} ) { push @sockets, "[".join(",", @{$socket})."]"; - $node .= join(",", @{$socket}) + push @nodeCores, join(",", @{$socket}); } +$node = join(",", @nodeCores); $INFO{sockets} = join(",\n", @sockets); my @memDomains; @@ -212,9 +214,27 @@ print <<"END"; "socketsPerNode": $INFO{socketsPerNode}, "coresPerSocket": $INFO{coresPerSocket}, "threadsPerCore": $INFO{threadsPerCore}, - "flopRateScalar": $flopsScalar, - "flopRateSimd": $flopsSimd, - "memoryBandwidth": $memBw, + "flopRateScalar": { + "unit": { + "base": "F/s", + "prefix": "G" + }, + "value": $flopsScalar + }, + "flopRateSimd": { + "unit": { + "base": "F/s", + "prefix": "G" + }, + "value": $flopsSimd + }, + "memoryBandwidth": { + "unit": { + "base": "B/s", + "prefix": "G" + }, + "value": $memBw + }, "nodes": "", "topology": { "node": [$node], diff --git a/go.mod b/go.mod index e343d65..27a703c 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,9 @@ module github.com/ClusterCockpit/cc-backend -go 1.22 +go 1.23 require ( - github.com/99designs/gqlgen v0.17.49 + github.com/99designs/gqlgen v0.17.57 github.com/ClusterCockpit/cc-units v0.4.0 github.com/Masterminds/squirrel v1.5.4 github.com/coreos/go-oidc/v3 v3.11.0 @@ -15,7 +15,7 @@ require ( github.com/google/gops v0.3.28 github.com/gorilla/handlers v1.5.2 github.com/gorilla/mux v1.8.1 - github.com/gorilla/sessions v1.3.0 + github.com/gorilla/sessions v1.4.0 github.com/influxdata/influxdb-client-go/v2 v2.13.0 github.com/jmoiron/sqlx v1.4.0 github.com/mattn/go-sqlite3 v1.14.22 @@ -24,9 +24,9 @@ require ( github.com/qustavo/sqlhooks/v2 v2.1.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/swaggo/http-swagger v1.3.4 - github.com/swaggo/swag v1.16.3 - github.com/vektah/gqlparser/v2 v2.5.16 - golang.org/x/crypto v0.25.0 + github.com/swaggo/swag v1.16.4 + github.com/vektah/gqlparser/v2 v2.5.20 + golang.org/x/crypto v0.29.0 golang.org/x/exp v0.0.0-20240707233637-46b078467d37 golang.org/x/oauth2 v0.21.0 ) @@ -35,11 +35,11 @@ require ( filippo.io/edwards25519 v1.1.0 // indirect github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/KyleBanks/depth v1.2.1 // indirect - github.com/agnivade/levenshtein v1.1.1 // indirect + github.com/agnivade/levenshtein v1.2.0 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect github.com/go-jose/go-jose/v4 v4.0.3 // indirect @@ -47,6 +47,7 @@ require ( github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/spec v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-viper/mapstructure/v2 v2.2.1 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/securecookie v1.1.2 // indirect github.com/gorilla/websocket v1.5.3 // indirect @@ -61,7 +62,6 @@ require ( github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -74,16 +74,16 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sosodev/duration v1.3.1 // indirect github.com/swaggo/files v1.0.1 // indirect - github.com/urfave/cli/v2 v2.27.2 // indirect - github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect + github.com/urfave/cli/v2 v2.27.5 // indirect + github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect go.uber.org/atomic v1.11.0 // indirect - golang.org/x/mod v0.19.0 // indirect - golang.org/x/net v0.27.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.22.0 // indirect - golang.org/x/text v0.16.0 // indirect - golang.org/x/tools v0.23.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect + golang.org/x/mod v0.22.0 // indirect + golang.org/x/net v0.31.0 // indirect + golang.org/x/sync v0.9.0 // indirect + golang.org/x/sys v0.27.0 // indirect + golang.org/x/text v0.20.0 // indirect + golang.org/x/tools v0.27.0 // indirect + google.golang.org/protobuf v1.35.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect diff --git a/go.sum b/go.sum index d8759fc..6506264 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= -github.com/99designs/gqlgen v0.17.49 h1:b3hNGexHd33fBSAd4NDT/c3NCcQzcAVkknhN9ym36YQ= -github.com/99designs/gqlgen v0.17.49/go.mod h1:tC8YFVZMed81x7UJ7ORUwXF4Kn6SXuucFqQBhN8+BU0= +github.com/99designs/gqlgen v0.17.57 h1:Ak4p60BRq6QibxY0lEc0JnQhDurfhxA67sp02lMjmPc= +github.com/99designs/gqlgen v0.17.57/go.mod h1:Jx61hzOSTcR4VJy/HFIgXiQ5rJ0Ypw8DxWLjbYDAUw0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= @@ -14,11 +14,11 @@ github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8 github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE= -github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk= +github.com/PuerkitoBio/goquery v1.9.3 h1:mpJr/ikUA9/GNJB/DBZcGeFDXUtosHRyRrwh7KGdTG0= +github.com/PuerkitoBio/goquery v1.9.3/go.mod h1:1ndLHPdTz+DyQPICCWYlYQMPl0oXZj0G6D4LCYA6u4U= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= -github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8= -github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= +github.com/agnivade/levenshtein v1.2.0 h1:U9L4IOT0Y3i0TIlUIDJ7rVUziKi/zPbrJGaFrtYH3SY= +github.com/agnivade/levenshtein v1.2.0/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU= github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI= github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= @@ -36,13 +36,13 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/coreos/go-oidc/v3 v3.11.0 h1:Ia3MxdwpSw702YW0xgfmP1GVCMA9aEFWu12XUZ3/OtI= github.com/coreos/go-oidc/v3 v3.11.0/go.mod h1:gE3LgjOgFoHi9a4ce4/tJczr0Ai2/BoDhf0r5lltWI0= -github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc= +github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g= -github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= +github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54 h1:SG7nF6SRlWhcT7cNTs5R6Hk4V2lcmLz2NsG2VnInyNo= +github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= github.com/dhui/dktest v0.4.1 h1:/w+IWuDXVymg3IrRJCHHOkMK10m9aNVMOyD0X12YVTg= github.com/dhui/dktest v0.4.1/go.mod h1:DdOqcUpL7vgyP4GlF3X3w7HbSlz8cEQzwewPveYEQbA= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= @@ -75,6 +75,8 @@ github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= +github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= @@ -99,8 +101,8 @@ github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+ github.com/gorilla/securecookie v1.1.2 h1:YCIWL56dvtr73r6715mJs5ZvhtnY73hBvEF8kXD8ePA= github.com/gorilla/securecookie v1.1.2/go.mod h1:NfCASbcHqRSY+3a8tlWJwsQap2VX5pwzwo4h3eOamfo= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= -github.com/gorilla/sessions v1.3.0 h1:XYlkq7KcpOB2ZhHBPv5WpjMIxrQosiZanfoy1HLZFzg= -github.com/gorilla/sessions v1.3.0/go.mod h1:ePLdVu+jbEgHH+KWw8I1z2wqd0BAdAQh/8LRvBeoNcQ= +github.com/gorilla/sessions v1.4.0 h1:kpIYOp/oi6MG/p5PgxApU8srsSw9tuFbt46Lt7auzqQ= +github.com/gorilla/sessions v1.4.0/go.mod h1:FLWm50oby91+hl7p/wRxDth9bWSuk0qVL2emc7lT5ik= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -156,8 +158,6 @@ github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -214,20 +214,20 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE= github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg= github.com/swaggo/http-swagger v1.3.4 h1:q7t/XLx0n15H1Q9/tk3Y9L4n210XzJF5WtnDX64a5ww= github.com/swaggo/http-swagger v1.3.4/go.mod h1:9dAh0unqMBAlbp1uE2Uc2mQTxNMU/ha4UbucIg1MFkQ= -github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg= -github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk= -github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI= -github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM= -github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0vy5p8= -github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= -github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw= -github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk= +github.com/swaggo/swag v1.16.4 h1:clWJtd9LStiG3VeijiCfOVODP6VpHtKdQy9ELFG3s1A= +github.com/swaggo/swag v1.16.4/go.mod h1:VBsHJRsDvfYvqoiMKnsdwhNV9LEMHgEDZcyVYX0sxPg= +github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w= +github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= +github.com/vektah/gqlparser/v2 v2.5.20 h1:kPaWbhBntxoZPaNdBaIPT1Kh0i1b/onb5kXgEdP5JCo= +github.com/vektah/gqlparser/v2 v2.5.20/go.mod h1:xMl+ta8a5M1Yo1A1Iwt/k7gSpscwSnHZdw7tfhEGfTM= +github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= +github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= @@ -238,14 +238,14 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= -golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= +golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= golang.org/x/exp v0.0.0-20240707233637-46b078467d37 h1:uLDX+AfeFCct3a2C7uIWBKMJIR3CJMhcgfrUAqjRK6w= golang.org/x/exp v0.0.0-20240707233637-46b078467d37/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= -golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -255,15 +255,15 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= -golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= +golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= +golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -273,8 +273,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -287,17 +287,17 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= +golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= -golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= +golang.org/x/tools v0.27.0 h1:qEKojBykQkQ4EynWy4S8Weg69NumxKdn40Fce3uc/8o= +golang.org/x/tools v0.27.0/go.mod h1:sUi0ZgbwW9ZPAq26Ekut+weQPR5eIM6GQLQ1Yjm1H0Q= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= +google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/init/clustercockpit.service b/init/clustercockpit.service index 53fc429..0a9448d 100644 --- a/init/clustercockpit.service +++ b/init/clustercockpit.service @@ -1,5 +1,5 @@ [Unit] -Description=ClusterCockpit Web Server (Go edition) +Description=ClusterCockpit Web Server Documentation=https://github.com/ClusterCockpit/cc-backend Wants=network-online.target After=network-online.target diff --git a/internal/api/api_test.go b/internal/api/api_test.go index 0312e43..c47bd4d 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -14,9 +14,9 @@ import ( "os" "path/filepath" "reflect" - "strconv" "strings" "testing" + "time" "github.com/ClusterCockpit/cc-backend/internal/api" "github.com/ClusterCockpit/cc-backend/internal/archiver" @@ -120,7 +120,7 @@ func setup(t *testing.T) *api.RestApi { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(jobarchive, "version.txt"), []byte(fmt.Sprintf("%d", 1)), 0666); err != nil { + if err := os.WriteFile(filepath.Join(jobarchive, "version.txt"), []byte(fmt.Sprintf("%d", 2)), 0666); err != nil { t.Fatal(err) } @@ -200,6 +200,10 @@ func TestRestApi(t *testing.T) { r.StrictSlash(true) restapi.MountApiRoutes(r) + var TestJobId int64 = 123 + var TestClusterName string = "testcluster" + var TestStartTime int64 = 123456789 + const startJobBody string = `{ "jobId": 123, "user": "testuser", @@ -225,7 +229,6 @@ func TestRestApi(t *testing.T) { "startTime": 123456789 }` - var dbid int64 const contextUserKey repository.ContextKey = "user" contextUserValue := &schema.User{ Username: "testuser", @@ -246,14 +249,8 @@ func TestRestApi(t *testing.T) { if response.StatusCode != http.StatusCreated { t.Fatal(response.Status, recorder.Body.String()) } - - var res api.StartJobApiResponse - if err := json.Unmarshal(recorder.Body.Bytes(), &res); err != nil { - t.Fatal(err) - } - resolver := graph.GetResolverInstance() - job, err := resolver.Query().Job(ctx, strconv.Itoa(int(res.DBID))) + job, err := restapi.JobRepository.Find(&TestJobId, &TestClusterName, &TestStartTime) if err != nil { t.Fatal(err) } @@ -285,8 +282,6 @@ func TestRestApi(t *testing.T) { if len(job.Tags) != 1 || job.Tags[0].Type != "testTagType" || job.Tags[0].Name != "testTagName" || job.Tags[0].Scope != "testuser" { t.Fatalf("unexpected tags: %#v", job.Tags) } - - dbid = res.DBID }); !ok { return } @@ -314,8 +309,7 @@ func TestRestApi(t *testing.T) { } archiver.WaitForArchiving() - resolver := graph.GetResolverInstance() - job, err := resolver.Query().Job(ctx, strconv.Itoa(int(dbid))) + job, err := restapi.JobRepository.Find(&TestJobId, &TestClusterName, &TestStartTime) if err != nil { t.Fatal(err) } @@ -404,8 +398,10 @@ func TestRestApi(t *testing.T) { t.Fatal("subtest failed") } + time.Sleep(1 * time.Second) + const stopJobBodyFailed string = `{ - "jobId": 12345, + "jobId": 12345, "cluster": "testcluster", "jobState": "failed", diff --git a/internal/api/docs.go b/internal/api/docs.go index e5ec50b..7c1daac 100644 --- a/internal/api/docs.go +++ b/internal/api/docs.go @@ -601,88 +601,6 @@ const docTemplate = `{ } } }, - "/jobs/stop_job/{id}": { - "post": { - "security": [ - { - "ApiKeyAuth": [] - } - ], - "description": "Job to stop is specified by database ID. Only stopTime and final state are required in request body.\nReturns full job resource information according to 'JobMeta' scheme.", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Job add and modify" - ], - "summary": "Marks job as completed and triggers archiving", - "parameters": [ - { - "type": "integer", - "description": "Database ID of Job", - "name": "id", - "in": "path", - "required": true - }, - { - "description": "stopTime and final state in request body", - "name": "request", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/api.StopJobApiRequest" - } - } - ], - "responses": { - "200": { - "description": "Job resource", - "schema": { - "$ref": "#/definitions/schema.JobMeta" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/api.ErrorResponse" - } - }, - "401": { - "description": "Unauthorized", - "schema": { - "$ref": "#/definitions/api.ErrorResponse" - } - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/api.ErrorResponse" - } - }, - "404": { - "description": "Resource not found", - "schema": { - "$ref": "#/definitions/api.ErrorResponse" - } - }, - "422": { - "description": "Unprocessable Entity: finding job failed: sql: no rows in result set", - "schema": { - "$ref": "#/definitions/api.ErrorResponse" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/api.ErrorResponse" - } - } - } - } - }, "/jobs/tag_job/{id}": { "post": { "security": [ @@ -690,7 +608,7 @@ const docTemplate = `{ "ApiKeyAuth": [] } ], - "description": "Adds tag(s) to a job specified by DB ID. Name and Type of Tag(s) can be chosen freely.\nIf tagged job is already finished: Tag will be written directly to respective archive files.", + "description": "Adds tag(s) to a job specified by DB ID. Name and Type of Tag(s) can be chosen freely.\nTag Scope for frontend visibility will default to \"global\" if none entered, other options: \"admin\" or specific username.\nIf tagged job is already finished: Tag will be written directly to respective archive files.", "consumes": [ "application/json" ], @@ -1283,6 +1201,11 @@ const docTemplate = `{ "type": "string", "example": "Testjob" }, + "scope": { + "description": "Tag Scope for Frontend Display", + "type": "string", + "example": "global" + }, "type": { "description": "Tag Type", "type": "string", @@ -1410,9 +1333,8 @@ const docTemplate = `{ "api.StartJobApiResponse": { "type": "object", "properties": { - "id": { - "description": "Database ID of new job", - "type": "integer" + "msg": { + "type": "string" } } }, @@ -1424,17 +1346,14 @@ const docTemplate = `{ ], "properties": { "cluster": { - "description": "Cluster of job", "type": "string", "example": "fritz" }, "jobId": { - "description": "Cluster Job ID of job", "type": "integer", "example": 123000 }, "jobState": { - "description": "Final job state", "allOf": [ { "$ref": "#/definitions/schema.JobState" @@ -1443,12 +1362,10 @@ const docTemplate = `{ "example": "completed" }, "startTime": { - "description": "Start Time of job as epoch", "type": "integer", "example": 1649723812 }, "stopTime": { - "description": "Stop Time of job as epoch", "type": "integer", "example": 1649763839 } @@ -1493,12 +1410,10 @@ const docTemplate = `{ "type": "object", "properties": { "arrayJobId": { - "description": "The unique identifier of an array job", "type": "integer", "example": 123000 }, "cluster": { - "description": "The unique identifier of a cluster", "type": "string", "example": "fritz" }, @@ -1506,33 +1421,39 @@ const docTemplate = `{ "$ref": "#/definitions/schema.JobLinkResultList" }, "duration": { - "description": "Duration of job in seconds (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 43200 }, + "energy": { + "type": "number" + }, + "energyFootprint": { + "type": "object", + "additionalProperties": { + "type": "number" + } + }, "exclusive": { - "description": "Specifies how nodes are shared: 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple jobs of same user", "type": "integer", "maximum": 2, "minimum": 0, "example": 1 }, - "flopsAnyAvg": { - "description": "FlopsAnyAvg as Float64", - "type": "number" + "footprint": { + "type": "object", + "additionalProperties": { + "type": "number" + } }, "id": { - "description": "The unique identifier of a job in the database", "type": "integer" }, "jobId": { - "description": "The unique identifier of a job", "type": "integer", "example": 123000 }, "jobState": { - "description": "Final state of job", "enum": [ "completed", "failed", @@ -1548,95 +1469,69 @@ const docTemplate = `{ ], "example": "completed" }, - "loadAvg": { - "description": "LoadAvg as Float64", - "type": "number" - }, - "memBwAvg": { - "description": "MemBwAvg as Float64", - "type": "number" - }, - "memUsedMax": { - "description": "MemUsedMax as Float64", - "type": "number" - }, "metaData": { - "description": "Additional information about the job", "type": "object", "additionalProperties": { "type": "string" } }, "monitoringStatus": { - "description": "State of monitoring system during job run: 0 - Disabled, 1 - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull", "type": "integer", "maximum": 3, "minimum": 0, "example": 1 }, "numAcc": { - "description": "Number of accelerators used (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 2 }, "numHwthreads": { - "description": "NumCores int32 ` + "`" + `json:\"numCores\" db:\"num_cores\" example:\"20\" minimum:\"1\"` + "`" + ` // Number of HWThreads used (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 20 }, "numNodes": { - "description": "Number of nodes used (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 2 }, "partition": { - "description": "The Slurm partition to which the job was submitted", "type": "string", "example": "main" }, "project": { - "description": "The unique identifier of a project", "type": "string", "example": "abcd200" }, "resources": { - "description": "Resources used by job", "type": "array", "items": { "$ref": "#/definitions/schema.Resource" } }, "smt": { - "description": "SMT threads used by job", "type": "integer", "example": 4 }, "startTime": { - "description": "Start time as 'time.Time' data type", "type": "string" }, "subCluster": { - "description": "The unique identifier of a sub cluster", "type": "string", "example": "main" }, "tags": { - "description": "List of tags", "type": "array", "items": { "$ref": "#/definitions/schema.Tag" } }, "user": { - "description": "The unique identifier of a user", "type": "string", "example": "abcd100h" }, "walltime": { - "description": "Requested walltime of job in seconds (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 86400 @@ -1673,12 +1568,10 @@ const docTemplate = `{ "type": "object", "properties": { "arrayJobId": { - "description": "The unique identifier of an array job", "type": "integer", "example": 123000 }, "cluster": { - "description": "The unique identifier of a cluster", "type": "string", "example": "fritz" }, @@ -1686,29 +1579,39 @@ const docTemplate = `{ "$ref": "#/definitions/schema.JobLinkResultList" }, "duration": { - "description": "Duration of job in seconds (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 43200 }, + "energy": { + "type": "number" + }, + "energyFootprint": { + "type": "object", + "additionalProperties": { + "type": "number" + } + }, "exclusive": { - "description": "Specifies how nodes are shared: 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple jobs of same user", "type": "integer", "maximum": 2, "minimum": 0, "example": 1 }, + "footprint": { + "type": "object", + "additionalProperties": { + "type": "number" + } + }, "id": { - "description": "The unique identifier of a job in the database", "type": "integer" }, "jobId": { - "description": "The unique identifier of a job", "type": "integer", "example": 123000 }, "jobState": { - "description": "Final state of job", "enum": [ "completed", "failed", @@ -1725,91 +1628,76 @@ const docTemplate = `{ "example": "completed" }, "metaData": { - "description": "Additional information about the job", "type": "object", "additionalProperties": { "type": "string" } }, "monitoringStatus": { - "description": "State of monitoring system during job run: 0 - Disabled, 1 - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull", "type": "integer", "maximum": 3, "minimum": 0, "example": 1 }, "numAcc": { - "description": "Number of accelerators used (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 2 }, "numHwthreads": { - "description": "NumCores int32 ` + "`" + `json:\"numCores\" db:\"num_cores\" example:\"20\" minimum:\"1\"` + "`" + ` // Number of HWThreads used (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 20 }, "numNodes": { - "description": "Number of nodes used (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 2 }, "partition": { - "description": "The Slurm partition to which the job was submitted", "type": "string", "example": "main" }, "project": { - "description": "The unique identifier of a project", "type": "string", "example": "abcd200" }, "resources": { - "description": "Resources used by job", "type": "array", "items": { "$ref": "#/definitions/schema.Resource" } }, "smt": { - "description": "SMT threads used by job", "type": "integer", "example": 4 }, "startTime": { - "description": "Start epoch time stamp in seconds (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 1649723812 }, "statistics": { - "description": "Metric statistics of job", "type": "object", "additionalProperties": { "$ref": "#/definitions/schema.JobStatistics" } }, "subCluster": { - "description": "The unique identifier of a sub cluster", "type": "string", "example": "main" }, "tags": { - "description": "List of tags", "type": "array", "items": { "$ref": "#/definitions/schema.Tag" } }, "user": { - "description": "The unique identifier of a user", "type": "string", "example": "abcd100h" }, "walltime": { - "description": "Requested walltime of job in seconds (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 86400 @@ -1898,6 +1786,15 @@ const docTemplate = `{ "caution": { "type": "number" }, + "energy": { + "type": "string" + }, + "footprint": { + "type": "string" + }, + "lowerIsBetter": { + "type": "boolean" + }, "name": { "type": "string" }, @@ -1975,22 +1872,18 @@ const docTemplate = `{ "type": "object", "properties": { "accelerators": { - "description": "List of of accelerator device ids", "type": "array", "items": { "type": "string" } }, "configuration": { - "description": "The configuration options of the node", "type": "string" }, "hostname": { - "description": "Name of the host (= node)", "type": "string" }, "hwthreads": { - "description": "List of OS processor ids", "type": "array", "items": { "type": "integer" @@ -2033,6 +1926,12 @@ const docTemplate = `{ "type": "number" } }, + "median": { + "type": "array", + "items": { + "type": "number" + } + }, "min": { "type": "array", "items": { @@ -2056,15 +1955,33 @@ const docTemplate = `{ "coresPerSocket": { "type": "integer" }, + "energyFootprint": { + "type": "array", + "items": { + "type": "string" + } + }, "flopRateScalar": { "$ref": "#/definitions/schema.MetricValue" }, "flopRateSimd": { "$ref": "#/definitions/schema.MetricValue" }, + "footprint": { + "type": "array", + "items": { + "type": "string" + } + }, "memoryBandwidth": { "$ref": "#/definitions/schema.MetricValue" }, + "metricConfig": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.MetricConfig" + } + }, "name": { "type": "string" }, @@ -2094,6 +2011,15 @@ const docTemplate = `{ "caution": { "type": "number" }, + "energy": { + "type": "string" + }, + "footprint": { + "type": "string" + }, + "lowerIsBetter": { + "type": "boolean" + }, "name": { "type": "string" }, @@ -2113,16 +2039,17 @@ const docTemplate = `{ "type": "object", "properties": { "id": { - "description": "The unique DB identifier of a tag", "type": "integer" }, "name": { - "description": "Tag Name", "type": "string", "example": "Testjob" }, + "scope": { + "type": "string", + "example": "global" + }, "type": { - "description": "Tag Type", "type": "string", "example": "Debug" } diff --git a/internal/api/rest.go b/internal/api/rest.go index 62b2f19..4e52701 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -72,7 +72,6 @@ func (api *RestApi) MountApiRoutes(r *mux.Router) { r.HandleFunc("/jobs/start_job/", api.startJob).Methods(http.MethodPost, http.MethodPut) r.HandleFunc("/jobs/stop_job/", api.stopJobByRequest).Methods(http.MethodPost, http.MethodPut) - r.HandleFunc("/jobs/stop_job/{id}", api.stopJobById).Methods(http.MethodPost, http.MethodPut) // r.HandleFunc("/jobs/import/", api.importJob).Methods(http.MethodPost, http.MethodPut) r.HandleFunc("/jobs/", api.getJobs).Methods(http.MethodGet) @@ -111,6 +110,7 @@ func (api *RestApi) MountConfigApiRoutes(r *mux.Router) { r.HandleFunc("/users/", api.getUsers).Methods(http.MethodGet) r.HandleFunc("/users/", api.deleteUser).Methods(http.MethodDelete) r.HandleFunc("/user/{id}", api.updateUser).Methods(http.MethodPost) + r.HandleFunc("/notice/", api.editNotice).Methods(http.MethodPost) } } @@ -123,19 +123,8 @@ func (api *RestApi) MountFrontendApiRoutes(r *mux.Router) { } } -// StartJobApiResponse model -type StartJobApiResponse struct { - // Database ID of new job - DBID int64 `json:"id"` -} - -// DeleteJobApiResponse model -type DeleteJobApiResponse struct { - Message string `json:"msg"` -} - -// UpdateUserApiResponse model -type UpdateUserApiResponse struct { +// DefaultApiResponse model +type DefaultJobApiResponse struct { Message string `json:"msg"` } @@ -342,7 +331,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { withMetadata := false filter := &model.JobFilter{} page := &model.PageRequest{ItemsPerPage: 25, Page: 1} - order := &model.OrderByInput{Field: "startTime", Order: model.SortDirectionEnumDesc} + order := &model.OrderByInput{Field: "startTime", Type: "col", Order: model.SortDirectionEnumDesc} for key, vals := range r.URL.Query() { switch key { @@ -421,7 +410,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { StartTime: job.StartTime.Unix(), } - res.Tags, err = api.JobRepository.GetTags(r.Context(), &job.ID) + res.Tags, err = api.JobRepository.GetTags(repository.GetUserFromContext(r.Context()), &job.ID) if err != nil { handleError(err, http.StatusInternalServerError, rw) return @@ -486,15 +475,15 @@ func (api *RestApi) getCompleteJobById(rw http.ResponseWriter, r *http.Request) job, err = api.JobRepository.FindById(r.Context(), id) // Get Job from Repo by ID } else { - handleError(errors.New("the parameter 'id' is required"), http.StatusBadRequest, rw) + handleError(fmt.Errorf("the parameter 'id' is required"), http.StatusBadRequest, rw) return } if err != nil { - handleError(fmt.Errorf("finding job failed: %w", err), http.StatusUnprocessableEntity, rw) + handleError(fmt.Errorf("finding job with db id %s failed: %w", id, err), http.StatusUnprocessableEntity, rw) return } - job.Tags, err = api.JobRepository.GetTags(r.Context(), &job.ID) + job.Tags, err = api.JobRepository.GetTags(repository.GetUserFromContext(r.Context()), &job.ID) if err != nil { handleError(err, http.StatusInternalServerError, rw) return @@ -526,7 +515,7 @@ func (api *RestApi) getCompleteJobById(rw http.ResponseWriter, r *http.Request) if r.URL.Query().Get("all-metrics") == "true" { data, err = metricDataDispatcher.LoadData(job, nil, scopes, r.Context(), resolution) if err != nil { - log.Warn("Error while loading job data") + log.Warnf("REST: error while loading all-metrics job data for JobID %d on %s", job.JobID, job.Cluster) return } } @@ -583,11 +572,11 @@ func (api *RestApi) getJobById(rw http.ResponseWriter, r *http.Request) { return } if err != nil { - handleError(fmt.Errorf("finding job failed: %w", err), http.StatusUnprocessableEntity, rw) + handleError(fmt.Errorf("finding job with db id %s failed: %w", id, err), http.StatusUnprocessableEntity, rw) return } - job.Tags, err = api.JobRepository.GetTags(r.Context(), &job.ID) + job.Tags, err = api.JobRepository.GetTags(repository.GetUserFromContext(r.Context()), &job.ID) if err != nil { handleError(err, http.StatusInternalServerError, rw) return @@ -622,7 +611,7 @@ func (api *RestApi) getJobById(rw http.ResponseWriter, r *http.Request) { data, err := metricDataDispatcher.LoadData(job, metrics, scopes, r.Context(), resolution) if err != nil { - log.Warn("Error while loading job data") + log.Warnf("REST: error while loading job data for JobID %d on %s", job.JobID, job.Cluster) return } @@ -728,7 +717,7 @@ func (api *RestApi) tagJob(rw http.ResponseWriter, r *http.Request) { return } - job.Tags, err = api.JobRepository.GetTags(r.Context(), &job.ID) + job.Tags, err = api.JobRepository.GetTags(repository.GetUserFromContext(r.Context()), &job.ID) if err != nil { http.Error(rw, err.Error(), http.StatusInternalServerError) return @@ -741,7 +730,7 @@ func (api *RestApi) tagJob(rw http.ResponseWriter, r *http.Request) { } for _, tag := range req { - tagId, err := api.JobRepository.AddTagOrCreate(r.Context(), job.ID, tag.Type, tag.Name, tag.Scope) + tagId, err := api.JobRepository.AddTagOrCreate(repository.GetUserFromContext(r.Context()), job.ID, tag.Type, tag.Name, tag.Scope) if err != nil { http.Error(rw, err.Error(), http.StatusInternalServerError) return @@ -819,7 +808,7 @@ func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) { unlockOnce.Do(api.RepositoryMutex.Unlock) for _, tag := range req.Tags { - if _, err := api.JobRepository.AddTagOrCreate(r.Context(), id, tag.Type, tag.Name, tag.Scope); err != nil { + if _, err := api.JobRepository.AddTagOrCreate(repository.GetUserFromContext(r.Context()), id, tag.Type, tag.Name, tag.Scope); err != nil { http.Error(rw, err.Error(), http.StatusInternalServerError) handleError(fmt.Errorf("adding tag to new job %d failed: %w", id, err), http.StatusInternalServerError, rw) return @@ -829,61 +818,11 @@ func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) { log.Printf("new job (id: %d): cluster=%s, jobId=%d, user=%s, startTime=%d", id, req.Cluster, req.JobID, req.User, req.StartTime) rw.Header().Add("Content-Type", "application/json") rw.WriteHeader(http.StatusCreated) - json.NewEncoder(rw).Encode(StartJobApiResponse{ - DBID: id, + json.NewEncoder(rw).Encode(DefaultJobApiResponse{ + Message: "success", }) } -// stopJobById godoc -// @summary Marks job as completed and triggers archiving -// @tags Job add and modify -// @description Job to stop is specified by database ID. Only stopTime and final state are required in request body. -// @description Returns full job resource information according to 'JobMeta' scheme. -// @accept json -// @produce json -// @param id path int true "Database ID of Job" -// @param request body api.StopJobApiRequest true "stopTime and final state in request body" -// @success 200 {object} schema.JobMeta "Job resource" -// @failure 400 {object} api.ErrorResponse "Bad Request" -// @failure 401 {object} api.ErrorResponse "Unauthorized" -// @failure 403 {object} api.ErrorResponse "Forbidden" -// @failure 404 {object} api.ErrorResponse "Resource not found" -// @failure 422 {object} api.ErrorResponse "Unprocessable Entity: finding job failed: sql: no rows in result set" -// @failure 500 {object} api.ErrorResponse "Internal Server Error" -// @security ApiKeyAuth -// @router /jobs/stop_job/{id} [post] -func (api *RestApi) stopJobById(rw http.ResponseWriter, r *http.Request) { - // Parse request body: Only StopTime and State - req := StopJobApiRequest{} - if err := decode(r.Body, &req); err != nil { - handleError(fmt.Errorf("parsing request body failed: %w", err), http.StatusBadRequest, rw) - return - } - - // Fetch job (that will be stopped) from db - id, ok := mux.Vars(r)["id"] - var job *schema.Job - var err error - if ok { - id, e := strconv.ParseInt(id, 10, 64) - if e != nil { - handleError(fmt.Errorf("integer expected in path for id: %w", e), http.StatusBadRequest, rw) - return - } - - job, err = api.JobRepository.FindById(r.Context(), id) - } else { - handleError(errors.New("the parameter 'id' is required"), http.StatusBadRequest, rw) - return - } - if err != nil { - handleError(fmt.Errorf("finding job failed: %w", err), http.StatusUnprocessableEntity, rw) - return - } - - api.checkAndHandleStopJob(rw, job, req) -} - // stopJobByRequest godoc // @summary Marks job as completed and triggers archiving // @tags Job add and modify @@ -916,6 +855,7 @@ func (api *RestApi) stopJobByRequest(rw http.ResponseWriter, r *http.Request) { return } + // log.Printf("loading db job for stopJobByRequest... : stopJobApiRequest=%v", req) job, err = api.JobRepository.Find(req.JobId, req.Cluster, req.StartTime) if err != nil { handleError(fmt.Errorf("finding job failed: %w", err), http.StatusUnprocessableEntity, rw) @@ -962,7 +902,7 @@ func (api *RestApi) deleteJobById(rw http.ResponseWriter, r *http.Request) { } rw.Header().Add("Content-Type", "application/json") rw.WriteHeader(http.StatusOK) - json.NewEncoder(rw).Encode(DeleteJobApiResponse{ + json.NewEncoder(rw).Encode(DefaultJobApiResponse{ Message: fmt.Sprintf("Successfully deleted job %s", id), }) } @@ -1013,7 +953,7 @@ func (api *RestApi) deleteJobByRequest(rw http.ResponseWriter, r *http.Request) rw.Header().Add("Content-Type", "application/json") rw.WriteHeader(http.StatusOK) - json.NewEncoder(rw).Encode(DeleteJobApiResponse{ + json.NewEncoder(rw).Encode(DefaultJobApiResponse{ Message: fmt.Sprintf("Successfully deleted job %d", job.ID), }) } @@ -1057,7 +997,7 @@ func (api *RestApi) deleteJobBefore(rw http.ResponseWriter, r *http.Request) { rw.Header().Add("Content-Type", "application/json") rw.WriteHeader(http.StatusOK) - json.NewEncoder(rw).Encode(DeleteJobApiResponse{ + json.NewEncoder(rw).Encode(DefaultJobApiResponse{ Message: fmt.Sprintf("Successfully deleted %d jobs", cnt), }) } @@ -1065,12 +1005,12 @@ func (api *RestApi) deleteJobBefore(rw http.ResponseWriter, r *http.Request) { func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Job, req StopJobApiRequest) { // Sanity checks if job == nil || job.StartTime.Unix() >= req.StopTime || job.State != schema.JobStateRunning { - handleError(errors.New("stopTime must be larger than startTime and only running jobs can be stopped"), http.StatusBadRequest, rw) + handleError(fmt.Errorf("jobId %d (id %d) on %s : stopTime %d must be larger than startTime %d and only running jobs can be stopped (state is: %s)", job.JobID, job.ID, job.Cluster, req.StopTime, job.StartTime.Unix(), job.State), http.StatusBadRequest, rw) return } if req.State != "" && !req.State.Valid() { - handleError(fmt.Errorf("invalid job state: %#v", req.State), http.StatusBadRequest, rw) + handleError(fmt.Errorf("jobId %d (id %d) on %s : invalid requested job state: %#v", job.JobID, job.ID, job.Cluster, req.State), http.StatusBadRequest, rw) return } else if req.State == "" { req.State = schema.JobStateCompleted @@ -1080,11 +1020,11 @@ func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Jo job.Duration = int32(req.StopTime - job.StartTime.Unix()) job.State = req.State if err := api.JobRepository.Stop(job.ID, job.Duration, job.State, job.MonitoringStatus); err != nil { - handleError(fmt.Errorf("marking job as stopped failed: %w", err), http.StatusInternalServerError, rw) + handleError(fmt.Errorf("jobId %d (id %d) on %s : marking job as '%s' (duration: %d) in DB failed: %w", job.JobID, job.ID, job.Cluster, job.State, job.Duration, err), http.StatusInternalServerError, rw) return } - log.Printf("archiving job... (dbid: %d): cluster=%s, jobId=%d, user=%s, startTime=%s", job.ID, job.Cluster, job.JobID, job.User, job.StartTime) + log.Printf("archiving job... (dbid: %d): cluster=%s, jobId=%d, user=%s, startTime=%s, duration=%d, state=%s", job.ID, job.Cluster, job.JobID, job.User, job.StartTime, job.Duration, job.State) // Send a response (with status OK). This means that erros that happen from here on forward // can *NOT* be communicated to the client. If reading from a MetricDataRepository or @@ -1356,6 +1296,69 @@ func (api *RestApi) updateUser(rw http.ResponseWriter, r *http.Request) { } } +// editNotice godoc +// @summary Updates or empties the notice box content +// @tags User +// @description Modifies the content of notice.txt, shown as notice box on the homepage. +// @description If more than one formValue is set then only the highest priority field is used. +// @description Only accessible from IPs registered with apiAllowedIPs configuration option. +// @accept mpfd +// @produce plain +// @param new-content formData string false "Priority 1: New content to display" +// @success 200 {string} string "Success Response Message" +// @failure 400 {string} string "Bad Request" +// @failure 401 {string} string "Unauthorized" +// @failure 403 {string} string "Forbidden" +// @failure 422 {string} string "Unprocessable Entity: The user could not be updated" +// @failure 500 {string} string "Internal Server Error" +// @security ApiKeyAuth +// @router /notice/ [post] +func (api *RestApi) editNotice(rw http.ResponseWriter, r *http.Request) { + err := securedCheck(r) + if err != nil { + http.Error(rw, err.Error(), http.StatusForbidden) + return + } + + if user := repository.GetUserFromContext(r.Context()); !user.HasRole(schema.RoleAdmin) { + http.Error(rw, "Only admins are allowed to update the notice.txt file", http.StatusForbidden) + return + } + + // Get Value + newContent := r.FormValue("new-content") + + // Check FIle + noticeExists := util.CheckFileExists("./var/notice.txt") + if !noticeExists { + ntxt, err := os.Create("./var/notice.txt") + if err != nil { + log.Errorf("Creating ./var/notice.txt failed: %s", err.Error()) + http.Error(rw, err.Error(), http.StatusUnprocessableEntity) + return + } + ntxt.Close() + } + + if newContent != "" { + if err := os.WriteFile("./var/notice.txt", []byte(newContent), 0o666); err != nil { + log.Errorf("Writing to ./var/notice.txt failed: %s", err.Error()) + http.Error(rw, err.Error(), http.StatusUnprocessableEntity) + return + } else { + rw.Write([]byte("Update Notice Content Success")) + } + } else { + if err := os.WriteFile("./var/notice.txt", []byte(""), 0o666); err != nil { + log.Errorf("Writing to ./var/notice.txt failed: %s", err.Error()) + http.Error(rw, err.Error(), http.StatusUnprocessableEntity) + return + } else { + rw.Write([]byte("Empty Notice Content Success")) + } + } +} + func (api *RestApi) getJWT(rw http.ResponseWriter, r *http.Request) { err := securedCheck(r) if err != nil { diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 270989f..15b6532 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -188,6 +188,10 @@ func (auth *Authentication) SaveSession(rw http.ResponseWriter, r *http.Request, if auth.SessionMaxAge != 0 { session.Options.MaxAge = int(auth.SessionMaxAge.Seconds()) } + if config.Keys.HttpsCertFile == "" && config.Keys.HttpsKeyFile == "" { + session.Options.Secure = false + } + session.Options.SameSite = http.SameSiteStrictMode session.Values["username"] = user.Username session.Values["projects"] = user.Projects session.Values["roles"] = user.Roles @@ -201,7 +205,6 @@ func (auth *Authentication) SaveSession(rw http.ResponseWriter, r *http.Request, } func (auth *Authentication) Login( - onsuccess http.Handler, onfailure func(rw http.ResponseWriter, r *http.Request, loginErr error), ) http.Handler { return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { @@ -238,7 +241,13 @@ func (auth *Authentication) Login( log.Infof("login successfull: user: %#v (roles: %v, projects: %v)", user.Username, user.Roles, user.Projects) ctx := context.WithValue(r.Context(), repository.ContextUserKey, user) - onsuccess.ServeHTTP(rw, r.WithContext(ctx)) + + if r.FormValue("redirect") != "" { + http.RedirectHandler(r.FormValue("redirect"), http.StatusFound).ServeHTTP(rw, r.WithContext(ctx)) + return + } + + http.RedirectHandler("/", http.StatusFound).ServeHTTP(rw, r.WithContext(ctx)) return } diff --git a/internal/config/config.go b/internal/config/config.go index 0217d85..4f1a8c3 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -29,10 +29,9 @@ var Keys schema.ProgramConfig = schema.ProgramConfig{ "analysis_view_histogramMetrics": []string{"flops_any", "mem_bw", "mem_used"}, "analysis_view_scatterPlotMetrics": [][]string{{"flops_any", "mem_bw"}, {"flops_any", "cpu_load"}, {"cpu_load", "mem_bw"}}, "job_view_nodestats_selectedMetrics": []string{"flops_any", "mem_bw", "mem_used"}, - "job_view_polarPlotMetrics": []string{"flops_any", "mem_bw", "mem_used"}, "job_view_selectedMetrics": []string{"flops_any", "mem_bw", "mem_used"}, "job_view_showFootprint": true, - "job_list_usePaging": true, + "job_list_usePaging": false, "plot_general_colorBackground": true, "plot_general_colorscheme": []string{"#00bfff", "#0000ff", "#ff00ff", "#ff0000", "#ff8000", "#ffff00", "#80ff00"}, "plot_general_lineWidth": 3, diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index 73090a8..96fea41 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -31,15 +31,12 @@ func (r *clusterResolver) Partitions(ctx context.Context, obj *schema.Cluster) ( // Tags is the resolver for the tags field. func (r *jobResolver) Tags(ctx context.Context, obj *schema.Job) ([]*schema.Tag, error) { - return r.Repo.GetTags(ctx, &obj.ID) + return r.Repo.GetTags(repository.GetUserFromContext(ctx), &obj.ID) } // ConcurrentJobs is the resolver for the concurrentJobs field. func (r *jobResolver) ConcurrentJobs(ctx context.Context, obj *schema.Job) (*model.JobLinkResultList, error) { - if obj.State == schema.JobStateRunning { - obj.Duration = int32(time.Now().Unix() - obj.StartTimeUnix) - } - + // FIXME: Make the hardcoded duration configurable if obj.Exclusive != 1 && obj.Duration > 600 { return r.Repo.FindConcurrentJobs(ctx, obj) } @@ -159,7 +156,7 @@ func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds return nil, err } - if tags, err = r.Repo.AddTag(ctx, jid, tid); err != nil { + if tags, err = r.Repo.AddTag(repository.GetUserFromContext(ctx), jid, tid); err != nil { log.Warn("Error while adding tag") return nil, err } @@ -185,7 +182,7 @@ func (r *mutationResolver) RemoveTagsFromJob(ctx context.Context, job string, ta return nil, err } - if tags, err = r.Repo.RemoveTag(ctx, jid, tid); err != nil { + if tags, err = r.Repo.RemoveTag(repository.GetUserFromContext(ctx), jid, tid); err != nil { log.Warn("Error while removing tag") return nil, err } @@ -211,7 +208,7 @@ func (r *queryResolver) Clusters(ctx context.Context) ([]*schema.Cluster, error) // Tags is the resolver for the tags field. func (r *queryResolver) Tags(ctx context.Context) ([]*schema.Tag, error) { - return r.Repo.GetTags(ctx, nil) + return r.Repo.GetTags(repository.GetUserFromContext(ctx), nil) } // GlobalMetrics is the resolver for the globalMetrics field. @@ -496,9 +493,11 @@ func (r *Resolver) Query() generated.QueryResolver { return &queryResolver{r} } // SubCluster returns generated.SubClusterResolver implementation. func (r *Resolver) SubCluster() generated.SubClusterResolver { return &subClusterResolver{r} } -type clusterResolver struct{ *Resolver } -type jobResolver struct{ *Resolver } -type metricValueResolver struct{ *Resolver } -type mutationResolver struct{ *Resolver } -type queryResolver struct{ *Resolver } -type subClusterResolver struct{ *Resolver } +type ( + clusterResolver struct{ *Resolver } + jobResolver struct{ *Resolver } + metricValueResolver struct{ *Resolver } + mutationResolver struct{ *Resolver } + queryResolver struct{ *Resolver } + subClusterResolver struct{ *Resolver } +) diff --git a/internal/importer/handleImport.go b/internal/importer/handleImport.go index 153402a..01773a5 100644 --- a/internal/importer/handleImport.go +++ b/internal/importer/handleImport.go @@ -8,6 +8,7 @@ import ( "bytes" "encoding/json" "fmt" + "math" "os" "strings" @@ -84,7 +85,8 @@ func HandleImportFlag(flag string) error { } name := fmt.Sprintf("%s_%s", fp, statType) - job.Footprint[fp] = repository.LoadJobStat(&job, name, statType) + + job.Footprint[name] = repository.LoadJobStat(&job, fp, statType) } job.RawFootprint, err = json.Marshal(job.Footprint) @@ -92,6 +94,34 @@ func HandleImportFlag(flag string) error { log.Warn("Error while marshaling job footprint") return err } + + job.EnergyFootprint = make(map[string]float64) + var totalEnergy float64 + var energy float64 + + for _, fp := range sc.EnergyFootprint { + if i, err := archive.MetricIndex(sc.MetricConfig, fp); err == nil { + // Note: For DB data, calculate and save as kWh + // Energy: Power (in Watts) * Time (in Seconds) + if sc.MetricConfig[i].Energy == "energy" { // this metric has energy as unit (Joules) + } else if sc.MetricConfig[i].Energy == "power" { // this metric has power as unit (Watt) + // Unit: ( W * s ) / 3600 / 1000 = kWh ; Rounded to 2 nearest digits + energy = math.Round(((repository.LoadJobStat(&job, fp, "avg")*float64(job.Duration))/3600/1000)*100) / 100 + } + } else { + log.Warnf("Error while collecting energy metric %s for job, DB ID '%v', return '0.0'", fp, job.ID) + } + + job.EnergyFootprint[fp] = energy + totalEnergy += energy + } + + job.Energy = (math.Round(totalEnergy*100) / 100) + if job.RawEnergyFootprint, err = json.Marshal(job.EnergyFootprint); err != nil { + log.Warnf("Error while marshaling energy footprint for job INTO BYTES, DB ID '%v'", job.ID) + return err + } + job.RawResources, err = json.Marshal(job.Resources) if err != nil { log.Warn("Error while marshaling job resources") diff --git a/internal/importer/importer_test.go b/internal/importer/importer_test.go index ce0d2e1..4e839cf 100644 --- a/internal/importer/importer_test.go +++ b/internal/importer/importer_test.go @@ -82,7 +82,7 @@ func setup(t *testing.T) *repository.JobRepository { if err := os.Mkdir(jobarchive, 0777); err != nil { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(jobarchive, "version.txt"), []byte(fmt.Sprintf("%d", 1)), 0666); err != nil { + if err := os.WriteFile(filepath.Join(jobarchive, "version.txt"), []byte(fmt.Sprintf("%d", 2)), 0666); err != nil { t.Fatal(err) } fritzArchive := filepath.Join(tmpdir, "job-archive", "fritz") diff --git a/internal/importer/initDB.go b/internal/importer/initDB.go index 5f06f36..fa2ee6e 100644 --- a/internal/importer/initDB.go +++ b/internal/importer/initDB.go @@ -7,6 +7,7 @@ package importer import ( "encoding/json" "fmt" + "math" "strings" "time" @@ -70,6 +71,7 @@ func InitDB() error { log.Errorf("cannot get subcluster: %s", err.Error()) return err } + job.Footprint = make(map[string]float64) for _, fp := range sc.Footprint { @@ -81,7 +83,7 @@ func InitDB() error { name := fmt.Sprintf("%s_%s", fp, statType) - job.Footprint[fp] = repository.LoadJobStat(jobMeta, name, statType) + job.Footprint[name] = repository.LoadJobStat(jobMeta, fp, statType) } job.RawFootprint, err = json.Marshal(job.Footprint) @@ -90,6 +92,33 @@ func InitDB() error { return err } + job.EnergyFootprint = make(map[string]float64) + var totalEnergy float64 + var energy float64 + + for _, fp := range sc.EnergyFootprint { + if i, err := archive.MetricIndex(sc.MetricConfig, fp); err == nil { + // Note: For DB data, calculate and save as kWh + // Energy: Power (in Watts) * Time (in Seconds) + if sc.MetricConfig[i].Energy == "energy" { // this metric has energy as unit (Joules) + } else if sc.MetricConfig[i].Energy == "power" { // this metric has power as unit (Watt) + // Unit: ( W * s ) / 3600 / 1000 = kWh ; Rounded to 2 nearest digits + energy = math.Round(((repository.LoadJobStat(jobMeta, fp, "avg")*float64(jobMeta.Duration))/3600/1000)*100) / 100 + } + } else { + log.Warnf("Error while collecting energy metric %s for job, DB ID '%v', return '0.0'", fp, jobMeta.ID) + } + + job.EnergyFootprint[fp] = energy + totalEnergy += energy + } + + job.Energy = (math.Round(totalEnergy*100) / 100) + if job.RawEnergyFootprint, err = json.Marshal(job.EnergyFootprint); err != nil { + log.Warnf("Error while marshaling energy footprint for job INTO BYTES, DB ID '%v'", jobMeta.ID) + return err + } + job.RawResources, err = json.Marshal(job.Resources) if err != nil { log.Errorf("repository initDB(): %v", err) diff --git a/internal/repository/job.go b/internal/repository/job.go index d1c54a1..11f3b46 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -51,7 +51,7 @@ func GetJobRepository() *JobRepository { } var jobColumns []string = []string{ - "job.id", "job.job_id", "job.user", "job.project", "job.cluster", "job.subcluster", "job.start_time", "job.partition", "job.array_job_id", + "job.id", "job.job_id", "job.hpc_user", "job.project", "job.cluster", "job.subcluster", "job.start_time", "job.cluster_partition", "job.array_job_id", "job.num_nodes", "job.num_hwthreads", "job.num_acc", "job.exclusive", "job.monitoring_status", "job.smt", "job.job_state", "job.duration", "job.walltime", "job.resources", "job.footprint", "job.energy", } @@ -79,12 +79,9 @@ func scanJob(row interface{ Scan(...interface{}) error }) (*schema.Job, error) { } job.RawFootprint = nil - // if err := json.Unmarshal(job.RawMetaData, &job.MetaData); err != nil { - // return nil, err - // } - job.StartTime = time.Unix(job.StartTimeUnix, 0) - if job.Duration == 0 && job.State == schema.JobStateRunning { + // Always ensure accurate duration for running jobs + if job.State == schema.JobStateRunning { job.Duration = int32(time.Since(job.StartTime).Seconds()) } @@ -308,17 +305,17 @@ func (r *JobRepository) FindUserOrProjectOrJobname(user *schema.User, searchterm return searchterm, "", "", "" } else { // Has to have letters and logged-in user for other guesses if user != nil { - // Find username in jobs (match) - uresult, _ := r.FindColumnValue(user, searchterm, "job", "user", "user", false) + // Find username by username in job table (match) + uresult, _ := r.FindColumnValue(user, searchterm, "job", "hpc_user", "hpc_user", false) if uresult != "" { return "", uresult, "", "" } - // Find username by name (like) - nresult, _ := r.FindColumnValue(user, searchterm, "user", "username", "name", true) + // Find username by real name in hpc_user table (like) + nresult, _ := r.FindColumnValue(user, searchterm, "hpc_user", "username", "name", true) if nresult != "" { return "", nresult, "", "" } - // Find projectId in jobs (match) + // Find projectId by projectId in job table (match) presult, _ := r.FindColumnValue(user, searchterm, "job", "project", "project", false) if presult != "" { return "", "", presult, "" @@ -400,7 +397,7 @@ func (r *JobRepository) Partitions(cluster string) ([]string, error) { start := time.Now() partitions := r.cache.Get("partitions:"+cluster, func() (interface{}, time.Duration, int) { parts := []string{} - if err = r.DB.Select(&parts, `SELECT DISTINCT job.partition FROM job WHERE job.cluster = ?;`, cluster); err != nil { + if err = r.DB.Select(&parts, `SELECT DISTINCT job.cluster_partition FROM job WHERE job.cluster = ?;`, cluster); err != nil { return nil, 0, 1000 } @@ -457,6 +454,7 @@ func (r *JobRepository) AllocatedNodes(cluster string) (map[string]map[string]in return subclusters, nil } +// FIXME: Set duration to requested walltime? func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error { start := time.Now() res, err := sq.Update("job"). @@ -604,13 +602,12 @@ func (r *JobRepository) UpdateEnergy( for _, fp := range sc.EnergyFootprint { if i, err := archive.MetricIndex(sc.MetricConfig, fp); err == nil { // Note: For DB data, calculate and save as kWh - // Energy: Power (in Watts) * Time (in Seconds) - if sc.MetricConfig[i].Energy == "energy" { + if sc.MetricConfig[i].Energy == "energy" { // this metric has energy as unit (Joules or Wh) + // FIXME: Needs sum as stats type + } else if sc.MetricConfig[i].Energy == "power" { // this metric has power as unit (Watt) + // Energy: Power (in Watts) * Time (in Seconds) // Unit: ( W * s ) / 3600 / 1000 = kWh ; Rounded to 2 nearest digits energy = math.Round(((LoadJobStat(jobMeta, fp, "avg")*float64(jobMeta.Duration))/3600/1000)*100) / 100 - // Power: Use directly as sum (Or as: [Energy (in Ws) / Time (in s)] - } else if sc.MetricConfig[i].Energy == "power" { - // This assumes the metric is of aggregation type sum } } else { log.Warnf("Error while collecting energy metric %s for job, DB ID '%v', return '0.0'", fp, jobMeta.ID) diff --git a/internal/repository/jobCreate.go b/internal/repository/jobCreate.go index 43c26c1..9e47974 100644 --- a/internal/repository/jobCreate.go +++ b/internal/repository/jobCreate.go @@ -14,11 +14,11 @@ import ( ) const NamedJobInsert string = `INSERT INTO job ( - job_id, user, project, cluster, subcluster, ` + "`partition`" + `, array_job_id, num_nodes, num_hwthreads, num_acc, - exclusive, monitoring_status, smt, job_state, start_time, duration, walltime, footprint, resources, meta_data + job_id, hpc_user, project, cluster, subcluster, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, + exclusive, monitoring_status, smt, job_state, start_time, duration, walltime, footprint, energy, energy_footprint, resources, meta_data ) VALUES ( - :job_id, :user, :project, :cluster, :subcluster, :partition, :array_job_id, :num_nodes, :num_hwthreads, :num_acc, - :exclusive, :monitoring_status, :smt, :job_state, :start_time, :duration, :walltime, :footprint, :resources, :meta_data + :job_id, :hpc_user, :project, :cluster, :subcluster, :cluster_partition, :array_job_id, :num_nodes, :num_hwthreads, :num_acc, + :exclusive, :monitoring_status, :smt, :job_state, :start_time, :duration, :walltime, :footprint, :energy, :energy_footprint, :resources, :meta_data );` func (r *JobRepository) InsertJob(job *schema.JobMeta) (int64, error) { diff --git a/internal/repository/jobFind.go b/internal/repository/jobFind.go index 842d5f4..0354df0 100644 --- a/internal/repository/jobFind.go +++ b/internal/repository/jobFind.go @@ -37,8 +37,9 @@ func (r *JobRepository) Find( q = q.Where("job.start_time = ?", *startTime) } - log.Debugf("Timer Find %s", time.Since(start)) + q = q.OrderBy("job.id DESC") // always use newest matching job by db id if more than one match + log.Debugf("Timer Find %s", time.Since(start)) return scanJob(q.RunWith(r.stmtCache).QueryRow()) } @@ -98,6 +99,23 @@ func (r *JobRepository) FindById(ctx context.Context, jobId int64) (*schema.Job, return scanJob(q.RunWith(r.stmtCache).QueryRow()) } +// FindByIdWithUser executes a SQL query to find a specific batch job. +// The job is queried using the database id. The user is passed directly, +// instead as part of the context. +// It returns a pointer to a schema.Job data structure and an error variable. +// To check if no job was found test err == sql.ErrNoRows +func (r *JobRepository) FindByIdWithUser(user *schema.User, jobId int64) (*schema.Job, error) { + q := sq.Select(jobColumns...). + From("job").Where("job.id = ?", jobId) + + q, qerr := SecurityCheckWithUser(user, q) + if qerr != nil { + return nil, qerr + } + + return scanJob(q.RunWith(r.stmtCache).QueryRow()) +} + // FindByIdDirect executes a SQL query to find a specific batch job. // The job is queried using the database id. // It returns a pointer to a schema.Job data structure and an error variable. @@ -135,7 +153,7 @@ func (r *JobRepository) IsJobOwner(jobId int64, startTime int64, user string, cl q := sq.Select("id"). From("job"). Where("job.job_id = ?", jobId). - Where("job.user = ?", user). + Where("job.hpc_user = ?", user). Where("job.cluster = ?", cluster). Where("job.start_time = ?", startTime) diff --git a/internal/repository/jobQuery.go b/internal/repository/jobQuery.go index 5458043..b43b569 100644 --- a/internal/repository/jobQuery.go +++ b/internal/repository/jobQuery.go @@ -107,8 +107,7 @@ func (r *JobRepository) CountJobs( return count, nil } -func SecurityCheck(ctx context.Context, query sq.SelectBuilder) (sq.SelectBuilder, error) { - user := GetUserFromContext(ctx) +func SecurityCheckWithUser(user *schema.User, query sq.SelectBuilder) (sq.SelectBuilder, error) { if user == nil { var qnil sq.SelectBuilder return qnil, fmt.Errorf("user context is nil") @@ -121,19 +120,25 @@ func SecurityCheck(ctx context.Context, query sq.SelectBuilder) (sq.SelectBuilde return query, nil case user.HasRole(schema.RoleManager): // Manager : Add filter for managed projects' jobs only + personal jobs if len(user.Projects) != 0 { - return query.Where(sq.Or{sq.Eq{"job.project": user.Projects}, sq.Eq{"job.user": user.Username}}), nil + return query.Where(sq.Or{sq.Eq{"job.project": user.Projects}, sq.Eq{"job.hpc_user": user.Username}}), nil } else { log.Debugf("Manager-User '%s' has no defined projects to lookup! Query only personal jobs ...", user.Username) - return query.Where("job.user = ?", user.Username), nil + return query.Where("job.hpc_user = ?", user.Username), nil } case user.HasRole(schema.RoleUser): // User : Only personal jobs - return query.Where("job.user = ?", user.Username), nil + return query.Where("job.hpc_user = ?", user.Username), nil default: // No known Role, return error var qnil sq.SelectBuilder return qnil, fmt.Errorf("user has no or unknown roles") } } +func SecurityCheck(ctx context.Context, query sq.SelectBuilder) (sq.SelectBuilder, error) { + user := GetUserFromContext(ctx) + + return SecurityCheckWithUser(user, query) +} + // Build a sq.SelectBuilder out of a schema.JobFilter. func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.SelectBuilder { if filter.Tags != nil { @@ -147,7 +152,7 @@ func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.Select query = query.Where("job.array_job_id = ?", *filter.ArrayJobID) } if filter.User != nil { - query = buildStringCondition("job.user", filter.User, query) + query = buildStringCondition("job.hpc_user", filter.User, query) } if filter.Project != nil { query = buildStringCondition("job.project", filter.Project, query) @@ -159,14 +164,13 @@ func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.Select query = buildStringCondition("job.cluster", filter.Cluster, query) } if filter.Partition != nil { - query = buildStringCondition("job.partition", filter.Partition, query) + query = buildStringCondition("job.cluster_partition", filter.Partition, query) } if filter.StartTime != nil { query = buildTimeCondition("job.start_time", filter.StartTime, query) } if filter.Duration != nil { - now := time.Now().Unix() // There does not seam to be a portable way to get the current unix timestamp accross different DBs. - query = query.Where("(CASE WHEN job.job_state = 'running' THEN (? - job.start_time) ELSE job.duration END) BETWEEN ? AND ?", now, filter.Duration.From, filter.Duration.To) + query = buildIntCondition("job.duration", filter.Duration, query) } if filter.MinRunningFor != nil { now := time.Now().Unix() // There does not seam to be a portable way to get the current unix timestamp accross different DBs. diff --git a/internal/repository/job_test.go b/internal/repository/job_test.go index f7b3783..363bb6c 100644 --- a/internal/repository/job_test.go +++ b/internal/repository/job_test.go @@ -59,7 +59,7 @@ func TestGetTags(t *testing.T) { ctx := context.WithValue(getContext(t), contextUserKey, contextUserValue) // Test Tag has Scope "global" - tags, counts, err := r.CountTags(ctx) + tags, counts, err := r.CountTags(GetUserFromContext(ctx)) if err != nil { t.Fatal(err) } diff --git a/internal/repository/migration.go b/internal/repository/migration.go index 970fbc2..d32a624 100644 --- a/internal/repository/migration.go +++ b/internal/repository/migration.go @@ -114,6 +114,14 @@ func MigrateDB(backend string, db string) error { return err } + v, dirty, err := m.Version() + + log.Infof("unsupported database version %d, need %d.\nPlease backup your database file and run cc-backend -migrate-db", v, Version) + + if dirty { + return fmt.Errorf("last migration to version %d has failed, please fix the db manually and force version with -force-db flag", Version) + } + if err := m.Up(); err != nil { if err == migrate.ErrNoChange { log.Info("DB already up to date!") diff --git a/internal/repository/migrations/mysql/08_add-footprint.down.sql b/internal/repository/migrations/mysql/08_add-footprint.down.sql new file mode 100644 index 0000000..57f2145 --- /dev/null +++ b/internal/repository/migrations/mysql/08_add-footprint.down.sql @@ -0,0 +1,83 @@ +ALTER TABLE job DROP energy; +ALTER TABLE job DROP energy_footprint; +ALTER TABLE job ADD COLUMN flops_any_avg; +ALTER TABLE job ADD COLUMN mem_bw_avg; +ALTER TABLE job ADD COLUMN mem_used_max; +ALTER TABLE job ADD COLUMN load_avg; +ALTER TABLE job ADD COLUMN net_bw_avg; +ALTER TABLE job ADD COLUMN net_data_vol_total; +ALTER TABLE job ADD COLUMN file_bw_avg; +ALTER TABLE job ADD COLUMN file_data_vol_total; + +UPDATE job SET flops_any_avg = json_extract(footprint, '$.flops_any_avg'); +UPDATE job SET mem_bw_avg = json_extract(footprint, '$.mem_bw_avg'); +UPDATE job SET mem_used_max = json_extract(footprint, '$.mem_used_max'); +UPDATE job SET load_avg = json_extract(footprint, '$.cpu_load_avg'); +UPDATE job SET net_bw_avg = json_extract(footprint, '$.net_bw_avg'); +UPDATE job SET net_data_vol_total = json_extract(footprint, '$.net_data_vol_total'); +UPDATE job SET file_bw_avg = json_extract(footprint, '$.file_bw_avg'); +UPDATE job SET file_data_vol_total = json_extract(footprint, '$.file_data_vol_total'); + +ALTER TABLE job DROP footprint; +-- Do not use reserved keywords anymore +RENAME TABLE hpc_user TO `user`; +ALTER TABLE job RENAME COLUMN hpc_user TO `user`; +ALTER TABLE job RENAME COLUMN cluster_partition TO `partition`; + +DROP INDEX IF EXISTS jobs_cluster; +DROP INDEX IF EXISTS jobs_cluster_user; +DROP INDEX IF EXISTS jobs_cluster_project; +DROP INDEX IF EXISTS jobs_cluster_subcluster; +DROP INDEX IF EXISTS jobs_cluster_starttime; +DROP INDEX IF EXISTS jobs_cluster_duration; +DROP INDEX IF EXISTS jobs_cluster_numnodes; + +DROP INDEX IF EXISTS jobs_cluster_partition; +DROP INDEX IF EXISTS jobs_cluster_partition_starttime; +DROP INDEX IF EXISTS jobs_cluster_partition_duration; +DROP INDEX IF EXISTS jobs_cluster_partition_numnodes; + +DROP INDEX IF EXISTS jobs_cluster_partition_jobstate; +DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_user; +DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_project; +DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_starttime; +DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_duration; +DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_numnodes; + +DROP INDEX IF EXISTS jobs_cluster_jobstate; +DROP INDEX IF EXISTS jobs_cluster_jobstate_user; +DROP INDEX IF EXISTS jobs_cluster_jobstate_project; + +DROP INDEX IF EXISTS jobs_cluster_jobstate_starttime; +DROP INDEX IF EXISTS jobs_cluster_jobstate_duration; +DROP INDEX IF EXISTS jobs_cluster_jobstate_numnodes; + +DROP INDEX IF EXISTS jobs_user; +DROP INDEX IF EXISTS jobs_user_starttime; +DROP INDEX IF EXISTS jobs_user_duration; +DROP INDEX IF EXISTS jobs_user_numnodes; + +DROP INDEX IF EXISTS jobs_project; +DROP INDEX IF EXISTS jobs_project_user; +DROP INDEX IF EXISTS jobs_project_starttime; +DROP INDEX IF EXISTS jobs_project_duration; +DROP INDEX IF EXISTS jobs_project_numnodes; + +DROP INDEX IF EXISTS jobs_jobstate; +DROP INDEX IF EXISTS jobs_jobstate_user; +DROP INDEX IF EXISTS jobs_jobstate_project; +DROP INDEX IF EXISTS jobs_jobstate_starttime; +DROP INDEX IF EXISTS jobs_jobstate_duration; +DROP INDEX IF EXISTS jobs_jobstate_numnodes; + +DROP INDEX IF EXISTS jobs_arrayjobid_starttime; +DROP INDEX IF EXISTS jobs_cluster_arrayjobid_starttime; + +DROP INDEX IF EXISTS jobs_starttime; +DROP INDEX IF EXISTS jobs_duration; +DROP INDEX IF EXISTS jobs_numnodes; + +DROP INDEX IF EXISTS jobs_duration_starttime; +DROP INDEX IF EXISTS jobs_numnodes_starttime; +DROP INDEX IF EXISTS jobs_numacc_starttime; +DROP INDEX IF EXISTS jobs_energy_starttime; diff --git a/internal/repository/migrations/mysql/08_add-footprint.up.sql b/internal/repository/migrations/mysql/08_add-footprint.up.sql new file mode 100644 index 0000000..207ccf9 --- /dev/null +++ b/internal/repository/migrations/mysql/08_add-footprint.up.sql @@ -0,0 +1,123 @@ +DROP INDEX IF EXISTS job_stats ON job; +DROP INDEX IF EXISTS job_by_user ON job; +DROP INDEX IF EXISTS job_by_starttime ON job; +DROP INDEX IF EXISTS job_by_job_id ON job; +DROP INDEX IF EXISTS job_list ON job; +DROP INDEX IF EXISTS job_list_user ON job; +DROP INDEX IF EXISTS job_list_users ON job; +DROP INDEX IF EXISTS job_list_users_start ON job; + +ALTER TABLE job ADD COLUMN energy REAL NOT NULL DEFAULT 0.0; +ALTER TABLE job ADD COLUMN energy_footprint JSON; + +ALTER TABLE job ADD COLUMN footprint JSON; +ALTER TABLE tag ADD COLUMN tag_scope TEXT NOT NULL DEFAULT 'global'; + +-- Do not use reserved keywords anymore +RENAME TABLE `user` TO hpc_user; +ALTER TABLE job RENAME COLUMN `user` TO hpc_user; +ALTER TABLE job RENAME COLUMN `partition` TO cluster_partition; + +ALTER TABLE job MODIFY COLUMN cluster VARCHAR(50); +ALTER TABLE job MODIFY COLUMN hpc_user VARCHAR(50); +ALTER TABLE job MODIFY COLUMN subcluster VARCHAR(50); +ALTER TABLE job MODIFY COLUMN project VARCHAR(50); +ALTER TABLE job MODIFY COLUMN cluster_partition VARCHAR(50); +ALTER TABLE job MODIFY COLUMN job_state VARCHAR(25); + +UPDATE job SET footprint = '{"flops_any_avg": 0.0}'; +UPDATE job SET footprint = json_replace(footprint, '$.flops_any_avg', job.flops_any_avg); +UPDATE job SET footprint = json_insert(footprint, '$.mem_bw_avg', job.mem_bw_avg); +UPDATE job SET footprint = json_insert(footprint, '$.mem_used_max', job.mem_used_max); +UPDATE job SET footprint = json_insert(footprint, '$.cpu_load_avg', job.load_avg); +UPDATE job SET footprint = json_insert(footprint, '$.net_bw_avg', job.net_bw_avg) WHERE job.net_bw_avg != 0; +UPDATE job SET footprint = json_insert(footprint, '$.net_data_vol_total', job.net_data_vol_total) WHERE job.net_data_vol_total != 0; +UPDATE job SET footprint = json_insert(footprint, '$.file_bw_avg', job.file_bw_avg) WHERE job.file_bw_avg != 0; +UPDATE job SET footprint = json_insert(footprint, '$.file_data_vol_total', job.file_data_vol_total) WHERE job.file_data_vol_total != 0; + +ALTER TABLE job DROP flops_any_avg; +ALTER TABLE job DROP mem_bw_avg; +ALTER TABLE job DROP mem_used_max; +ALTER TABLE job DROP load_avg; +ALTER TABLE job DROP net_bw_avg; +ALTER TABLE job DROP net_data_vol_total; +ALTER TABLE job DROP file_bw_avg; +ALTER TABLE job DROP file_data_vol_total; + +-- Indices for: Single filters, combined filters, sorting, sorting with filters +-- Cluster Filter +CREATE INDEX IF NOT EXISTS jobs_cluster ON job (cluster); +CREATE INDEX IF NOT EXISTS jobs_cluster_user ON job (cluster, hpc_user); +CREATE INDEX IF NOT EXISTS jobs_cluster_project ON job (cluster, project); +CREATE INDEX IF NOT EXISTS jobs_cluster_subcluster ON job (cluster, subcluster); +-- Cluster Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_cluster_starttime ON job (cluster, start_time); +CREATE INDEX IF NOT EXISTS jobs_cluster_duration ON job (cluster, duration); +CREATE INDEX IF NOT EXISTS jobs_cluster_numnodes ON job (cluster, num_nodes); + +-- Cluster+Partition Filter +CREATE INDEX IF NOT EXISTS jobs_cluster_partition ON job (cluster, cluster_partition); +-- Cluster+Partition Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_starttime ON job (cluster, cluster_partition, start_time); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_duration ON job (cluster, cluster_partition, duration); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numnodes ON job (cluster, cluster_partition, num_nodes); + +-- Cluster+Partition+Jobstate Filter +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate ON job (cluster, cluster_partition, job_state); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_user ON job (cluster, cluster_partition, job_state, hpc_user); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_project ON job (cluster, cluster_partition, job_state, project); +-- Cluster+Partition+Jobstate Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_starttime ON job (cluster, cluster_partition, job_state, start_time); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_duration ON job (cluster, cluster_partition, job_state, duration); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_numnodes ON job (cluster, cluster_partition, job_state, num_nodes); + +-- Cluster+JobState Filter +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate ON job (cluster, job_state); +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_user ON job (cluster, job_state, hpc_user); +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_project ON job (cluster, job_state, project); +-- Cluster+JobState Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_starttime ON job (cluster, job_state, start_time); +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_duration ON job (cluster, job_state, duration); +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numnodes ON job (cluster, job_state, num_nodes); + +-- User Filter +CREATE INDEX IF NOT EXISTS jobs_user ON job (hpc_user); +-- User Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_user_starttime ON job (hpc_user, start_time); +CREATE INDEX IF NOT EXISTS jobs_user_duration ON job (hpc_user, duration); +CREATE INDEX IF NOT EXISTS jobs_user_numnodes ON job (hpc_user, num_nodes); + +-- Project Filter +CREATE INDEX IF NOT EXISTS jobs_project ON job (project); +CREATE INDEX IF NOT EXISTS jobs_project_user ON job (project, hpc_user); +-- Project Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_project_starttime ON job (project, start_time); +CREATE INDEX IF NOT EXISTS jobs_project_duration ON job (project, duration); +CREATE INDEX IF NOT EXISTS jobs_project_numnodes ON job (project, num_nodes); + +-- JobState Filter +CREATE INDEX IF NOT EXISTS jobs_jobstate ON job (job_state); +CREATE INDEX IF NOT EXISTS jobs_jobstate_user ON job (job_state, hpc_user); +CREATE INDEX IF NOT EXISTS jobs_jobstate_project ON job (job_state, project); +CREATE INDEX IF NOT EXISTS jobs_jobstate_cluster ON job (job_state, cluster); +-- JobState Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_jobstate_starttime ON job (job_state, start_time); +CREATE INDEX IF NOT EXISTS jobs_jobstate_duration ON job (job_state, duration); +CREATE INDEX IF NOT EXISTS jobs_jobstate_numnodes ON job (job_state, num_nodes); + +-- ArrayJob Filter +CREATE INDEX IF NOT EXISTS jobs_arrayjobid_starttime ON job (array_job_id, start_time); +CREATE INDEX IF NOT EXISTS jobs_cluster_arrayjobid_starttime ON job (cluster, array_job_id, start_time); + +-- Sorting without active filters +CREATE INDEX IF NOT EXISTS jobs_starttime ON job (start_time); +CREATE INDEX IF NOT EXISTS jobs_duration ON job (duration); +CREATE INDEX IF NOT EXISTS jobs_numnodes ON job (num_nodes); + +-- Single filters with default starttime sorting +CREATE INDEX IF NOT EXISTS jobs_duration_starttime ON job (duration, start_time); +CREATE INDEX IF NOT EXISTS jobs_numnodes_starttime ON job (num_nodes, start_time); +CREATE INDEX IF NOT EXISTS jobs_numacc_starttime ON job (num_acc, start_time); +CREATE INDEX IF NOT EXISTS jobs_energy_starttime ON job (energy, start_time); + +-- Optimize DB index usage diff --git a/internal/repository/migrations/sqlite3/08_add-footprint.down.sql b/internal/repository/migrations/sqlite3/08_add-footprint.down.sql index 8c99eb5..cc2d3e9 100644 --- a/internal/repository/migrations/sqlite3/08_add-footprint.down.sql +++ b/internal/repository/migrations/sqlite3/08_add-footprint.down.sql @@ -19,3 +19,85 @@ UPDATE job SET file_bw_avg = json_extract(footprint, '$.file_bw_avg'); UPDATE job SET file_data_vol_total = json_extract(footprint, '$.file_data_vol_total'); ALTER TABLE job DROP footprint; + +DROP INDEX IF EXISTS jobs_cluster; +DROP INDEX IF EXISTS jobs_cluster_user; +DROP INDEX IF EXISTS jobs_cluster_project; +DROP INDEX IF EXISTS jobs_cluster_subcluster; +DROP INDEX IF EXISTS jobs_cluster_starttime; +DROP INDEX IF EXISTS jobs_cluster_duration; +DROP INDEX IF EXISTS jobs_cluster_numnodes; +DROP INDEX IF EXISTS jobs_cluster_numhwthreads; +DROP INDEX IF EXISTS jobs_cluster_numacc; +DROP INDEX IF EXISTS jobs_cluster_energy; + +DROP INDEX IF EXISTS jobs_cluster_partition; +DROP INDEX IF EXISTS jobs_cluster_partition_starttime; +DROP INDEX IF EXISTS jobs_cluster_partition_duration; +DROP INDEX IF EXISTS jobs_cluster_partition_numnodes; +DROP INDEX IF EXISTS jobs_cluster_partition_numhwthreads; +DROP INDEX IF EXISTS jobs_cluster_partition_numacc; +DROP INDEX IF EXISTS jobs_cluster_partition_energy; + +DROP INDEX IF EXISTS jobs_cluster_partition_jobstate; +DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_user; +DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_project; +DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_starttime; +DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_duration; +DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_numnodes; +DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_numhwthreads; +DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_numacc; +DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_energy; + +DROP INDEX IF EXISTS jobs_cluster_jobstate; +DROP INDEX IF EXISTS jobs_cluster_jobstate_user; +DROP INDEX IF EXISTS jobs_cluster_jobstate_project; + +DROP INDEX IF EXISTS jobs_cluster_jobstate_starttime; +DROP INDEX IF EXISTS jobs_cluster_jobstate_duration; +DROP INDEX IF EXISTS jobs_cluster_jobstate_numnodes; +DROP INDEX IF EXISTS jobs_cluster_jobstate_numhwthreads; +DROP INDEX IF EXISTS jobs_cluster_jobstate_numacc; +DROP INDEX IF EXISTS jobs_cluster_jobstate_energy; + +DROP INDEX IF EXISTS jobs_user; +DROP INDEX IF EXISTS jobs_user_starttime; +DROP INDEX IF EXISTS jobs_user_duration; +DROP INDEX IF EXISTS jobs_user_numnodes; +DROP INDEX IF EXISTS jobs_user_numhwthreads; +DROP INDEX IF EXISTS jobs_user_numacc; +DROP INDEX IF EXISTS jobs_user_energy; + +DROP INDEX IF EXISTS jobs_project; +DROP INDEX IF EXISTS jobs_project_user; +DROP INDEX IF EXISTS jobs_project_starttime; +DROP INDEX IF EXISTS jobs_project_duration; +DROP INDEX IF EXISTS jobs_project_numnodes; +DROP INDEX IF EXISTS jobs_project_numhwthreads; +DROP INDEX IF EXISTS jobs_project_numacc; +DROP INDEX IF EXISTS jobs_project_energy; + +DROP INDEX IF EXISTS jobs_jobstate; +DROP INDEX IF EXISTS jobs_jobstate_user; +DROP INDEX IF EXISTS jobs_jobstate_project; +DROP INDEX IF EXISTS jobs_jobstate_starttime; +DROP INDEX IF EXISTS jobs_jobstate_duration; +DROP INDEX IF EXISTS jobs_jobstate_numnodes; +DROP INDEX IF EXISTS jobs_jobstate_numhwthreads; +DROP INDEX IF EXISTS jobs_jobstate_numacc; + +DROP INDEX IF EXISTS jobs_arrayjobid_starttime; +DROP INDEX IF EXISTS jobs_cluster_arrayjobid_starttime; + +DROP INDEX IF EXISTS jobs_starttime; +DROP INDEX IF EXISTS jobs_duration; +DROP INDEX IF EXISTS jobs_numnodes; +DROP INDEX IF EXISTS jobs_numhwthreads; +DROP INDEX IF EXISTS jobs_numacc; +DROP INDEX IF EXISTS jobs_energy; + +DROP INDEX IF EXISTS jobs_duration_starttime; +DROP INDEX IF EXISTS jobs_numnodes_starttime; +DROP INDEX IF EXISTS jobs_numhwthreads_starttime; +DROP INDEX IF EXISTS jobs_numacc_starttime; +DROP INDEX IF EXISTS jobs_energy_starttime; diff --git a/internal/repository/migrations/sqlite3/08_add-footprint.up.sql b/internal/repository/migrations/sqlite3/08_add-footprint.up.sql index 9c9e53e..5c28da9 100644 --- a/internal/repository/migrations/sqlite3/08_add-footprint.up.sql +++ b/internal/repository/migrations/sqlite3/08_add-footprint.up.sql @@ -1,11 +1,11 @@ -DROP INDEX job_stats; -DROP INDEX job_by_user; -DROP INDEX job_by_starttime; -DROP INDEX job_by_job_id; -DROP INDEX job_list; -DROP INDEX job_list_user; -DROP INDEX job_list_users; -DROP INDEX job_list_users_start; +DROP INDEX IF EXISTS job_stats; +DROP INDEX IF EXISTS job_by_user; +DROP INDEX IF EXISTS job_by_starttime; +DROP INDEX IF EXISTS job_by_job_id; +DROP INDEX IF EXISTS job_list; +DROP INDEX IF EXISTS job_list_user; +DROP INDEX IF EXISTS job_list_users; +DROP INDEX IF EXISTS job_list_users_start; ALTER TABLE job ADD COLUMN energy REAL NOT NULL DEFAULT 0.0; ALTER TABLE job ADD COLUMN energy_footprint TEXT DEFAULT NULL; @@ -13,6 +13,11 @@ ALTER TABLE job ADD COLUMN energy_footprint TEXT DEFAULT NULL; ALTER TABLE job ADD COLUMN footprint TEXT DEFAULT NULL; ALTER TABLE tag ADD COLUMN tag_scope TEXT NOT NULL DEFAULT 'global'; +-- Do not use reserved keywords anymore +ALTER TABLE "user" RENAME TO hpc_user; +ALTER TABLE job RENAME COLUMN "user" TO hpc_user; +ALTER TABLE job RENAME COLUMN "partition" TO cluster_partition; + UPDATE job SET footprint = '{"flops_any_avg": 0.0}'; UPDATE job SET footprint = json_replace(footprint, '$.flops_any_avg', job.flops_any_avg); UPDATE job SET footprint = json_insert(footprint, '$.mem_bw_avg', job.mem_bw_avg); @@ -35,7 +40,7 @@ ALTER TABLE job DROP file_data_vol_total; -- Indices for: Single filters, combined filters, sorting, sorting with filters -- Cluster Filter CREATE INDEX IF NOT EXISTS jobs_cluster ON job (cluster); -CREATE INDEX IF NOT EXISTS jobs_cluster_user ON job (cluster, user); +CREATE INDEX IF NOT EXISTS jobs_cluster_user ON job (cluster, hpc_user); CREATE INDEX IF NOT EXISTS jobs_cluster_project ON job (cluster, project); CREATE INDEX IF NOT EXISTS jobs_cluster_subcluster ON job (cluster, subcluster); -- Cluster Filter Sorting @@ -47,30 +52,30 @@ CREATE INDEX IF NOT EXISTS jobs_cluster_numacc ON job (cluster, num_acc); CREATE INDEX IF NOT EXISTS jobs_cluster_energy ON job (cluster, energy); -- Cluster+Partition Filter -CREATE INDEX IF NOT EXISTS jobs_cluster_partition ON job (cluster, partition); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition ON job (cluster, cluster_partition); -- Cluster+Partition Filter Sorting -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_starttime ON job (cluster, partition, start_time); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_duration ON job (cluster, partition, duration); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numnodes ON job (cluster, partition, num_nodes); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numhwthreads ON job (cluster, partition, num_hwthreads); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numacc ON job (cluster, partition, num_acc); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_energy ON job (cluster, partition, energy); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_starttime ON job (cluster, cluster_partition, start_time); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_duration ON job (cluster, cluster_partition, duration); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numnodes ON job (cluster, cluster_partition, num_nodes); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numhwthreads ON job (cluster, cluster_partition, num_hwthreads); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numacc ON job (cluster, cluster_partition, num_acc); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_energy ON job (cluster, cluster_partition, energy); -- Cluster+Partition+Jobstate Filter -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate ON job (cluster, partition, job_state); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_user ON job (cluster, partition, job_state, user); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_project ON job (cluster, partition, job_state, project); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate ON job (cluster, cluster_partition, job_state); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_user ON job (cluster, cluster_partition, job_state, hpc_user); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_project ON job (cluster, cluster_partition, job_state, project); -- Cluster+Partition+Jobstate Filter Sorting -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_starttime ON job (cluster, partition, job_state, start_time); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_duration ON job (cluster, partition, job_state, duration); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_numnodes ON job (cluster, partition, job_state, num_nodes); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_numhwthreads ON job (cluster, partition, job_state, num_hwthreads); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_numacc ON job (cluster, partition, job_state, num_acc); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_energy ON job (cluster, partition, job_state, energy); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_starttime ON job (cluster, cluster_partition, job_state, start_time); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_duration ON job (cluster, cluster_partition, job_state, duration); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_numnodes ON job (cluster, cluster_partition, job_state, num_nodes); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_numhwthreads ON job (cluster, cluster_partition, job_state, num_hwthreads); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_numacc ON job (cluster, cluster_partition, job_state, num_acc); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_energy ON job (cluster, cluster_partition, job_state, energy); -- Cluster+JobState Filter CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate ON job (cluster, job_state); -CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_user ON job (cluster, job_state, user); +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_user ON job (cluster, job_state, hpc_user); CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_project ON job (cluster, job_state, project); -- Cluster+JobState Filter Sorting CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_starttime ON job (cluster, job_state, start_time); @@ -81,18 +86,18 @@ CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numacc ON job (cluster, job_sta CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_energy ON job (cluster, job_state, energy); -- User Filter -CREATE INDEX IF NOT EXISTS jobs_user ON job (user); +CREATE INDEX IF NOT EXISTS jobs_user ON job (hpc_user); -- User Filter Sorting -CREATE INDEX IF NOT EXISTS jobs_user_starttime ON job (user, start_time); -CREATE INDEX IF NOT EXISTS jobs_user_duration ON job (user, duration); -CREATE INDEX IF NOT EXISTS jobs_user_numnodes ON job (user, num_nodes); -CREATE INDEX IF NOT EXISTS jobs_user_numhwthreads ON job (user, num_hwthreads); -CREATE INDEX IF NOT EXISTS jobs_user_numacc ON job (user, num_acc); -CREATE INDEX IF NOT EXISTS jobs_user_energy ON job (user, energy); +CREATE INDEX IF NOT EXISTS jobs_user_starttime ON job (hpc_user, start_time); +CREATE INDEX IF NOT EXISTS jobs_user_duration ON job (hpc_user, duration); +CREATE INDEX IF NOT EXISTS jobs_user_numnodes ON job (hpc_user, num_nodes); +CREATE INDEX IF NOT EXISTS jobs_user_numhwthreads ON job (hpc_user, num_hwthreads); +CREATE INDEX IF NOT EXISTS jobs_user_numacc ON job (hpc_user, num_acc); +CREATE INDEX IF NOT EXISTS jobs_user_energy ON job (hpc_user, energy); -- Project Filter CREATE INDEX IF NOT EXISTS jobs_project ON job (project); -CREATE INDEX IF NOT EXISTS jobs_project_user ON job (project, user); +CREATE INDEX IF NOT EXISTS jobs_project_user ON job (project, hpc_user); -- Project Filter Sorting CREATE INDEX IF NOT EXISTS jobs_project_starttime ON job (project, start_time); CREATE INDEX IF NOT EXISTS jobs_project_duration ON job (project, duration); @@ -103,7 +108,7 @@ CREATE INDEX IF NOT EXISTS jobs_project_energy ON job (project, energy); -- JobState Filter CREATE INDEX IF NOT EXISTS jobs_jobstate ON job (job_state); -CREATE INDEX IF NOT EXISTS jobs_jobstate_user ON job (job_state, user); +CREATE INDEX IF NOT EXISTS jobs_jobstate_user ON job (job_state, hpc_user); CREATE INDEX IF NOT EXISTS jobs_jobstate_project ON job (job_state, project); CREATE INDEX IF NOT EXISTS jobs_jobstate_cluster ON job (job_state, cluster); -- JobState Filter Sorting diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index 6d1fbfc..1ca9ec5 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -111,7 +111,7 @@ func BenchmarkDB_QueryJobs(b *testing.B) { user := "mppi133h" filter.User = &model.StringInput{Eq: &user} page := &model.PageRequest{ItemsPerPage: 50, Page: 1} - order := &model.OrderByInput{Field: "startTime", Order: model.SortDirectionEnumDesc} + order := &model.OrderByInput{Field: "startTime", Type: "col", Order: model.SortDirectionEnumDesc} b.Run("QueryJobs", func(b *testing.B) { db := setup(b) diff --git a/internal/repository/stats.go b/internal/repository/stats.go index ba7a8aa..484851d 100644 --- a/internal/repository/stats.go +++ b/internal/repository/stats.go @@ -22,7 +22,7 @@ import ( // GraphQL validation should make sure that no unkown values can be specified. var groupBy2column = map[model.Aggregate]string{ - model.AggregateUser: "job.user", + model.AggregateUser: "job.hpc_user", model.AggregateProject: "job.project", model.AggregateCluster: "job.cluster", } @@ -86,7 +86,7 @@ func (r *JobRepository) buildStatsQuery( fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_hwthreads) / 3600) as %s) as totalCoreHours`, time.Now().Unix(), castType), fmt.Sprintf(`CAST(SUM(job.num_acc) as %s) as totalAccs`, castType), fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_acc) / 3600) as %s) as totalAccHours`, time.Now().Unix(), castType), - ).From("job").Join("user ON user.username = job.user").GroupBy(col) + ).From("job").LeftJoin("hpc_user ON hpc_user.username = job.hpc_user").GroupBy(col) } else { // Scan columns: totalJobs, name, totalWalltime, totalNodes, totalNodeHours, totalCores, totalCoreHours, totalAccs, totalAccHours query = sq.Select("COUNT(job.id)", @@ -109,7 +109,7 @@ func (r *JobRepository) buildStatsQuery( // func (r *JobRepository) getUserName(ctx context.Context, id string) string { // user := GetUserFromContext(ctx) -// name, _ := r.FindColumnValue(user, id, "user", "name", "username", false) +// name, _ := r.FindColumnValue(user, id, "hpc_user", "name", "username", false) // if name != "" { // return name // } else { @@ -210,7 +210,7 @@ func (r *JobRepository) JobsStatsGrouped( totalAccHours = int(accHours.Int64) } - if col == "job.user" { + if col == "job.hpc_user" { // name := r.getUserName(ctx, id.String) stats = append(stats, &model.JobsStatistics{ @@ -560,9 +560,9 @@ func (r *JobRepository) jobsMetricStatisticsHistogram( ) (*model.MetricHistoPoints, error) { // Get specific Peak or largest Peak var metricConfig *schema.MetricConfig - var peak float64 = 0.0 - var unit string = "" - var footprintStat string = "" + var peak float64 + var unit string + var footprintStat string for _, f := range filters { if f.Cluster != nil { @@ -712,8 +712,8 @@ func (r *JobRepository) runningJobsMetricStatisticsHistogram( for idx, metric := range metrics { // Get specific Peak or largest Peak var metricConfig *schema.MetricConfig - var peak float64 = 0.0 - var unit string = "" + var peak float64 + var unit string for _, f := range filters { if f.Cluster != nil { diff --git a/internal/repository/tags.go b/internal/repository/tags.go index 48ea9ec..8120364 100644 --- a/internal/repository/tags.go +++ b/internal/repository/tags.go @@ -5,7 +5,6 @@ package repository import ( - "context" "fmt" "strings" @@ -16,9 +15,8 @@ import ( ) // Add the tag with id `tagId` to the job with the database id `jobId`. -func (r *JobRepository) AddTag(ctx context.Context, job int64, tag int64) ([]*schema.Tag, error) { - - j, err := r.FindById(ctx, job) +func (r *JobRepository) AddTag(user *schema.User, job int64, tag int64) ([]*schema.Tag, error) { + j, err := r.FindByIdWithUser(user, job) if err != nil { log.Warn("Error while finding job by id") return nil, err @@ -32,7 +30,7 @@ func (r *JobRepository) AddTag(ctx context.Context, job int64, tag int64) ([]*sc return nil, err } - tags, err := r.GetTags(ctx, &job) + tags, err := r.GetTags(user, &job) if err != nil { log.Warn("Error while getting tags for job") return nil, err @@ -48,9 +46,8 @@ func (r *JobRepository) AddTag(ctx context.Context, job int64, tag int64) ([]*sc } // Removes a tag from a job -func (r *JobRepository) RemoveTag(ctx context.Context, job, tag int64) ([]*schema.Tag, error) { - - j, err := r.FindById(ctx, job) +func (r *JobRepository) RemoveTag(user *schema.User, job, tag int64) ([]*schema.Tag, error) { + j, err := r.FindByIdWithUser(user, job) if err != nil { log.Warn("Error while finding job by id") return nil, err @@ -64,7 +61,7 @@ func (r *JobRepository) RemoveTag(ctx context.Context, job, tag int64) ([]*schem return nil, err } - tags, err := r.GetTags(ctx, &job) + tags, err := r.GetTags(user, &job) if err != nil { log.Warn("Error while getting tags for job") return nil, err @@ -81,7 +78,6 @@ func (r *JobRepository) RemoveTag(ctx context.Context, job, tag int64) ([]*schem // CreateTag creates a new tag with the specified type and name and returns its database id. func (r *JobRepository) CreateTag(tagType string, tagName string, tagScope string) (tagId int64, err error) { - // Default to "Global" scope if none defined if tagScope == "" { tagScope = "global" @@ -99,7 +95,7 @@ func (r *JobRepository) CreateTag(tagType string, tagName string, tagScope strin return res.LastInsertId() } -func (r *JobRepository) CountTags(ctx context.Context) (tags []schema.Tag, counts map[string]int, err error) { +func (r *JobRepository) CountTags(user *schema.User) (tags []schema.Tag, counts map[string]int, err error) { // Fetch all Tags in DB for Display in Frontend Tag-View tags = make([]schema.Tag, 0, 100) xrows, err := r.DB.Queryx("SELECT id, tag_type, tag_name, tag_scope FROM tag") @@ -114,7 +110,7 @@ func (r *JobRepository) CountTags(ctx context.Context) (tags []schema.Tag, count } // Handle Scope Filtering: Tag Scope is Global, Private (== Username) or User is auth'd to view Admin Tags - readable, err := r.checkScopeAuth(ctx, "read", t.Scope) + readable, err := r.checkScopeAuth(user, "read", t.Scope) if err != nil { return nil, nil, err } @@ -123,8 +119,6 @@ func (r *JobRepository) CountTags(ctx context.Context) (tags []schema.Tag, count } } - user := GetUserFromContext(ctx) - // Query and Count Jobs with attached Tags q := sq.Select("t.tag_name, t.id, count(jt.tag_id)"). From("tag t"). @@ -147,9 +141,9 @@ func (r *JobRepository) CountTags(ctx context.Context) (tags []schema.Tag, count // Unchanged: Needs to be own case still, due to UserRole/NoRole compatibility handling in else case } else if user != nil && user.HasRole(schema.RoleManager) { // MANAGER: Count own jobs plus project's jobs // Build ("project1", "project2", ...) list of variable length directly in SQL string - q = q.Where("jt.job_id IN (SELECT id FROM job WHERE job.user = ? OR job.project IN (\""+strings.Join(user.Projects, "\",\"")+"\"))", user.Username) + q = q.Where("jt.job_id IN (SELECT id FROM job WHERE job.hpc_user = ? OR job.project IN (\""+strings.Join(user.Projects, "\",\"")+"\"))", user.Username) } else if user != nil { // USER OR NO ROLE (Compatibility): Only count own jobs - q = q.Where("jt.job_id IN (SELECT id FROM job WHERE job.user = ?)", user.Username) + q = q.Where("jt.job_id IN (SELECT id FROM job WHERE job.hpc_user = ?)", user.Username) } rows, err := q.RunWith(r.stmtCache).Query() @@ -175,14 +169,13 @@ func (r *JobRepository) CountTags(ctx context.Context) (tags []schema.Tag, count // AddTagOrCreate adds the tag with the specified type and name to the job with the database id `jobId`. // If such a tag does not yet exist, it is created. -func (r *JobRepository) AddTagOrCreate(ctx context.Context, jobId int64, tagType string, tagName string, tagScope string) (tagId int64, err error) { - +func (r *JobRepository) AddTagOrCreate(user *schema.User, jobId int64, tagType string, tagName string, tagScope string) (tagId int64, err error) { // Default to "Global" scope if none defined if tagScope == "" { tagScope = "global" } - writable, err := r.checkScopeAuth(ctx, "write", tagScope) + writable, err := r.checkScopeAuth(user, "write", tagScope) if err != nil { return 0, err } @@ -198,7 +191,7 @@ func (r *JobRepository) AddTagOrCreate(ctx context.Context, jobId int64, tagType } } - if _, err := r.AddTag(ctx, jobId, tagId); err != nil { + if _, err := r.AddTag(user, jobId, tagId); err != nil { return 0, err } @@ -217,7 +210,7 @@ func (r *JobRepository) TagId(tagType string, tagName string, tagScope string) ( } // GetTags returns a list of all scoped tags if job is nil or of the tags that the job with that database ID has. -func (r *JobRepository) GetTags(ctx context.Context, job *int64) ([]*schema.Tag, error) { +func (r *JobRepository) GetTags(user *schema.User, job *int64) ([]*schema.Tag, error) { q := sq.Select("id", "tag_type", "tag_name", "tag_scope").From("tag") if job != nil { q = q.Join("jobtag ON jobtag.tag_id = tag.id").Where("jobtag.job_id = ?", *job) @@ -238,7 +231,7 @@ func (r *JobRepository) GetTags(ctx context.Context, job *int64) ([]*schema.Tag, return nil, err } // Handle Scope Filtering: Tag Scope is Global, Private (== Username) or User is auth'd to view Admin Tags - readable, err := r.checkScopeAuth(ctx, "read", tag.Scope) + readable, err := r.checkScopeAuth(user, "read", tag.Scope) if err != nil { return nil, err } @@ -299,8 +292,7 @@ func (r *JobRepository) ImportTag(jobId int64, tagType string, tagName string, t return nil } -func (r *JobRepository) checkScopeAuth(ctx context.Context, operation string, scope string) (pass bool, err error) { - user := GetUserFromContext(ctx) +func (r *JobRepository) checkScopeAuth(user *schema.User, operation string, scope string) (pass bool, err error) { if user != nil { switch { case operation == "write" && scope == "admin": diff --git a/internal/repository/testdata/job.db b/internal/repository/testdata/job.db index 23eba6f..43ec9d3 100644 Binary files a/internal/repository/testdata/job.db and b/internal/repository/testdata/job.db differ diff --git a/internal/repository/testdata/job.db-shm b/internal/repository/testdata/job.db-shm deleted file mode 100644 index fe9ac28..0000000 Binary files a/internal/repository/testdata/job.db-shm and /dev/null differ diff --git a/internal/repository/testdata/job.db-wal b/internal/repository/testdata/job.db-wal deleted file mode 100644 index e69de29..0000000 diff --git a/internal/repository/user.go b/internal/repository/user.go index db96136..9beca26 100644 --- a/internal/repository/user.go +++ b/internal/repository/user.go @@ -46,8 +46,8 @@ func GetUserRepository() *UserRepository { func (r *UserRepository) GetUser(username string) (*schema.User, error) { user := &schema.User{Username: username} var hashedPassword, name, rawRoles, email, rawProjects sql.NullString - if err := sq.Select("password", "ldap", "name", "roles", "email", "projects").From("user"). - Where("user.username = ?", username).RunWith(r.DB). + if err := sq.Select("password", "ldap", "name", "roles", "email", "projects").From("hpc_user"). + Where("hpc_user.username = ?", username).RunWith(r.DB). QueryRow().Scan(&hashedPassword, &user.AuthSource, &name, &rawRoles, &email, &rawProjects); err != nil { log.Warnf("Error while querying user '%v' from database", username) return nil, err @@ -73,7 +73,7 @@ func (r *UserRepository) GetUser(username string) (*schema.User, error) { func (r *UserRepository) GetLdapUsernames() ([]string, error) { var users []string - rows, err := r.DB.Query(`SELECT username FROM user WHERE user.ldap = 1`) + rows, err := r.DB.Query(`SELECT username FROM hpc_user WHERE hpc_user.ldap = 1`) if err != nil { log.Warn("Error while querying usernames") return nil, err @@ -121,7 +121,7 @@ func (r *UserRepository) AddUser(user *schema.User) error { vals = append(vals, int(user.AuthSource)) } - if _, err := sq.Insert("user").Columns(cols...).Values(vals...).RunWith(r.DB).Exec(); err != nil { + if _, err := sq.Insert("hpc_user").Columns(cols...).Values(vals...).RunWith(r.DB).Exec(); err != nil { log.Errorf("Error while inserting new user '%v' into DB", user.Username) return err } @@ -134,7 +134,7 @@ func (r *UserRepository) UpdateUser(dbUser *schema.User, user *schema.User) erro // user contains updated info, apply to dbuser // TODO: Discuss updatable fields if dbUser.Name != user.Name { - if _, err := sq.Update("user").Set("name", user.Name).Where("user.username = ?", dbUser.Username).RunWith(r.DB).Exec(); err != nil { + if _, err := sq.Update("hpc_user").Set("name", user.Name).Where("hpc_user.username = ?", dbUser.Username).RunWith(r.DB).Exec(); err != nil { log.Errorf("error while updating name of user '%s'", user.Username) return err } @@ -143,7 +143,7 @@ func (r *UserRepository) UpdateUser(dbUser *schema.User, user *schema.User) erro // Toggled until greenlit // if dbUser.HasRole(schema.RoleManager) && !reflect.DeepEqual(dbUser.Projects, user.Projects) { // projects, _ := json.Marshal(user.Projects) - // if _, err := sq.Update("user").Set("projects", projects).Where("user.username = ?", dbUser.Username).RunWith(r.DB).Exec(); err != nil { + // if _, err := sq.Update("hpc_user").Set("projects", projects).Where("hpc_user.username = ?", dbUser.Username).RunWith(r.DB).Exec(); err != nil { // return err // } // } @@ -152,7 +152,7 @@ func (r *UserRepository) UpdateUser(dbUser *schema.User, user *schema.User) erro } func (r *UserRepository) DelUser(username string) error { - _, err := r.DB.Exec(`DELETE FROM user WHERE user.username = ?`, username) + _, err := r.DB.Exec(`DELETE FROM hpc_user WHERE hpc_user.username = ?`, username) if err != nil { log.Errorf("Error while deleting user '%s' from DB", username) return err @@ -162,7 +162,7 @@ func (r *UserRepository) DelUser(username string) error { } func (r *UserRepository) ListUsers(specialsOnly bool) ([]*schema.User, error) { - q := sq.Select("username", "name", "email", "roles", "projects").From("user") + q := sq.Select("username", "name", "email", "roles", "projects").From("hpc_user") if specialsOnly { q = q.Where("(roles != '[\"user\"]' AND roles != '[]')") } @@ -223,7 +223,7 @@ func (r *UserRepository) AddRole( } roles, _ := json.Marshal(append(user.Roles, newRole)) - if _, err := sq.Update("user").Set("roles", roles).Where("user.username = ?", username).RunWith(r.DB).Exec(); err != nil { + if _, err := sq.Update("hpc_user").Set("roles", roles).Where("hpc_user.username = ?", username).RunWith(r.DB).Exec(); err != nil { log.Errorf("error while adding new role for user '%s'", user.Username) return err } @@ -259,7 +259,7 @@ func (r *UserRepository) RemoveRole(ctx context.Context, username string, queryr } mroles, _ := json.Marshal(newroles) - if _, err := sq.Update("user").Set("roles", mroles).Where("user.username = ?", username).RunWith(r.DB).Exec(); err != nil { + if _, err := sq.Update("hpc_user").Set("roles", mroles).Where("hpc_user.username = ?", username).RunWith(r.DB).Exec(); err != nil { log.Errorf("Error while removing role for user '%s'", user.Username) return err } @@ -285,7 +285,7 @@ func (r *UserRepository) AddProject( } projects, _ := json.Marshal(append(user.Projects, project)) - if _, err := sq.Update("user").Set("projects", projects).Where("user.username = ?", username).RunWith(r.DB).Exec(); err != nil { + if _, err := sq.Update("hpc_user").Set("projects", projects).Where("hpc_user.username = ?", username).RunWith(r.DB).Exec(); err != nil { return err } @@ -323,7 +323,7 @@ func (r *UserRepository) RemoveProject(ctx context.Context, username string, pro } else { result, _ = json.Marshal(newprojects) } - if _, err := sq.Update("user").Set("projects", result).Where("user.username = ?", username).RunWith(r.DB).Exec(); err != nil { + if _, err := sq.Update("hpc_user").Set("projects", result).Where("hpc_user.username = ?", username).RunWith(r.DB).Exec(); err != nil { return err } return nil @@ -339,9 +339,10 @@ const ContextUserKey ContextKey = "user" func GetUserFromContext(ctx context.Context) *schema.User { x := ctx.Value(ContextUserKey) if x == nil { + log.Warnf("no user retrieved from context") return nil } - + // log.Infof("user retrieved from context: %v", x.(*schema.User)) return x.(*schema.User) } @@ -354,7 +355,7 @@ func (r *UserRepository) FetchUserInCtx(ctx context.Context, username string) (* user := &model.User{Username: username} var name, email sql.NullString - if err := sq.Select("name", "email").From("user").Where("user.username = ?", username). + if err := sq.Select("name", "email").From("hpc_user").Where("hpc_user.username = ?", username). RunWith(r.DB).QueryRow().Scan(&name, &email); err != nil { if err == sql.ErrNoRows { /* This warning will be logged *often* for non-local users, i.e. users mentioned only in job-table or archive, */ diff --git a/internal/routerConfig/routes.go b/internal/routerConfig/routes.go index e6cb376..18d606d 100644 --- a/internal/routerConfig/routes.go +++ b/internal/routerConfig/routes.go @@ -35,7 +35,7 @@ type Route struct { var routes []Route = []Route{ {"/", "home.tmpl", "ClusterCockpit", false, setupHomeRoute}, - {"/config", "config.tmpl", "Settings", false, func(i InfoType, r *http.Request) InfoType { return i }}, + {"/config", "config.tmpl", "Settings", false, setupConfigRoute}, {"/monitoring/jobs/", "monitoring/jobs.tmpl", "Jobs - ClusterCockpit", true, func(i InfoType, r *http.Request) InfoType { return i }}, {"/monitoring/job/{id:[0-9]+}", "monitoring/job.tmpl", "Job - ClusterCockpit", false, setupJobRoute}, {"/monitoring/users/", "monitoring/list.tmpl", "Users - ClusterCockpit", true, func(i InfoType, r *http.Request) InfoType { i["listType"] = "USER"; return i }}, @@ -53,15 +53,19 @@ func setupHomeRoute(i InfoType, r *http.Request) InfoType { jobRepo := repository.GetJobRepository() groupBy := model.AggregateCluster + // startJobCount := time.Now() stats, err := jobRepo.JobCountGrouped(r.Context(), nil, &groupBy) if err != nil { log.Warnf("failed to count jobs: %s", err.Error()) } + // log.Infof("Timer HOME ROUTE startJobCount: %s", time.Since(startJobCount)) + // startRunningJobCount := time.Now() stats, err = jobRepo.AddJobCountGrouped(r.Context(), nil, &groupBy, stats, "running") if err != nil { log.Warnf("failed to count running jobs: %s", err.Error()) } + // log.Infof("Timer HOME ROUTE startRunningJobCount: %s", time.Since(startRunningJobCount)) i["clusters"] = stats @@ -77,6 +81,17 @@ func setupHomeRoute(i InfoType, r *http.Request) InfoType { return i } +func setupConfigRoute(i InfoType, r *http.Request) InfoType { + if util.CheckFileExists("./var/notice.txt") { + msg, err := os.ReadFile("./var/notice.txt") + if err == nil { + i["ncontent"] = string(msg) + } + } + + return i +} + func setupJobRoute(i InfoType, r *http.Request) InfoType { i["id"] = mux.Vars(r)["id"] if config.Keys.EmissionConstant != 0 { @@ -157,7 +172,7 @@ func setupAnalysisRoute(i InfoType, r *http.Request) InfoType { func setupTaglistRoute(i InfoType, r *http.Request) InfoType { jobRepo := repository.GetJobRepository() - tags, counts, err := jobRepo.CountTags(r.Context()) + tags, counts, err := jobRepo.CountTags(repository.GetUserFromContext(r.Context())) tagMap := make(map[string][]map[string]interface{}) if err != nil { log.Warnf("GetTags failed: %s", err.Error()) @@ -196,6 +211,7 @@ func setupTaglistRoute(i InfoType, r *http.Request) InfoType { return i } +// FIXME: Lots of redundant code. Needs refactoring func buildFilterPresets(query url.Values) map[string]interface{} { filterPresets := map[string]interface{}{} @@ -258,6 +274,16 @@ func buildFilterPresets(query url.Values) map[string]interface{} { } } } + if query.Get("numHWThreads") != "" { + parts := strings.Split(query.Get("numHWThreads"), "-") + if len(parts) == 2 { + a, e1 := strconv.Atoi(parts[0]) + b, e2 := strconv.Atoi(parts[1]) + if e1 == nil && e2 == nil { + filterPresets["numHWThreads"] = map[string]int{"from": a, "to": b} + } + } + } if query.Get("numAccelerators") != "" { parts := strings.Split(query.Get("numAccelerators"), "-") if len(parts) == 2 { @@ -299,7 +325,35 @@ func buildFilterPresets(query url.Values) map[string]interface{} { } } } - + if query.Get("energy") != "" { + parts := strings.Split(query.Get("energy"), "-") + if len(parts) == 2 { + a, e1 := strconv.Atoi(parts[0]) + b, e2 := strconv.Atoi(parts[1]) + if e1 == nil && e2 == nil { + filterPresets["energy"] = map[string]int{"from": a, "to": b} + } + } + } + if len(query["stat"]) != 0 { + statList := make([]map[string]interface{}, 0) + for _, statEntry := range query["stat"] { + parts := strings.Split(statEntry, "-") + if len(parts) == 3 { // Metric Footprint Stat Field, from - to + a, e1 := strconv.ParseInt(parts[1], 10, 64) + b, e2 := strconv.ParseInt(parts[2], 10, 64) + if e1 == nil && e2 == nil { + statEntry := map[string]interface{}{ + "field": parts[0], + "from": a, + "to": b, + } + statList = append(statList, statEntry) + } + } + } + filterPresets["stats"] = statList + } return filterPresets } @@ -322,6 +376,7 @@ func SetupRoutes(router *mux.Router, buildInfo web.Build) { // Get User -> What if NIL? user := repository.GetUserFromContext(r.Context()) + // Get Roles availableRoles, _ := schema.GetValidRolesMap(user) diff --git a/internal/taskManager/updateDurationService.go b/internal/taskManager/updateDurationService.go index fc7f446..81d799e 100644 --- a/internal/taskManager/updateDurationService.go +++ b/internal/taskManager/updateDurationService.go @@ -14,7 +14,7 @@ import ( func RegisterUpdateDurationWorker() { var frequency string - if config.Keys.CronFrequency.DurationWorker != "" { + if config.Keys.CronFrequency != nil && config.Keys.CronFrequency.DurationWorker != "" { frequency = config.Keys.CronFrequency.DurationWorker } else { frequency = "5m" diff --git a/internal/taskManager/updateFootprintService.go b/internal/taskManager/updateFootprintService.go index a0eccc8..d30d766 100644 --- a/internal/taskManager/updateFootprintService.go +++ b/internal/taskManager/updateFootprintService.go @@ -10,7 +10,7 @@ import ( "time" "github.com/ClusterCockpit/cc-backend/internal/config" - "github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher" + "github.com/ClusterCockpit/cc-backend/internal/metricdata" "github.com/ClusterCockpit/cc-backend/pkg/archive" "github.com/ClusterCockpit/cc-backend/pkg/log" "github.com/ClusterCockpit/cc-backend/pkg/schema" @@ -20,7 +20,7 @@ import ( func RegisterFootprintWorker() { var frequency string - if config.Keys.CronFrequency.FootprintWorker != "" { + if config.Keys.CronFrequency != nil && config.Keys.CronFrequency.FootprintWorker != "" { frequency = config.Keys.CronFrequency.FootprintWorker } else { frequency = "10m" @@ -37,32 +37,38 @@ func RegisterFootprintWorker() { cl := 0 log.Printf("Update Footprints started at %s", s.Format(time.RFC3339)) - t, err := jobRepo.TransactionInit() - if err != nil { - log.Errorf("Failed TransactionInit %v", err) - } - for _, cluster := range archive.Clusters { + s_cluster := time.Now() jobs, err := jobRepo.FindRunningJobs(cluster.Name) if err != nil { continue } + // NOTE: Additional Subcluster Loop Could Allow For Limited List Of Footprint-Metrics Only. + // - Chunk-Size Would Then Be 'SubCluster' (Running Jobs, Transactions) as Lists Can Change Within SCs + // - Would Require Review of 'updateFootprint' Usage (Logic Could Possibly Be Included Here Completely) allMetrics := make([]string, 0) metricConfigs := archive.GetCluster(cluster.Name).MetricConfig for _, mc := range metricConfigs { allMetrics = append(allMetrics, mc.Name) } - scopes := []schema.MetricScope{schema.MetricScopeNode} - scopes = append(scopes, schema.MetricScopeCore) - scopes = append(scopes, schema.MetricScopeAccelerator) + repo, err := metricdata.GetMetricDataRepo(cluster.Name) + if err != nil { + log.Errorf("no metric data repository configured for '%s'", cluster.Name) + continue + } + + pendingStatements := []sq.UpdateBuilder{} for _, job := range jobs { - log.Debugf("Try job %d", job.JobID) + log.Debugf("Prepare job %d", job.JobID) cl++ - jobData, err := metricDataDispatcher.LoadData(job, allMetrics, scopes, context.Background(), 0) // 0 Resolution-Value retrieves highest res + + s_job := time.Now() + + jobStats, err := repo.LoadStats(job, allMetrics, context.Background()) if err != nil { - log.Errorf("Error wile loading job data for footprint update: %v", err) + log.Errorf("error wile loading job data stats for footprint update: %v", err) ce++ continue } @@ -73,19 +79,19 @@ func RegisterFootprintWorker() { Statistics: make(map[string]schema.JobStatistics), } - for metric, data := range jobData { - avg, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32 - nodeData, ok := data["node"] - if !ok { - // This should never happen ? - ce++ - continue - } + for _, metric := range allMetrics { + avg, min, max := 0.0, 0.0, 0.0 + data, ok := jobStats[metric] // JobStats[Metric1:[Hostname1:[Stats], Hostname2:[Stats], ...], Metric2[...] ...] + if ok { + for _, res := range job.Resources { + hostStats, ok := data[res.Hostname] + if ok { + avg += hostStats.Avg + min = math.Min(min, hostStats.Min) + max = math.Max(max, hostStats.Max) + } - for _, series := range nodeData.Series { - avg += series.Statistics.Avg - min = math.Min(min, series.Statistics.Min) - max = math.Max(max, series.Statistics.Max) + } } // Add values rounded to 2 digits @@ -100,44 +106,41 @@ func RegisterFootprintWorker() { } } - // Init UpdateBuilder + // Build Statement per Job, Add to Pending Array stmt := sq.Update("job") - // Add SET queries stmt, err = jobRepo.UpdateFootprint(stmt, jobMeta) if err != nil { - log.Errorf("Update job (dbid: %d) failed at update Footprint step: %s", job.ID, err.Error()) + log.Errorf("update job (dbid: %d) statement build failed at footprint step: %s", job.ID, err.Error()) ce++ continue } - stmt, err = jobRepo.UpdateEnergy(stmt, jobMeta) - if err != nil { - log.Errorf("Update job (dbid: %d) failed at update Energy step: %s", job.ID, err.Error()) - ce++ - continue - } - // Add WHERE Filter stmt = stmt.Where("job.id = ?", job.ID) - query, args, err := stmt.ToSql() - if err != nil { - log.Errorf("Failed in ToSQL conversion: %v", err) - ce++ - continue - } - - // Args: JSON, JSON, ENERGY, JOBID - jobRepo.TransactionAdd(t, query, args...) - // if err := jobRepo.Execute(stmt); err != nil { - // log.Errorf("Update job footprint (dbid: %d) failed at db execute: %s", job.ID, err.Error()) - // continue - // } - c++ - log.Debugf("Finish Job %d", job.JobID) + pendingStatements = append(pendingStatements, stmt) + log.Debugf("Job %d took %s", job.JobID, time.Since(s_job)) } - jobRepo.TransactionCommit(t) - log.Debugf("Finish Cluster %s", cluster.Name) + + t, err := jobRepo.TransactionInit() + if err != nil { + log.Errorf("failed TransactionInit %v", err) + log.Errorf("skipped %d transactions for cluster %s", len(pendingStatements), cluster.Name) + ce += len(pendingStatements) + } else { + for _, ps := range pendingStatements { + query, args, err := ps.ToSql() + if err != nil { + log.Errorf("failed in ToSQL conversion: %v", err) + ce++ + } else { + // args...: Footprint-JSON, Energyfootprint-JSON, TotalEnergy, JobID + jobRepo.TransactionAdd(t, query, args...) + c++ + } + } + jobRepo.TransactionEnd(t) + } + log.Debugf("Finish Cluster %s, took %s", cluster.Name, time.Since(s_cluster)) } - jobRepo.TransactionEnd(t) log.Printf("Updating %d (of %d; Skipped %d) Footprints is done and took %s", c, cl, ce, time.Since(s)) })) } diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go index 52a760f..c6c04e4 100644 --- a/pkg/archive/archive.go +++ b/pkg/archive/archive.go @@ -14,7 +14,7 @@ import ( "github.com/ClusterCockpit/cc-backend/pkg/schema" ) -const Version uint64 = 1 +const Version uint64 = 2 type ArchiveBackend interface { Init(rawConfig json.RawMessage) (uint64, error) diff --git a/pkg/archive/fsBackend_test.go b/pkg/archive/fsBackend_test.go index d60e478..9db68ed 100644 --- a/pkg/archive/fsBackend_test.go +++ b/pkg/archive/fsBackend_test.go @@ -48,7 +48,7 @@ func TestInit(t *testing.T) { if fsa.path != "testdata/archive" { t.Fail() } - if version != 1 { + if version != 2 { t.Fail() } if len(fsa.clusters) != 3 || fsa.clusters[1] != "emmy" { diff --git a/pkg/archive/testdata/archive/version.txt b/pkg/archive/testdata/archive/version.txt index d00491f..0cfbf08 100644 --- a/pkg/archive/testdata/archive/version.txt +++ b/pkg/archive/testdata/archive/version.txt @@ -1 +1 @@ -1 +2 diff --git a/pkg/resampler/resampler.go b/pkg/resampler/resampler.go index 26cead0..ebc7e88 100644 --- a/pkg/resampler/resampler.go +++ b/pkg/resampler/resampler.go @@ -8,20 +8,20 @@ import ( "github.com/ClusterCockpit/cc-backend/pkg/schema" ) -func SimpleResampler(data []schema.Float, old_frequency int64, new_frequency int64) ([]schema.Float, error) { - if old_frequency == 0 || new_frequency == 0 { - return nil, errors.New("either old or new frequency is set to 0") +func SimpleResampler(data []schema.Float, old_frequency int64, new_frequency int64) ([]schema.Float, int64, error) { + if old_frequency == 0 || new_frequency == 0 || new_frequency <= old_frequency { + return data, old_frequency, nil } if new_frequency%old_frequency != 0 { - return nil, errors.New("new sampling frequency should be multiple of the old frequency") + return nil, 0, errors.New("new sampling frequency should be multiple of the old frequency") } var step int = int(new_frequency / old_frequency) var new_data_length = len(data) / step if new_data_length == 0 || len(data) < 100 || new_data_length >= len(data) { - return data, nil + return data, old_frequency, nil } new_data := make([]schema.Float, new_data_length) @@ -30,14 +30,14 @@ func SimpleResampler(data []schema.Float, old_frequency int64, new_frequency int new_data[i] = data[i*step] } - return new_data, nil + return new_data, new_frequency, nil } // Inspired by one of the algorithms from https://skemman.is/bitstream/1946/15343/3/SS_MSthesis.pdf // Adapted from https://github.com/haoel/downsampling/blob/master/core/lttb.go func LargestTriangleThreeBucket(data []schema.Float, old_frequency int, new_frequency int) ([]schema.Float, int, error) { - if old_frequency == 0 || new_frequency == 0 { + if old_frequency == 0 || new_frequency == 0 || new_frequency <= old_frequency { return data, old_frequency, nil } diff --git a/pkg/schema/cluster.go b/pkg/schema/cluster.go index b9bf306..0c88c61 100644 --- a/pkg/schema/cluster.go +++ b/pkg/schema/cluster.go @@ -48,29 +48,29 @@ type SubCluster struct { type SubClusterConfig struct { Name string `json:"name"` Footprint string `json:"footprint,omitempty"` + Energy string `json:"energy"` Peak float64 `json:"peak"` Normal float64 `json:"normal"` Caution float64 `json:"caution"` Alert float64 `json:"alert"` Remove bool `json:"remove"` LowerIsBetter bool `json:"lowerIsBetter"` - Energy string `json:"energy"` } type MetricConfig struct { Unit Unit `json:"unit"` + Energy string `json:"energy"` Name string `json:"name"` Scope MetricScope `json:"scope"` Aggregation string `json:"aggregation"` Footprint string `json:"footprint,omitempty"` SubClusters []*SubClusterConfig `json:"subClusters,omitempty"` Peak float64 `json:"peak"` - Normal float64 `json:"normal"` Caution float64 `json:"caution"` Alert float64 `json:"alert"` Timestep int `json:"timestep"` + Normal float64 `json:"normal"` LowerIsBetter bool `json:"lowerIsBetter"` - Energy string `json:"energy"` } type Cluster struct { diff --git a/pkg/schema/config.go b/pkg/schema/config.go index b87841c..f9116cf 100644 --- a/pkg/schema/config.go +++ b/pkg/schema/config.go @@ -57,9 +57,9 @@ type IntRange struct { } type TimeRange struct { - Range string `json:"range,omitempty"` // Optional, e.g. 'last6h' From *time.Time `json:"from"` To *time.Time `json:"to"` + Range string `json:"range,omitempty"` } type FilterRanges struct { @@ -82,16 +82,16 @@ type Retention struct { } type ResampleConfig struct { - // Trigger next zoom level at less than this many visible datapoints - Trigger int `json:"trigger"` // Array of resampling target resolutions, in seconds; Example: [600,300,60] Resolutions []int `json:"resolutions"` + // Trigger next zoom level at less than this many visible datapoints + Trigger int `json:"trigger"` } type CronFrequency struct { // Duration Update Worker [Defaults to '5m'] DurationWorker string `json:"duration-worker"` - // Metric- and Energy Footprint Update Worker [Defaults to '10m'] + // Metric-Footprint Update Worker [Defaults to '10m'] FootprintWorker string `json:"footprint-worker"` } @@ -164,13 +164,13 @@ type ProgramConfig struct { // Defines time X in seconds in which jobs are considered to be "short" and will be filtered in specific views. ShortRunningJobsDuration int `json:"short-running-jobs-duration"` - // Array of Clusters - Clusters []*ClusterConfig `json:"clusters"` - // Energy Mix CO2 Emission Constant [g/kWh] // If entered, displays estimated CO2 emission for job based on jobs totalEnergy EmissionConstant int `json:"emission-constant"` // Frequency of cron job workers CronFrequency *CronFrequency `json:"cron-frequency"` + + // Array of Clusters + Clusters []*ClusterConfig `json:"clusters"` } diff --git a/pkg/schema/job.go b/pkg/schema/job.go index f5bcc62..5e3110b 100644 --- a/pkg/schema/job.go +++ b/pkg/schema/job.go @@ -18,9 +18,9 @@ import ( type BaseJob struct { Cluster string `json:"cluster" db:"cluster" example:"fritz"` SubCluster string `json:"subCluster" db:"subcluster" example:"main"` - Partition string `json:"partition,omitempty" db:"partition" example:"main"` + Partition string `json:"partition,omitempty" db:"cluster_partition" example:"main"` Project string `json:"project" db:"project" example:"abcd200"` - User string `json:"user" db:"user" example:"abcd100h"` + User string `json:"user" db:"hpc_user" example:"abcd100h"` State JobState `json:"jobState" db:"job_state" example:"completed" enums:"completed,failed,cancelled,stopped,timeout,out_of_memory"` Tags []*Tag `json:"tags,omitempty"` RawEnergyFootprint []byte `json:"-" db:"energy_footprint"` diff --git a/pkg/schema/metrics.go b/pkg/schema/metrics.go index 9db853d..bbc3c74 100644 --- a/pkg/schema/metrics.go +++ b/pkg/schema/metrics.go @@ -17,17 +17,17 @@ import ( type JobData map[string]map[MetricScope]*JobMetric type JobMetric struct { - Unit Unit `json:"unit"` - Timestep int `json:"timestep"` - Series []Series `json:"series"` StatisticsSeries *StatsSeries `json:"statisticsSeries,omitempty"` + Unit Unit `json:"unit"` + Series []Series `json:"series"` + Timestep int `json:"timestep"` } type Series struct { - Hostname string `json:"hostname"` Id *string `json:"id,omitempty"` - Statistics MetricStatistics `json:"statistics"` + Hostname string `json:"hostname"` Data []Float `json:"data"` + Statistics MetricStatistics `json:"statistics"` } type MetricStatistics struct { @@ -37,11 +37,11 @@ type MetricStatistics struct { } type StatsSeries struct { + Percentiles map[int][]Float `json:"percentiles,omitempty"` Mean []Float `json:"mean"` Median []Float `json:"median"` Min []Float `json:"min"` Max []Float `json:"max"` - Percentiles map[int][]Float `json:"percentiles,omitempty"` } type MetricScope string @@ -229,7 +229,7 @@ func (jd *JobData) AddNodeScope(metric string) bool { return false } - var maxScope MetricScope = MetricScopeInvalid + maxScope := MetricScopeInvalid for scope := range scopes { maxScope = maxScope.Max(scope) } diff --git a/pkg/schema/schemas/config.schema.json b/pkg/schema/schemas/config.schema.json index cc6c553..0a3905a 100644 --- a/pkg/schema/schemas/config.schema.json +++ b/pkg/schema/schemas/config.schema.json @@ -1,454 +1,497 @@ { - "$schema": "http://json-schema.org/draft/2020-12/schema", - "$id": "embedfs://config.schema.json", - "title": "cc-backend configuration file schema", - "type": "object", - "properties": { - "addr": { - "description": "Address where the http (or https) server will listen on (for example: 'localhost:80').", - "type": "string" + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "embedfs://config.schema.json", + "title": "cc-backend configuration file schema", + "type": "object", + "properties": { + "addr": { + "description": "Address where the http (or https) server will listen on (for example: 'localhost:80').", + "type": "string" + }, + "apiAllowedIPs": { + "description": "Addresses from which secured API endpoints can be reached", + "type": "array", + "items": { + "type": "string" + } + }, + "user": { + "description": "Drop root permissions once .env was read and the port was taken. Only applicable if using privileged port.", + "type": "string" + }, + "group": { + "description": "Drop root permissions once .env was read and the port was taken. Only applicable if using privileged port.", + "type": "string" + }, + "disable-authentication": { + "description": "Disable authentication (for everything: API, Web-UI, ...).", + "type": "boolean" + }, + "embed-static-files": { + "description": "If all files in `web/frontend/public` should be served from within the binary itself (they are embedded) or not.", + "type": "boolean" + }, + "static-files": { + "description": "Folder where static assets can be found, if embed-static-files is false.", + "type": "string" + }, + "db-driver": { + "description": "sqlite3 or mysql (mysql will work for mariadb as well).", + "type": "string", + "enum": [ + "sqlite3", + "mysql" + ] + }, + "db": { + "description": "For sqlite3 a filename, for mysql a DSN in this format: https://github.com/go-sql-driver/mysql#dsn-data-source-name (Without query parameters!).", + "type": "string" + }, + "archive": { + "description": "Configuration keys for job-archive", + "type": "object", + "properties": { + "kind": { + "description": "Backend type for job-archive", + "type": "string", + "enum": [ + "file", + "s3" + ] }, - "user": { - "description": "Drop root permissions once .env was read and the port was taken. Only applicable if using privileged port.", - "type": "string" + "path": { + "description": "Path to job archive for file backend", + "type": "string" }, - "group": { - "description": "Drop root permissions once .env was read and the port was taken. Only applicable if using privileged port.", - "type": "string" + "compression": { + "description": "Setup automatic compression for jobs older than number of days", + "type": "integer" }, - "disable-authentication": { - "description": "Disable authentication (for everything: API, Web-UI, ...).", - "type": "boolean" - }, - "embed-static-files": { - "description": "If all files in `web/frontend/public` should be served from within the binary itself (they are embedded) or not.", - "type": "boolean" - }, - "static-files": { - "description": "Folder where static assets can be found, if embed-static-files is false.", - "type": "string" - }, - "db-driver": { - "description": "sqlite3 or mysql (mysql will work for mariadb as well).", - "type": "string", - "enum": [ - "sqlite3", - "mysql" - ] - }, - "db": { - "description": "For sqlite3 a filename, for mysql a DSN in this format: https://github.com/go-sql-driver/mysql#dsn-data-source-name (Without query parameters!).", - "type": "string" - }, - "job-archive": { - "description": "Configuration keys for job-archive", - "type": "object", - "properties": { - "kind": { - "description": "Backend type for job-archive", - "type": "string", - "enum": [ - "file", - "s3" - ] - }, - "path": { - "description": "Path to job archive for file backend", - "type": "string" - }, - "compression": { - "description": "Setup automatic compression for jobs older than number of days", - "type": "integer" - }, - "retention": { - "description": "Configuration keys for retention", - "type": "object", - "properties": { - "policy": { - "description": "Retention policy", - "type": "string", - "enum": [ - "none", - "delete", - "move" - ] - }, - "includeDB": { - "description": "Also remove jobs from database", - "type": "boolean" - }, - "age": { - "description": "Act on jobs with startTime older than age (in days)", - "type": "integer" - }, - "location": { - "description": "The target directory for retention. Only applicable for retention move.", - "type": "string" - } - }, - "required": [ - "policy" - ] - } + "retention": { + "description": "Configuration keys for retention", + "type": "object", + "properties": { + "policy": { + "description": "Retention policy", + "type": "string", + "enum": [ + "none", + "delete", + "move" + ] }, - "required": [ - "kind" - ] + "includeDB": { + "description": "Also remove jobs from database", + "type": "boolean" + }, + "age": { + "description": "Act on jobs with startTime older than age (in days)", + "type": "integer" + }, + "location": { + "description": "The target directory for retention. Only applicable for retention move.", + "type": "string" + } + }, + "required": [ + "policy" + ] + } + }, + "required": [ + "kind" + ] + }, + "disable-archive": { + "description": "Keep all metric data in the metric data repositories, do not write to the job-archive.", + "type": "boolean" + }, + "validate": { + "description": "Validate all input json documents against json schema.", + "type": "boolean" + }, + "session-max-age": { + "description": "Specifies for how long a session shall be valid as a string parsable by time.ParseDuration(). If 0 or empty, the session/token does not expire!", + "type": "string" + }, + "https-cert-file": { + "description": "Filepath to SSL certificate. If also https-key-file is set use HTTPS using those certificates.", + "type": "string" + }, + "https-key-file": { + "description": "Filepath to SSL key file. If also https-cert-file is set use HTTPS using those certificates.", + "type": "string" + }, + "redirect-http-to": { + "description": "If not the empty string and addr does not end in :80, redirect every request incoming at port 80 to that url.", + "type": "string" + }, + "stop-jobs-exceeding-walltime": { + "description": "If not zero, automatically mark jobs as stopped running X seconds longer than their walltime. Only applies if walltime is set for job.", + "type": "integer" + }, + "short-running-jobs-duration": { + "description": "Do not show running jobs shorter than X seconds.", + "type": "integer" + }, + "emission-constant": { + "description": ".", + "type": "integer" + }, + "cron-frequency": { + "description": "Frequency of cron job workers.", + "type": "object", + "properties": { + "duration-worker": { + "description": "Duration Update Worker [Defaults to '5m']", + "type": "string" }, - "disable-archive": { - "description": "Keep all metric data in the metric data repositories, do not write to the job-archive.", - "type": "boolean" + "footprint-worker": { + "description": "Metric-Footprint Update Worker [Defaults to '10m']", + "type": "string" + } + } + }, + "enable-resampling": { + "description": "Enable dynamic zoom in frontend metric plots.", + "type": "object", + "properties": { + "trigger": { + "description": "Trigger next zoom level at less than this many visible datapoints.", + "type": "integer" }, - "validate": { - "description": "Validate all input json documents against json schema.", - "type": "boolean" - }, - "session-max-age": { - "description": "Specifies for how long a session shall be valid as a string parsable by time.ParseDuration(). If 0 or empty, the session/token does not expire!", - "type": "string" - }, - "https-cert-file": { - "description": "Filepath to SSL certificate. If also https-key-file is set use HTTPS using those certificates.", - "type": "string" - }, - "https-key-file": { - "description": "Filepath to SSL key file. If also https-cert-file is set use HTTPS using those certificates.", - "type": "string" - }, - "redirect-http-to": { - "description": "If not the empty string and addr does not end in :80, redirect every request incoming at port 80 to that url.", - "type": "string" - }, - "stop-jobs-exceeding-walltime": { - "description": "If not zero, automatically mark jobs as stopped running X seconds longer than their walltime. Only applies if walltime is set for job.", + "resolutions": { + "description": "Array of resampling target resolutions, in seconds.", + "type": "array", + "items": { "type": "integer" + } + } + }, + "required": [ + "trigger", + "resolutions" + ] + }, + "jwts": { + "description": "For JWT token authentication.", + "type": "object", + "properties": { + "max-age": { + "description": "Configure how long a token is valid. As string parsable by time.ParseDuration()", + "type": "string" }, - "short-running-jobs-duration": { - "description": "Do not show running jobs shorter than X seconds.", - "type": "integer" + "cookieName": { + "description": "Cookie that should be checked for a JWT token.", + "type": "string" }, - "jwts": { - "description": "For JWT token authentication.", + "validateUser": { + "description": "Deny login for users not in database (but defined in JWT). Overwrite roles in JWT with database roles.", + "type": "boolean" + }, + "trustedIssuer": { + "description": "Issuer that should be accepted when validating external JWTs ", + "type": "string" + }, + "syncUserOnLogin": { + "description": "Add non-existent user to DB at login attempt with values provided in JWT.", + "type": "boolean" + } + }, + "required": [ + "max-age" + ] + }, + "oidc": { + "provider": { + "description": "", + "type": "string" + }, + "syncUserOnLogin": { + "description": "", + "type": "boolean" + }, + "updateUserOnLogin": { + "description": "", + "type": "boolean" + }, + "required": [ + "provider" + ] + }, + "ldap": { + "description": "For LDAP Authentication and user synchronisation.", + "type": "object", + "properties": { + "url": { + "description": "URL of LDAP directory server.", + "type": "string" + }, + "user_base": { + "description": "Base DN of user tree root.", + "type": "string" + }, + "search_dn": { + "description": "DN for authenticating LDAP admin account with general read rights.", + "type": "string" + }, + "user_bind": { + "description": "Expression used to authenticate users via LDAP bind. Must contain uid={username}.", + "type": "string" + }, + "user_filter": { + "description": "Filter to extract users for syncing.", + "type": "string" + }, + "username_attr": { + "description": "Attribute with full username. Default: gecos", + "type": "string" + }, + "sync_interval": { + "description": "Interval used for syncing local user table with LDAP directory. Parsed using time.ParseDuration.", + "type": "string" + }, + "sync_del_old_users": { + "description": "Delete obsolete users in database.", + "type": "boolean" + }, + "syncUserOnLogin": { + "description": "Add non-existent user to DB at login attempt if user exists in Ldap directory", + "type": "boolean" + } + }, + "required": [ + "url", + "user_base", + "search_dn", + "user_bind", + "user_filter" + ] + }, + "clusters": { + "description": "Configuration for the clusters to be displayed.", + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "description": "The name of the cluster.", + "type": "string" + }, + "metricDataRepository": { + "description": "Type of the metric data repository for this cluster", "type": "object", "properties": { - "max-age": { - "description": "Configure how long a token is valid. As string parsable by time.ParseDuration()", - "type": "string" - }, - "cookieName": { - "description": "Cookie that should be checked for a JWT token.", - "type": "string" - }, - "validateUser": { - "description": "Deny login for users not in database (but defined in JWT). Overwrite roles in JWT with database roles.", - "type": "boolean" - }, - "trustedIssuer": { - "description": "Issuer that should be accepted when validating external JWTs ", - "type": "string" - }, - "syncUserOnLogin": { - "description": "Add non-existent user to DB at login attempt with values provided in JWT.", - "type": "boolean" - } + "kind": { + "type": "string", + "enum": [ + "influxdb", + "prometheus", + "cc-metric-store", + "test" + ] + }, + "url": { + "type": "string" + }, + "token": { + "type": "string" + } }, "required": [ - "max-age" + "kind", + "url" ] - }, - "ldap": { - "description": "For LDAP Authentication and user synchronisation.", + }, + "filterRanges": { + "description": "This option controls the slider ranges for the UI controls of numNodes, duration, and startTime.", "type": "object", "properties": { - "url": { - "description": "URL of LDAP directory server.", - "type": "string" - }, - "user_base": { - "description": "Base DN of user tree root.", - "type": "string" - }, - "search_dn": { - "description": "DN for authenticating LDAP admin account with general read rights.", - "type": "string" - }, - "user_bind": { - "description": "Expression used to authenticate users via LDAP bind. Must contain uid={username}.", - "type": "string" - }, - "user_filter": { - "description": "Filter to extract users for syncing.", - "type": "string" - }, - "username_attr": { - "description": "Attribute with full username. Default: gecos", - "type": "string" - }, - "sync_interval": { - "description": "Interval used for syncing local user table with LDAP directory. Parsed using time.ParseDuration.", - "type": "string" - }, - "sync_del_old_users": { - "description": "Delete obsolete users in database.", - "type": "boolean" - }, - "syncUserOnLogin": { - "description": "Add non-existent user to DB at login attempt if user exists in Ldap directory", - "type": "boolean" - } - }, - "required": [ - "url", - "user_base", - "search_dn", - "user_bind", - "user_filter" - ] - }, - "clusters": { - "description": "Configuration for the clusters to be displayed.", - "type": "array", - "items": { + "numNodes": { + "description": "UI slider range for number of nodes", "type": "object", "properties": { - "name": { - "description": "The name of the cluster.", - "type": "string" - }, - "metricDataRepository": { - "description": "Type of the metric data repository for this cluster", - "type": "object", - "properties": { - "kind": { - "type": "string", - "enum": [ - "influxdb", - "prometheus", - "cc-metric-store", - "test" - ] - }, - "url": { - "type": "string" - }, - "token": { - "type": "string" - } - }, - "required": [ - "kind", - "url" - ] - }, - "filterRanges": { - "description": "This option controls the slider ranges for the UI controls of numNodes, duration, and startTime.", - "type": "object", - "properties": { - "numNodes": { - "description": "UI slider range for number of nodes", - "type": "object", - "properties": { - "from": { - "type": "integer" - }, - "to": { - "type": "integer" - } - }, - "required": [ - "from", - "to" - ] - }, - "duration": { - "description": "UI slider range for duration", - "type": "object", - "properties": { - "from": { - "type": "integer" - }, - "to": { - "type": "integer" - } - }, - "required": [ - "from", - "to" - ] - }, - "startTime": { - "description": "UI slider range for start time", - "type": "object", - "properties": { - "from": { - "type": "string", - "format": "date-time" - }, - "to": { - "type": "null" - } - }, - "required": [ - "from", - "to" - ] - } - }, - "required": [ - "numNodes", - "duration", - "startTime" - ] - } + "from": { + "type": "integer" + }, + "to": { + "type": "integer" + } }, "required": [ - "name", - "metricDataRepository", - "filterRanges" - ], - "minItems": 1 - } - }, - "ui-defaults": { - "description": "Default configuration for web UI", - "type": "object", - "properties": { - "plot_general_colorBackground": { - "description": "Color plot background according to job average threshold limits", - "type": "boolean" - }, - "plot_general_lineWidth": { - "description": "Initial linewidth", + "from", + "to" + ] + }, + "duration": { + "description": "UI slider range for duration", + "type": "object", + "properties": { + "from": { "type": "integer" - }, - "plot_list_jobsPerPage": { - "description": "Jobs shown per page in job lists", + }, + "to": { "type": "integer" + } }, - "plot_view_plotsPerRow": { - "description": "Number of plots per row in single job view", - "type": "integer" + "required": [ + "from", + "to" + ] + }, + "startTime": { + "description": "UI slider range for start time", + "type": "object", + "properties": { + "from": { + "type": "string", + "format": "date-time" + }, + "to": { + "type": "null" + } }, - "plot_view_showPolarplot": { - "description": "Option to toggle polar plot in single job view", - "type": "boolean" - }, - "plot_view_showRoofline": { - "description": "Option to toggle roofline plot in single job view", - "type": "boolean" - }, - "plot_view_showStatTable": { - "description": "Option to toggle the node statistic table in single job view", - "type": "boolean" - }, - "system_view_selectedMetric": { - "description": "Initial metric shown in system view", - "type": "string" - }, - "analysis_view_histogramMetrics": { - "description": "Metrics to show as job count histograms in analysis view", - "type": "array", - "items": { - "type": "string", - "minItems": 1 - } - }, - "analysis_view_scatterPlotMetrics": { - "description": "Initial scatter plto configuration in analysis view", - "type": "array", - "items": { - "type": "array", - "items": { - "type": "string", - "minItems": 2, - "maxItems": 2 - }, - "minItems": 1 - } - }, - "job_view_nodestats_selectedMetrics": { - "description": "Initial metrics shown in node statistics table of single job view", - "type": "array", - "items": { - "type": "string", - "minItems": 1 - } - }, - "job_view_polarPlotMetrics": { - "description": "Metrics shown in polar plot of single job view", - "type": "array", - "items": { - "type": "string", - "minItems": 1 - } - }, - "job_view_selectedMetrics": { - "description": "", - "type": "array", - "items": { - "type": "string", - "minItems": 1 - } - }, - "plot_general_colorscheme": { - "description": "Initial color scheme", - "type": "array", - "items": { - "type": "string", - "minItems": 1 - } - }, - "plot_list_selectedMetrics": { - "description": "Initial metric plots shown in jobs lists", - "type": "array", - "items": { - "type": "string", - "minItems": 1 - } - } + "required": [ + "from", + "to" + ] + } }, "required": [ - "plot_general_colorBackground", - "plot_general_lineWidth", - "plot_list_jobsPerPage", - "plot_view_plotsPerRow", - "plot_view_showPolarplot", - "plot_view_showRoofline", - "plot_view_showStatTable", - "system_view_selectedMetric", - "analysis_view_histogramMetrics", - "analysis_view_scatterPlotMetrics", - "job_view_nodestats_selectedMetrics", - "job_view_polarPlotMetrics", - "job_view_selectedMetrics", - "plot_general_colorscheme", - "plot_list_selectedMetrics" + "numNodes", + "duration", + "startTime" ] + } }, - "enable-resampling": { - "description": "Enable dynamic zoom in frontend metric plots.", - "type": "object", - "properties": { - "trigger": { - "description": "Trigger next zoom level at less than this many visible datapoints.", - "type": "integer" - }, - "resolutions": { - "description": "Array of resampling target resolutions, in seconds.", - "type": "array", - "items": { - "type": "integer" - } - } - }, - "required": [ - "trigger", - "resolutions" - ] - } + "required": [ + "name", + "metricDataRepository", + "filterRanges" + ], + "minItems": 1 + } }, - "required": [ - "jwts", - "clusters" - ] + "ui-defaults": { + "description": "Default configuration for web UI", + "type": "object", + "properties": { + "plot_general_colorBackground": { + "description": "Color plot background according to job average threshold limits", + "type": "boolean" + }, + "plot_general_lineWidth": { + "description": "Initial linewidth", + "type": "integer" + }, + "plot_list_jobsPerPage": { + "description": "Jobs shown per page in job lists", + "type": "integer" + }, + "plot_view_plotsPerRow": { + "description": "Number of plots per row in single job view", + "type": "integer" + }, + "plot_view_showPolarplot": { + "description": "Option to toggle polar plot in single job view", + "type": "boolean" + }, + "plot_view_showRoofline": { + "description": "Option to toggle roofline plot in single job view", + "type": "boolean" + }, + "plot_view_showStatTable": { + "description": "Option to toggle the node statistic table in single job view", + "type": "boolean" + }, + "system_view_selectedMetric": { + "description": "Initial metric shown in system view", + "type": "string" + }, + "job_view_showFootprint": { + "description": "Option to toggle footprint ui in single job view", + "type": "boolean" + }, + "job_list_usePaging": { + "description": "Option to switch from continous scroll to paging", + "type": "boolean" + }, + "analysis_view_histogramMetrics": { + "description": "Metrics to show as job count histograms in analysis view", + "type": "array", + "items": { + "type": "string", + "minItems": 1 + } + }, + "analysis_view_scatterPlotMetrics": { + "description": "Initial scatter plto configuration in analysis view", + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string", + "minItems": 2, + "maxItems": 2 + }, + "minItems": 1 + } + }, + "job_view_nodestats_selectedMetrics": { + "description": "Initial metrics shown in node statistics table of single job view", + "type": "array", + "items": { + "type": "string", + "minItems": 1 + } + }, + "job_view_selectedMetrics": { + "description": "", + "type": "array", + "items": { + "type": "string", + "minItems": 1 + } + }, + "plot_general_colorscheme": { + "description": "Initial color scheme", + "type": "array", + "items": { + "type": "string", + "minItems": 1 + } + }, + "plot_list_selectedMetrics": { + "description": "Initial metric plots shown in jobs lists", + "type": "array", + "items": { + "type": "string", + "minItems": 1 + } + } + }, + "required": [ + "plot_general_colorBackground", + "plot_general_lineWidth", + "plot_list_jobsPerPage", + "plot_view_plotsPerRow", + "plot_view_showPolarplot", + "plot_view_showRoofline", + "plot_view_showStatTable", + "system_view_selectedMetric", + "job_view_showFootprint", + "job_list_usePaging", + "analysis_view_histogramMetrics", + "analysis_view_scatterPlotMetrics", + "job_view_nodestats_selectedMetrics", + "job_view_selectedMetrics", + "plot_general_colorscheme", + "plot_list_selectedMetrics" + ] + } + }, + "required": [ + "jwts", + "clusters" + ] } diff --git a/pkg/schema/schemas/job-data.schema.json b/pkg/schema/schemas/job-data.schema.json index e8a5739..c0c492b 100644 --- a/pkg/schema/schemas/job-data.schema.json +++ b/pkg/schema/schemas/job-data.schema.json @@ -1,490 +1,490 @@ { - "$schema": "http://json-schema.org/draft/2020-12/schema", - "$id": "embedfs://job-data.schema.json", - "title": "Job metric data list", - "description": "Collection of metric data of a HPC job", - "type": "object", - "properties": { - "mem_used": { - "description": "Memory capacity used", - "type": "object", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "flops_any": { - "description": "Total flop rate with DP flops scaled up", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "socket": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "memoryDomain": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "core": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "hwthread": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "minProperties": 1 - }, - "mem_bw": { - "description": "Main memory bandwidth", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "socket": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "memoryDomain": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "minProperties": 1 - }, - "net_bw": { - "description": "Total fast interconnect network bandwidth", - "type": "object", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "ipc": { - "description": "Instructions executed per cycle", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "socket": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "memoryDomain": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "core": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "hwthread": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "minProperties": 1 - }, - "cpu_user": { - "description": "CPU user active core utilization", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "socket": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "memoryDomain": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "core": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "hwthread": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "minProperties": 1 - }, - "cpu_load": { - "description": "CPU requested core utilization (load 1m)", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "flops_dp": { - "description": "Double precision flop rate", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "socket": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "memoryDomain": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "core": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "hwthread": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "minProperties": 1 - }, - "flops_sp": { - "description": "Single precision flops rate", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "socket": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "memoryDomain": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "core": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "hwthread": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "minProperties": 1 - }, - "vectorization_ratio": { - "description": "Fraction of arithmetic instructions using SIMD instructions", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "socket": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "memoryDomain": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "core": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "hwthread": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "minProperties": 1 - }, - "cpu_power": { - "description": "CPU power consumption", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "socket": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "minProperties": 1 - }, - "mem_power": { - "description": "Memory power consumption", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "socket": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "minProperties": 1 - }, - "acc_utilization": { - "description": "GPU utilization", - "properties": { - "accelerator": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "required": [ - "accelerator" - ] - }, - "acc_mem_used": { - "description": "GPU memory capacity used", - "properties": { - "accelerator": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "required": [ - "accelerator" - ] - }, - "acc_power": { - "description": "GPU power consumption", - "properties": { - "accelerator": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "required": [ - "accelerator" - ] - }, - "clock": { - "description": "Average core frequency", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "socket": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "memoryDomain": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "core": { - "$ref": "embedfs://job-metric-data.schema.json" - }, - "hwthread": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "minProperties": 1 - }, - "eth_read_bw": { - "description": "Ethernet read bandwidth", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "eth_write_bw": { - "description": "Ethernet write bandwidth", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "filesystems": { - "description": "Array of filesystems", - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "type": { - "type": "string", - "enum": [ - "nfs", - "lustre", - "gpfs", - "nvme", - "ssd", - "hdd", - "beegfs" - ] - }, - "read_bw": { - "description": "File system read bandwidth", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "write_bw": { - "description": "File system write bandwidth", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "read_req": { - "description": "File system read requests", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "write_req": { - "description": "File system write requests", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "inodes": { - "description": "File system write requests", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "accesses": { - "description": "File system open and close", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "fsync": { - "description": "File system fsync", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "create": { - "description": "File system create", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "open": { - "description": "File system open", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "close": { - "description": "File system close", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "seek": { - "description": "File system seek", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - } - }, - "required": [ - "name", - "type", - "read_bw", - "write_bw" - ] - }, - "minItems": 1 + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "embedfs://job-data.schema.json", + "title": "Job metric data list", + "description": "Collection of metric data of a HPC job", + "type": "object", + "properties": { + "mem_used": { + "description": "Memory capacity used", + "type": "object", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" } + }, + "required": [ + "node" + ] }, - "ic_rcv_packets": { - "description": "Network interconnect read packets", + "flops_any": { + "description": "Total flop rate with DP flops scaled up", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "socket": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "memoryDomain": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "core": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "hwthread": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "minProperties": 1 + }, + "mem_bw": { + "description": "Main memory bandwidth", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "socket": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "memoryDomain": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "minProperties": 1 + }, + "net_bw": { + "description": "Total fast interconnect network bandwidth", + "type": "object", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "ipc": { + "description": "Instructions executed per cycle", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "socket": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "memoryDomain": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "core": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "hwthread": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "minProperties": 1 + }, + "cpu_user": { + "description": "CPU user active core utilization", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "socket": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "memoryDomain": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "core": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "hwthread": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "minProperties": 1 + }, + "cpu_load": { + "description": "CPU requested core utilization (load 1m)", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "flops_dp": { + "description": "Double precision flop rate", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "socket": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "memoryDomain": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "core": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "hwthread": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "minProperties": 1 + }, + "flops_sp": { + "description": "Single precision flops rate", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "socket": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "memoryDomain": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "core": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "hwthread": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "minProperties": 1 + }, + "vectorization_ratio": { + "description": "Fraction of arithmetic instructions using SIMD instructions", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "socket": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "memoryDomain": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "core": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "hwthread": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "minProperties": 1 + }, + "cpu_power": { + "description": "CPU power consumption", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "socket": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "minProperties": 1 + }, + "mem_power": { + "description": "Memory power consumption", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "socket": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "minProperties": 1 + }, + "acc_utilization": { + "description": "GPU utilization", + "properties": { + "accelerator": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "required": [ + "accelerator" + ] + }, + "acc_mem_used": { + "description": "GPU memory capacity used", + "properties": { + "accelerator": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "required": [ + "accelerator" + ] + }, + "acc_power": { + "description": "GPU power consumption", + "properties": { + "accelerator": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "required": [ + "accelerator" + ] + }, + "clock": { + "description": "Average core frequency", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "socket": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "memoryDomain": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "core": { + "$ref": "embedfs://job-metric-data.schema.json" + }, + "hwthread": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "minProperties": 1 + }, + "eth_read_bw": { + "description": "Ethernet read bandwidth", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "eth_write_bw": { + "description": "Ethernet write bandwidth", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "filesystems": { + "description": "Array of filesystems", + "type": "array", + "items": { + "type": "object", "properties": { - "node": { + "name": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "nfs", + "lustre", + "gpfs", + "nvme", + "ssd", + "hdd", + "beegfs" + ] + }, + "read_bw": { + "description": "File system read bandwidth", + "properties": { + "node": { "$ref": "embedfs://job-metric-data.schema.json" - } + } + }, + "required": [ + "node" + ] + }, + "write_bw": { + "description": "File system write bandwidth", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "read_req": { + "description": "File system read requests", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "write_req": { + "description": "File system write requests", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "inodes": { + "description": "File system write requests", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "accesses": { + "description": "File system open and close", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "fsync": { + "description": "File system fsync", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "create": { + "description": "File system create", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "open": { + "description": "File system open", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "close": { + "description": "File system close", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "seek": { + "description": "File system seek", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + } }, "required": [ - "node" - ] - }, - "ic_send_packets": { - "description": "Network interconnect send packet", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "ic_read_bw": { - "description": "Network interconnect read bandwidth", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "ic_write_bw": { - "description": "Network interconnect write bandwidth", - "properties": { - "node": { - "$ref": "embedfs://job-metric-data.schema.json" - } - }, - "required": [ - "node" + "name", + "type", + "read_bw", + "write_bw" ] + }, + "minItems": 1 + } + }, + "ic_rcv_packets": { + "description": "Network interconnect read packets", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + } }, "required": [ - "cpu_user", - "cpu_load", - "mem_used", - "flops_any", - "mem_bw", - "net_bw", - "filesystems" + "node" ] + }, + "ic_send_packets": { + "description": "Network interconnect send packet", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "ic_read_bw": { + "description": "Network interconnect read bandwidth", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "ic_write_bw": { + "description": "Network interconnect write bandwidth", + "properties": { + "node": { + "$ref": "embedfs://job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "required": [ + "cpu_user", + "cpu_load", + "mem_used", + "flops_any", + "mem_bw", + "net_bw", + "filesystems" + ] } diff --git a/pkg/schema/schemas/job-meta.schema.json b/pkg/schema/schemas/job-meta.schema.json index b907d7f..db7475c 100644 --- a/pkg/schema/schemas/job-meta.schema.json +++ b/pkg/schema/schemas/job-meta.schema.json @@ -1,351 +1,351 @@ { - "$schema": "http://json-schema.org/draft/2020-12/schema", - "$id": "embedfs://job-meta.schema.json", - "title": "Job meta data", - "description": "Meta data information of a HPC job", - "type": "object", - "properties": { - "jobId": { - "description": "The unique identifier of a job", - "type": "integer" - }, - "user": { - "description": "The unique identifier of a user", + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "embedfs://job-meta.schema.json", + "title": "Job meta data", + "description": "Meta data information of a HPC job", + "type": "object", + "properties": { + "jobId": { + "description": "The unique identifier of a job", + "type": "integer" + }, + "user": { + "description": "The unique identifier of a user", + "type": "string" + }, + "project": { + "description": "The unique identifier of a project", + "type": "string" + }, + "cluster": { + "description": "The unique identifier of a cluster", + "type": "string" + }, + "subCluster": { + "description": "The unique identifier of a sub cluster", + "type": "string" + }, + "partition": { + "description": "The Slurm partition to which the job was submitted", + "type": "string" + }, + "arrayJobId": { + "description": "The unique identifier of an array job", + "type": "integer" + }, + "numNodes": { + "description": "Number of nodes used", + "type": "integer", + "exclusiveMinimum": 0 + }, + "numHwthreads": { + "description": "Number of HWThreads used", + "type": "integer", + "exclusiveMinimum": 0 + }, + "numAcc": { + "description": "Number of accelerators used", + "type": "integer", + "exclusiveMinimum": 0 + }, + "exclusive": { + "description": "Specifies how nodes are shared. 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive, 2 - Shared among multiple jobs of same user", + "type": "integer", + "minimum": 0, + "maximum": 2 + }, + "monitoringStatus": { + "description": "State of monitoring system during job run", + "type": "integer" + }, + "smt": { + "description": "SMT threads used by job", + "type": "integer" + }, + "walltime": { + "description": "Requested walltime of job in seconds", + "type": "integer", + "exclusiveMinimum": 0 + }, + "jobState": { + "description": "Final state of job", + "type": "string", + "enum": [ + "completed", + "failed", + "cancelled", + "stopped", + "out_of_memory", + "timeout" + ] + }, + "startTime": { + "description": "Start epoch time stamp in seconds", + "type": "integer", + "exclusiveMinimum": 0 + }, + "duration": { + "description": "Duration of job in seconds", + "type": "integer", + "exclusiveMinimum": 0 + }, + "resources": { + "description": "Resources used by job", + "type": "array", + "items": { + "type": "object", + "properties": { + "hostname": { "type": "string" - }, - "project": { - "description": "The unique identifier of a project", - "type": "string" - }, - "cluster": { - "description": "The unique identifier of a cluster", - "type": "string" - }, - "subCluster": { - "description": "The unique identifier of a sub cluster", - "type": "string" - }, - "partition": { - "description": "The Slurm partition to which the job was submitted", - "type": "string" - }, - "arrayJobId": { - "description": "The unique identifier of an array job", - "type": "integer" - }, - "numNodes": { - "description": "Number of nodes used", - "type": "integer", - "exclusiveMinimum": 0 - }, - "numHwthreads": { - "description": "Number of HWThreads used", - "type": "integer", - "exclusiveMinimum": 0 - }, - "numAcc": { - "description": "Number of accelerators used", - "type": "integer", - "exclusiveMinimum": 0 - }, - "exclusive": { - "description": "Specifies how nodes are shared. 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive, 2 - Shared among multiple jobs of same user", - "type": "integer", - "minimum": 0, - "maximum": 2 - }, - "monitoringStatus": { - "description": "State of monitoring system during job run", - "type": "integer" - }, - "smt": { - "description": "SMT threads used by job", - "type": "integer" - }, - "walltime": { - "description": "Requested walltime of job in seconds", - "type": "integer", - "exclusiveMinimum": 0 - }, - "jobState": { - "description": "Final state of job", + }, + "hwthreads": { + "type": "array", + "description": "List of OS processor ids", + "items": { + "type": "integer" + } + }, + "accelerators": { + "type": "array", + "description": "List of of accelerator device ids", + "items": { + "type": "string" + } + }, + "configuration": { "type": "string", - "enum": [ - "completed", - "failed", - "cancelled", - "stopped", - "out_of_memory", - "timeout" - ] + "description": "The configuration options of the node" + } }, - "startTime": { - "description": "Start epoch time stamp in seconds", - "type": "integer", - "exclusiveMinimum": 0 + "required": [ + "hostname" + ], + "minItems": 1 + } + }, + "metaData": { + "description": "Additional information about the job", + "type": "object", + "properties": { + "jobScript": { + "type": "string", + "description": "The batch script of the job" }, - "duration": { - "description": "Duration of job in seconds", - "type": "integer", - "exclusiveMinimum": 0 + "jobName": { + "type": "string", + "description": "Slurm Job name" }, - "resources": { - "description": "Resources used by job", - "type": "array", - "items": { - "type": "object", - "properties": { - "hostname": { - "type": "string" - }, - "hwthreads": { - "type": "array", - "description": "List of OS processor ids", - "items": { - "type": "integer" - } - }, - "accelerators": { - "type": "array", - "description": "List of of accelerator device ids", - "items": { - "type": "string" - } - }, - "configuration": { - "type": "string", - "description": "The configuration options of the node" - } - }, - "required": [ - "hostname" - ], - "minItems": 1 - } + "slurmInfo": { + "type": "string", + "description": "Additional slurm infos as show by scontrol show job" + } + } + }, + "tags": { + "description": "List of tags", + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "type": { + "type": "string" + } }, - "metaData": { - "description": "Additional information about the job", + "required": [ + "name", + "type" + ] + }, + "uniqueItems": true + }, + "statistics": { + "description": "Job statistic data", + "type": "object", + "properties": { + "mem_used": { + "description": "Memory capacity used (required)", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "cpu_load": { + "description": "CPU requested core utilization (load 1m) (required)", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "flops_any": { + "description": "Total flop rate with DP flops scaled up (required)", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "mem_bw": { + "description": "Main memory bandwidth (required)", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "net_bw": { + "description": "Total fast interconnect network bandwidth (required)", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "file_bw": { + "description": "Total file IO bandwidth (required)", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "ipc": { + "description": "Instructions executed per cycle", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "cpu_user": { + "description": "CPU user active core utilization", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "flops_dp": { + "description": "Double precision flop rate", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "flops_sp": { + "description": "Single precision flops rate", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "rapl_power": { + "description": "CPU power consumption", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "acc_used": { + "description": "GPU utilization", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "acc_mem_used": { + "description": "GPU memory capacity used", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "acc_power": { + "description": "GPU power consumption", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "clock": { + "description": "Average core frequency", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "eth_read_bw": { + "description": "Ethernet read bandwidth", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "eth_write_bw": { + "description": "Ethernet write bandwidth", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "ic_rcv_packets": { + "description": "Network interconnect read packets", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "ic_send_packets": { + "description": "Network interconnect send packet", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "ic_read_bw": { + "description": "Network interconnect read bandwidth", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "ic_write_bw": { + "description": "Network interconnect write bandwidth", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "filesystems": { + "description": "Array of filesystems", + "type": "array", + "items": { "type": "object", "properties": { - "jobScript": { - "type": "string", - "description": "The batch script of the job" - }, - "jobName": { - "type": "string", - "description": "Slurm Job name" - }, - "slurmInfo": { - "type": "string", - "description": "Additional slurm infos as show by scontrol show job" - } - } - }, - "tags": { - "description": "List of tags", - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "type": { - "type": "string" - } - }, - "required": [ - "name", - "type" + "name": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "nfs", + "lustre", + "gpfs", + "nvme", + "ssd", + "hdd", + "beegfs" ] - }, - "uniqueItems": true - }, - "statistics": { - "description": "Job statistic data", - "type": "object", - "properties": { - "mem_used": { - "description": "Memory capacity used (required)", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "cpu_load": { - "description": "CPU requested core utilization (load 1m) (required)", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "flops_any": { - "description": "Total flop rate with DP flops scaled up (required)", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "mem_bw": { - "description": "Main memory bandwidth (required)", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "net_bw": { - "description": "Total fast interconnect network bandwidth (required)", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "file_bw": { - "description": "Total file IO bandwidth (required)", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "ipc": { - "description": "Instructions executed per cycle", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "cpu_user": { - "description": "CPU user active core utilization", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "flops_dp": { - "description": "Double precision flop rate", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "flops_sp": { - "description": "Single precision flops rate", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "rapl_power": { - "description": "CPU power consumption", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "acc_used": { - "description": "GPU utilization", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "acc_mem_used": { - "description": "GPU memory capacity used", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "acc_power": { - "description": "GPU power consumption", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "clock": { - "description": "Average core frequency", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "eth_read_bw": { - "description": "Ethernet read bandwidth", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "eth_write_bw": { - "description": "Ethernet write bandwidth", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "ic_rcv_packets": { - "description": "Network interconnect read packets", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "ic_send_packets": { - "description": "Network interconnect send packet", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "ic_read_bw": { - "description": "Network interconnect read bandwidth", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "ic_write_bw": { - "description": "Network interconnect write bandwidth", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "filesystems": { - "description": "Array of filesystems", - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "type": { - "type": "string", - "enum": [ - "nfs", - "lustre", - "gpfs", - "nvme", - "ssd", - "hdd", - "beegfs" - ] - }, - "read_bw": { - "description": "File system read bandwidth", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "write_bw": { - "description": "File system write bandwidth", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "read_req": { - "description": "File system read requests", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "write_req": { - "description": "File system write requests", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "inodes": { - "description": "File system write requests", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "accesses": { - "description": "File system open and close", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "fsync": { - "description": "File system fsync", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "create": { - "description": "File system create", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "open": { - "description": "File system open", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "close": { - "description": "File system close", - "$ref": "embedfs://job-metric-statistics.schema.json" - }, - "seek": { - "description": "File system seek", - "$ref": "embedfs://job-metric-statistics.schema.json" - } - }, - "required": [ - "name", - "type", - "read_bw", - "write_bw" - ] - }, - "minItems": 1 - } + }, + "read_bw": { + "description": "File system read bandwidth", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "write_bw": { + "description": "File system write bandwidth", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "read_req": { + "description": "File system read requests", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "write_req": { + "description": "File system write requests", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "inodes": { + "description": "File system write requests", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "accesses": { + "description": "File system open and close", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "fsync": { + "description": "File system fsync", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "create": { + "description": "File system create", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "open": { + "description": "File system open", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "close": { + "description": "File system close", + "$ref": "embedfs://job-metric-statistics.schema.json" + }, + "seek": { + "description": "File system seek", + "$ref": "embedfs://job-metric-statistics.schema.json" + } }, "required": [ - "cpu_user", - "cpu_load", - "mem_used", - "flops_any", - "mem_bw" + "name", + "type", + "read_bw", + "write_bw" ] + }, + "minItems": 1 } - }, - "required": [ - "jobId", - "user", - "project", - "cluster", - "subCluster", - "numNodes", - "exclusive", - "startTime", - "jobState", - "duration", - "resources", - "statistics" - ] + }, + "required": [ + "cpu_user", + "cpu_load", + "mem_used", + "flops_any", + "mem_bw" + ] + } + }, + "required": [ + "jobId", + "user", + "project", + "cluster", + "subCluster", + "numNodes", + "exclusive", + "startTime", + "jobState", + "duration", + "resources", + "statistics" + ] } diff --git a/pkg/schema/schemas/job-metric-data.schema.json b/pkg/schema/schemas/job-metric-data.schema.json index 3f2b934..ad499bf 100644 --- a/pkg/schema/schemas/job-metric-data.schema.json +++ b/pkg/schema/schemas/job-metric-data.schema.json @@ -1,216 +1,216 @@ { - "$schema": "http://json-schema.org/draft/2020-12/schema", - "$id": "embedfs://job-metric-data.schema.json", - "title": "Job metric data", - "description": "Metric data of a HPC job", - "type": "object", - "properties": { - "unit": { - "description": "Metric unit", - "$ref": "embedfs://unit.schema.json" - }, - "timestep": { - "description": "Measurement interval in seconds", - "type": "integer" - }, - "thresholds": { - "description": "Metric thresholds for specific system", - "type": "object", - "properties": { - "peak": { - "type": "number" - }, - "normal": { - "type": "number" - }, - "caution": { - "type": "number" - }, - "alert": { - "type": "number" - } - } - }, - "statisticsSeries": { - "type": "object", - "description": "Statistics series across topology", - "properties": { - "min": { - "type": "array", - "items": { - "type": "number", - "minimum": 0 - }, - "minItems": 3 - }, - "max": { - "type": "array", - "items": { - "type": "number", - "minimum": 0 - }, - "minItems": 3 - }, - "mean": { - "type": "array", - "items": { - "type": "number", - "minimum": 0 - }, - "minItems": 3 - }, - "percentiles": { - "type": "object", - "properties": { - "10": { - "type": "array", - "items": { - "type": "number", - "minimum": 0 - }, - "minItems": 3 - }, - "20": { - "type": "array", - "items": { - "type": "number", - "minimum": 0 - }, - "minItems": 3 - }, - "30": { - "type": "array", - "items": { - "type": "number", - "minimum": 0 - }, - "minItems": 3 - }, - "40": { - "type": "array", - "items": { - "type": "number", - "minimum": 0 - }, - "minItems": 3 - }, - "50": { - "type": "array", - "items": { - "type": "number", - "minimum": 0 - }, - "minItems": 3 - }, - "60": { - "type": "array", - "items": { - "type": "number", - "minimum": 0 - }, - "minItems": 3 - }, - "70": { - "type": "array", - "items": { - "type": "number", - "minimum": 0 - }, - "minItems": 3 - }, - "80": { - "type": "array", - "items": { - "type": "number", - "minimum": 0 - }, - "minItems": 3 - }, - "90": { - "type": "array", - "items": { - "type": "number", - "minimum": 0 - }, - "minItems": 3 - }, - "25": { - "type": "array", - "items": { - "type": "number", - "minimum": 0 - }, - "minItems": 3 - }, - "75": { - "type": "array", - "items": { - "type": "number", - "minimum": 0 - }, - "minItems": 3 - } - } - } - } - }, - "series": { - "type": "array", - "items": { - "type": "object", - "properties": { - "hostname": { - "type": "string" - }, - "id": { - "type": "string" - }, - "statistics": { - "type": "object", - "description": "Statistics across time dimension", - "properties": { - "avg": { - "description": "Series average", - "type": "number", - "minimum": 0 - }, - "min": { - "description": "Series minimum", - "type": "number", - "minimum": 0 - }, - "max": { - "description": "Series maximum", - "type": "number", - "minimum": 0 - } - }, - "required": [ - "avg", - "min", - "max" - ] - }, - "data": { - "type": "array", - "contains": { - "type": "number", - "minimum": 0 - }, - "minItems": 1 - } - }, - "required": [ - "hostname", - "statistics", - "data" - ] - } - } + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "embedfs://job-metric-data.schema.json", + "title": "Job metric data", + "description": "Metric data of a HPC job", + "type": "object", + "properties": { + "unit": { + "description": "Metric unit", + "$ref": "embedfs://unit.schema.json" }, - "required": [ - "unit", - "timestep", - "series" - ] + "timestep": { + "description": "Measurement interval in seconds", + "type": "integer" + }, + "thresholds": { + "description": "Metric thresholds for specific system", + "type": "object", + "properties": { + "peak": { + "type": "number" + }, + "normal": { + "type": "number" + }, + "caution": { + "type": "number" + }, + "alert": { + "type": "number" + } + } + }, + "statisticsSeries": { + "type": "object", + "description": "Statistics series across topology", + "properties": { + "min": { + "type": "array", + "items": { + "type": "number", + "minimum": 0 + }, + "minItems": 3 + }, + "max": { + "type": "array", + "items": { + "type": "number", + "minimum": 0 + }, + "minItems": 3 + }, + "mean": { + "type": "array", + "items": { + "type": "number", + "minimum": 0 + }, + "minItems": 3 + }, + "percentiles": { + "type": "object", + "properties": { + "10": { + "type": "array", + "items": { + "type": "number", + "minimum": 0 + }, + "minItems": 3 + }, + "20": { + "type": "array", + "items": { + "type": "number", + "minimum": 0 + }, + "minItems": 3 + }, + "30": { + "type": "array", + "items": { + "type": "number", + "minimum": 0 + }, + "minItems": 3 + }, + "40": { + "type": "array", + "items": { + "type": "number", + "minimum": 0 + }, + "minItems": 3 + }, + "50": { + "type": "array", + "items": { + "type": "number", + "minimum": 0 + }, + "minItems": 3 + }, + "60": { + "type": "array", + "items": { + "type": "number", + "minimum": 0 + }, + "minItems": 3 + }, + "70": { + "type": "array", + "items": { + "type": "number", + "minimum": 0 + }, + "minItems": 3 + }, + "80": { + "type": "array", + "items": { + "type": "number", + "minimum": 0 + }, + "minItems": 3 + }, + "90": { + "type": "array", + "items": { + "type": "number", + "minimum": 0 + }, + "minItems": 3 + }, + "25": { + "type": "array", + "items": { + "type": "number", + "minimum": 0 + }, + "minItems": 3 + }, + "75": { + "type": "array", + "items": { + "type": "number", + "minimum": 0 + }, + "minItems": 3 + } + } + } + } + }, + "series": { + "type": "array", + "items": { + "type": "object", + "properties": { + "hostname": { + "type": "string" + }, + "id": { + "type": "string" + }, + "statistics": { + "type": "object", + "description": "Statistics across time dimension", + "properties": { + "avg": { + "description": "Series average", + "type": "number", + "minimum": 0 + }, + "min": { + "description": "Series minimum", + "type": "number", + "minimum": 0 + }, + "max": { + "description": "Series maximum", + "type": "number", + "minimum": 0 + } + }, + "required": [ + "avg", + "min", + "max" + ] + }, + "data": { + "type": "array", + "contains": { + "type": "number", + "minimum": 0 + }, + "minItems": 1 + } + }, + "required": [ + "hostname", + "statistics", + "data" + ] + } + } + }, + "required": [ + "unit", + "timestep", + "series" + ] } diff --git a/pkg/schema/schemas/job-metric-statistics.schema.json b/pkg/schema/schemas/job-metric-statistics.schema.json index 3412c23..f753ed3 100644 --- a/pkg/schema/schemas/job-metric-statistics.schema.json +++ b/pkg/schema/schemas/job-metric-statistics.schema.json @@ -1,34 +1,34 @@ { - "$schema": "http://json-schema.org/draft/2020-12/schema", - "$id": "embedfs://job-metric-statistics.schema.json", - "title": "Job statistics", - "description": "Format specification for job metric statistics", - "type": "object", - "properties": { - "unit": { - "description": "Metric unit", - "$ref": "embedfs://unit.schema.json" - }, - "avg": { - "description": "Job metric average", - "type": "number", - "minimum": 0 - }, - "min": { - "description": "Job metric minimum", - "type": "number", - "minimum": 0 - }, - "max": { - "description": "Job metric maximum", - "type": "number", - "minimum": 0 - } + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "embedfs://job-metric-statistics.schema.json", + "title": "Job statistics", + "description": "Format specification for job metric statistics", + "type": "object", + "properties": { + "unit": { + "description": "Metric unit", + "$ref": "embedfs://unit.schema.json" }, - "required": [ - "unit", - "avg", - "min", - "max" - ] + "avg": { + "description": "Job metric average", + "type": "number", + "minimum": 0 + }, + "min": { + "description": "Job metric minimum", + "type": "number", + "minimum": 0 + }, + "max": { + "description": "Job metric maximum", + "type": "number", + "minimum": 0 + } + }, + "required": [ + "unit", + "avg", + "min", + "max" + ] } diff --git a/pkg/schema/schemas/unit.schema.json b/pkg/schema/schemas/unit.schema.json index 9ee781c..c0a3df3 100644 --- a/pkg/schema/schemas/unit.schema.json +++ b/pkg/schema/schemas/unit.schema.json @@ -1,40 +1,40 @@ { - "$schema": "http://json-schema.org/draft/2020-12/schema", - "$id": "embedfs://unit.schema.json", - "title": "Metric unit", - "description": "Format specification for job metric units", - "type": "object", - "properties": { - "base": { - "description": "Metric base unit", - "type": "string", - "enum": [ - "B", - "F", - "B/s", - "F/s", - "CPI", - "IPC", - "Hz", - "W", - "°C", - "" - ] - }, - "prefix": { - "description": "Unit prefix", - "type": "string", - "enum": [ - "K", - "M", - "G", - "T", - "P", - "E" - ] - } + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "embedfs://unit.schema.json", + "title": "Metric unit", + "description": "Format specification for job metric units", + "type": "object", + "properties": { + "base": { + "description": "Metric base unit", + "type": "string", + "enum": [ + "B", + "F", + "B/s", + "F/s", + "CPI", + "IPC", + "Hz", + "W", + "°C", + "" + ] }, - "required": [ - "base" - ] + "prefix": { + "description": "Unit prefix", + "type": "string", + "enum": [ + "K", + "M", + "G", + "T", + "P", + "E" + ] + } + }, + "required": [ + "base" + ] } diff --git a/pkg/schema/user.go b/pkg/schema/user.go index 7b1ca13..c004254 100644 --- a/pkg/schema/user.go +++ b/pkg/schema/user.go @@ -42,11 +42,11 @@ type User struct { Username string `json:"username"` Password string `json:"-"` Name string `json:"name"` + Email string `json:"email"` Roles []string `json:"roles"` + Projects []string `json:"projects"` AuthType AuthType `json:"authType"` AuthSource AuthSource `json:"authSource"` - Email string `json:"email"` - Projects []string `json:"projects"` } func (u *User) HasProject(project string) bool { diff --git a/tools/archive-migration/cluster.go b/tools/archive-migration/cluster.go deleted file mode 100644 index f9a45ad..0000000 --- a/tools/archive-migration/cluster.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. -// All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. -package main - -import ( - "github.com/ClusterCockpit/cc-backend/pkg/schema" -) - -// type Accelerator struct { -// ID string `json:"id"` -// Type string `json:"type"` -// Model string `json:"model"` -// } - -// type Topology struct { -// Node []int `json:"node"` -// Socket [][]int `json:"socket"` -// MemoryDomain [][]int `json:"memoryDomain"` -// Die [][]int `json:"die"` -// Core [][]int `json:"core"` -// Accelerators []*Accelerator `json:"accelerators"` -// } - -type SubCluster struct { - Name string `json:"name"` - Nodes string `json:"nodes"` - NumberOfNodes int `json:"numberOfNodes"` - ProcessorType string `json:"processorType"` - SocketsPerNode int `json:"socketsPerNode"` - CoresPerSocket int `json:"coresPerSocket"` - ThreadsPerCore int `json:"threadsPerCore"` - FlopRateScalar int `json:"flopRateScalar"` - FlopRateSimd int `json:"flopRateSimd"` - MemoryBandwidth int `json:"memoryBandwidth"` - Topology *schema.Topology `json:"topology"` -} - -// type SubClusterConfig struct { -// Name string `json:"name"` -// Peak float64 `json:"peak"` -// Normal float64 `json:"normal"` -// Caution float64 `json:"caution"` -// Alert float64 `json:"alert"` -// } - -type MetricConfig struct { - Name string `json:"name"` - Unit string `json:"unit"` - Scope schema.MetricScope `json:"scope"` - Aggregation string `json:"aggregation"` - Timestep int `json:"timestep"` - Peak float64 `json:"peak"` - Normal float64 `json:"normal"` - Caution float64 `json:"caution"` - Alert float64 `json:"alert"` - SubClusters []*schema.SubClusterConfig `json:"subClusters"` -} - -type Cluster struct { - Name string `json:"name"` - MetricConfig []*MetricConfig `json:"metricConfig"` - SubClusters []*SubCluster `json:"subClusters"` -} diff --git a/tools/archive-migration/clusterConfig.go b/tools/archive-migration/clusterConfig.go deleted file mode 100644 index 0f9f426..0000000 --- a/tools/archive-migration/clusterConfig.go +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. -// All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. -package main - -import ( - "errors" - "fmt" - - "github.com/ClusterCockpit/cc-backend/pkg/archive" - "github.com/ClusterCockpit/cc-backend/pkg/schema" -) - -var Clusters []*Cluster -var nodeLists map[string]map[string]archive.NodeList - -func initClusterConfig() error { - - Clusters = []*Cluster{} - nodeLists = map[string]map[string]archive.NodeList{} - - for _, c := range ar.GetClusters() { - - cluster, err := ar.LoadClusterCfg(c) - if err != nil { - return err - } - - if len(cluster.Name) == 0 || - len(cluster.MetricConfig) == 0 || - len(cluster.SubClusters) == 0 { - return errors.New("cluster.name, cluster.metricConfig and cluster.SubClusters should not be empty") - } - - for _, mc := range cluster.MetricConfig { - if len(mc.Name) == 0 { - return errors.New("cluster.metricConfig.name should not be empty") - } - if mc.Timestep < 1 { - return errors.New("cluster.metricConfig.timestep should not be smaller than one") - } - - // For backwards compability... - if mc.Scope == "" { - mc.Scope = schema.MetricScopeNode - } - if !mc.Scope.Valid() { - return errors.New("cluster.metricConfig.scope must be a valid scope ('node', 'scocket', ...)") - } - } - - Clusters = append(Clusters, cluster) - - nodeLists[cluster.Name] = make(map[string]archive.NodeList) - for _, sc := range cluster.SubClusters { - if sc.Nodes == "" { - continue - } - - nl, err := archive.ParseNodeList(sc.Nodes) - if err != nil { - return fmt.Errorf("in %s/cluster.json: %w", cluster.Name, err) - } - nodeLists[cluster.Name][sc.Name] = nl - } - } - - return nil -} - -func GetCluster(cluster string) *Cluster { - - for _, c := range Clusters { - if c.Name == cluster { - return c - } - } - return nil -} - -func GetSubCluster(cluster, subcluster string) *SubCluster { - - for _, c := range Clusters { - if c.Name == cluster { - for _, p := range c.SubClusters { - if p.Name == subcluster { - return p - } - } - } - } - return nil -} - -func GetMetricConfig(cluster, metric string) *MetricConfig { - - for _, c := range Clusters { - if c.Name == cluster { - for _, m := range c.MetricConfig { - if m.Name == metric { - return m - } - } - } - } - return nil -} - -// AssignSubCluster sets the `job.subcluster` property of the job based -// on its cluster and resources. -func AssignSubCluster(job *BaseJob) error { - - cluster := GetCluster(job.Cluster) - if cluster == nil { - return fmt.Errorf("unkown cluster: %#v", job.Cluster) - } - - if job.SubCluster != "" { - for _, sc := range cluster.SubClusters { - if sc.Name == job.SubCluster { - return nil - } - } - return fmt.Errorf("already assigned subcluster %#v unkown (cluster: %#v)", job.SubCluster, job.Cluster) - } - - if len(job.Resources) == 0 { - return fmt.Errorf("job without any resources/hosts") - } - - host0 := job.Resources[0].Hostname - for sc, nl := range nodeLists[job.Cluster] { - if nl != nil && nl.Contains(host0) { - job.SubCluster = sc - return nil - } - } - - if cluster.SubClusters[0].Nodes == "" { - job.SubCluster = cluster.SubClusters[0].Name - return nil - } - - return fmt.Errorf("no subcluster found for cluster %#v and host %#v", job.Cluster, host0) -} - -func GetSubClusterByNode(cluster, hostname string) (string, error) { - - for sc, nl := range nodeLists[cluster] { - if nl != nil && nl.Contains(hostname) { - return sc, nil - } - } - - c := GetCluster(cluster) - if c == nil { - return "", fmt.Errorf("unkown cluster: %#v", cluster) - } - - if c.SubClusters[0].Nodes == "" { - return c.SubClusters[0].Name, nil - } - - return "", fmt.Errorf("no subcluster found for cluster %#v and host %#v", cluster, hostname) -} diff --git a/tools/archive-migration/float.go b/tools/archive-migration/float.go deleted file mode 100644 index 3fbccf8..0000000 --- a/tools/archive-migration/float.go +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. -// All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. -package main - -import ( - "errors" - "io" - "math" - "strconv" -) - -// A custom float type is used so that (Un)MarshalJSON and -// (Un)MarshalGQL can be overloaded and NaN/null can be used. -// The default behaviour of putting every nullable value behind -// a pointer has a bigger overhead. -type Float float64 - -var NaN Float = Float(math.NaN()) -var nullAsBytes []byte = []byte("null") - -func (f Float) IsNaN() bool { - return math.IsNaN(float64(f)) -} - -// NaN will be serialized to `null`. -func (f Float) MarshalJSON() ([]byte, error) { - if f.IsNaN() { - return nullAsBytes, nil - } - - return strconv.AppendFloat(make([]byte, 0, 10), float64(f), 'f', 2, 64), nil -} - -// `null` will be unserialized to NaN. -func (f *Float) UnmarshalJSON(input []byte) error { - s := string(input) - if s == "null" { - *f = NaN - return nil - } - - val, err := strconv.ParseFloat(s, 64) - if err != nil { - return err - } - *f = Float(val) - return nil -} - -// UnmarshalGQL implements the graphql.Unmarshaler interface. -func (f *Float) UnmarshalGQL(v interface{}) error { - f64, ok := v.(float64) - if !ok { - return errors.New("invalid Float scalar") - } - - *f = Float(f64) - return nil -} - -// MarshalGQL implements the graphql.Marshaler interface. -// NaN will be serialized to `null`. -func (f Float) MarshalGQL(w io.Writer) { - if f.IsNaN() { - w.Write(nullAsBytes) - } else { - w.Write(strconv.AppendFloat(make([]byte, 0, 10), float64(f), 'f', 2, 64)) - } -} - -// Only used via REST-API, not via GraphQL. -// This uses a lot less allocations per series, -// but it turns out that the performance increase -// from using this is not that big. -func (s *Series) MarshalJSON() ([]byte, error) { - buf := make([]byte, 0, 512+len(s.Data)*8) - buf = append(buf, `{"hostname":"`...) - buf = append(buf, s.Hostname...) - buf = append(buf, '"') - if s.Id != nil { - buf = append(buf, `,"id":`...) - buf = strconv.AppendInt(buf, int64(*s.Id), 10) - } - if s.Statistics != nil { - buf = append(buf, `,"statistics":{"min":`...) - buf = strconv.AppendFloat(buf, s.Statistics.Min, 'f', 2, 64) - buf = append(buf, `,"avg":`...) - buf = strconv.AppendFloat(buf, s.Statistics.Avg, 'f', 2, 64) - buf = append(buf, `,"max":`...) - buf = strconv.AppendFloat(buf, s.Statistics.Max, 'f', 2, 64) - buf = append(buf, '}') - } - buf = append(buf, `,"data":[`...) - for i := 0; i < len(s.Data); i++ { - if i != 0 { - buf = append(buf, ',') - } - - if s.Data[i].IsNaN() { - buf = append(buf, `null`...) - } else { - buf = strconv.AppendFloat(buf, float64(s.Data[i]), 'f', 2, 32) - } - } - buf = append(buf, ']', '}') - return buf, nil -} diff --git a/tools/archive-migration/fsBackend.go b/tools/archive-migration/fsBackend.go deleted file mode 100644 index 81cf57e..0000000 --- a/tools/archive-migration/fsBackend.go +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. -// All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. -package main - -import ( - "bufio" - "bytes" - "encoding/json" - "fmt" - "os" - "path/filepath" - "strconv" - - "github.com/ClusterCockpit/cc-backend/pkg/log" -) - -type FsArchiveConfig struct { - Path string `json:"path"` -} - -type FsArchive struct { - path string - clusters []string -} - -func getPath( - job *JobMeta, - rootPath string, - file string) string { - - lvl1, lvl2 := fmt.Sprintf("%d", job.JobID/1000), fmt.Sprintf("%03d", job.JobID%1000) - return filepath.Join( - rootPath, - job.Cluster, - lvl1, lvl2, - strconv.FormatInt(job.StartTime, 10), file) -} - -func loadJobMeta(filename string) (*JobMeta, error) { - - f, err := os.Open(filename) - if err != nil { - log.Errorf("fsBackend loadJobMeta()- %v", err) - return &JobMeta{}, err - } - defer f.Close() - - return DecodeJobMeta(bufio.NewReader(f)) -} - -func (fsa *FsArchive) Init(rawConfig json.RawMessage) error { - - var config FsArchiveConfig - if err := json.Unmarshal(rawConfig, &config); err != nil { - log.Errorf("fsBackend Init()- %v", err) - return err - } - if config.Path == "" { - err := fmt.Errorf("fsBackend Init()- empty path") - log.Errorf("fsBackend Init()- %v", err) - return err - } - fsa.path = config.Path - - entries, err := os.ReadDir(fsa.path) - if err != nil { - log.Errorf("fsBackend Init()- %v", err) - return err - } - - for _, de := range entries { - fsa.clusters = append(fsa.clusters, de.Name()) - } - - return nil -} - -func (fsa *FsArchive) Iter() <-chan *JobMeta { - - ch := make(chan *JobMeta) - go func() { - clustersDir, err := os.ReadDir(fsa.path) - if err != nil { - log.Fatalf("Reading clusters failed: %s", err.Error()) - } - - for _, clusterDir := range clustersDir { - lvl1Dirs, err := os.ReadDir(filepath.Join(fsa.path, clusterDir.Name())) - if err != nil { - log.Fatalf("Reading jobs failed: %s", err.Error()) - } - - for _, lvl1Dir := range lvl1Dirs { - if !lvl1Dir.IsDir() { - // Could be the cluster.json file - continue - } - - lvl2Dirs, err := os.ReadDir(filepath.Join(fsa.path, clusterDir.Name(), lvl1Dir.Name())) - if err != nil { - log.Fatalf("Reading jobs failed: %s", err.Error()) - } - - for _, lvl2Dir := range lvl2Dirs { - dirpath := filepath.Join(fsa.path, clusterDir.Name(), lvl1Dir.Name(), lvl2Dir.Name()) - startTimeDirs, err := os.ReadDir(dirpath) - if err != nil { - log.Fatalf("Reading jobs failed: %s", err.Error()) - } - - for _, startTimeDir := range startTimeDirs { - if startTimeDir.IsDir() { - job, err := loadJobMeta(filepath.Join(dirpath, startTimeDir.Name(), "meta.json")) - if err != nil { - log.Errorf("in %s: %s", filepath.Join(dirpath, startTimeDir.Name()), err.Error()) - } else { - ch <- job - } - } - } - } - } - } - close(ch) - }() - return ch -} - -func (fsa *FsArchive) LoadClusterCfg(name string) (*Cluster, error) { - b, err := os.ReadFile(filepath.Join(fsa.path, name, "cluster.json")) - if err != nil { - log.Errorf("fsBackend LoadClusterCfg()- %v", err) - return &Cluster{}, err - } - return DecodeCluster(bytes.NewReader(b)) -} - -func (fsa *FsArchive) GetClusters() []string { - return fsa.clusters -} diff --git a/tools/archive-migration/job.go b/tools/archive-migration/job.go deleted file mode 100644 index 8705ce9..0000000 --- a/tools/archive-migration/job.go +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. -// All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. -package main - -import ( - "errors" - "fmt" - "io" - "time" - - "github.com/ClusterCockpit/cc-backend/pkg/schema" -) - -// Non-Swaggered Comment: BaseJob -// Non-Swaggered Comment: Common subset of Job and JobMeta. Use one of those, not this type directly. - -type BaseJob struct { - // The unique identifier of a job - JobID int64 `json:"jobId" db:"job_id" example:"123000"` - User string `json:"user" db:"user" example:"abcd100h"` // The unique identifier of a user - Project string `json:"project" db:"project" example:"abcd200"` // The unique identifier of a project - Cluster string `json:"cluster" db:"cluster" example:"fritz"` // The unique identifier of a cluster - SubCluster string `json:"subCluster" db:"subcluster" example:"main"` // The unique identifier of a sub cluster - Partition string `json:"partition" db:"partition" example:"main"` // The Slurm partition to which the job was submitted - ArrayJobId int64 `json:"arrayJobId" db:"array_job_id" example:"123000"` // The unique identifier of an array job - NumNodes int32 `json:"numNodes" db:"num_nodes" example:"2" minimum:"1"` // Number of nodes used (Min > 0) - NumHWThreads int32 `json:"numHwthreads" db:"num_hwthreads" example:"20" minimum:"1"` // Number of HWThreads used (Min > 0) - NumAcc int32 `json:"numAcc" db:"num_acc" example:"2" minimum:"1"` // Number of accelerators used (Min > 0) - Exclusive int32 `json:"exclusive" db:"exclusive" example:"1" minimum:"0" maximum:"2"` // Specifies how nodes are shared: 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple jobs of same user - MonitoringStatus int32 `json:"monitoringStatus" db:"monitoring_status" example:"1" minimum:"0" maximum:"3"` // State of monitoring system during job run: 0 - Disabled, 1 - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull - SMT int32 `json:"smt" db:"smt" example:"4"` // SMT threads used by job - State JobState `json:"jobState" db:"job_state" example:"completed" enums:"completed,failed,cancelled,stopped,timeout,out_of_memory"` // Final state of job - Duration int32 `json:"duration" db:"duration" example:"43200" minimum:"1"` // Duration of job in seconds (Min > 0) - Walltime int64 `json:"walltime" db:"walltime" example:"86400" minimum:"1"` // Requested walltime of job in seconds (Min > 0) - Tags []*schema.Tag `json:"tags"` // List of tags - RawResources []byte `json:"-" db:"resources"` // Resources used by job [As Bytes] - Resources []*Resource `json:"resources"` // Resources used by job - RawMetaData []byte `json:"-" db:"meta_data"` // Additional information about the job [As Bytes] - MetaData map[string]string `json:"metaData"` // Additional information about the job -} - -// Non-Swaggered Comment: Job -// Non-Swaggered Comment: This type is used as the GraphQL interface and using sqlx as a table row. - -// Job model -// @Description Information of a HPC job. -type Job struct { - // The unique identifier of a job in the database - ID int64 `json:"id" db:"id"` - BaseJob - StartTimeUnix int64 `json:"-" db:"start_time" example:"1649723812"` // Start epoch time stamp in seconds - StartTime time.Time `json:"startTime"` // Start time as 'time.Time' data type - MemUsedMax float64 `json:"-" db:"mem_used_max"` // MemUsedMax as Float64 - FlopsAnyAvg float64 `json:"-" db:"flops_any_avg"` // FlopsAnyAvg as Float64 - MemBwAvg float64 `json:"-" db:"mem_bw_avg"` // MemBwAvg as Float64 - LoadAvg float64 `json:"-" db:"load_avg"` // LoadAvg as Float64 - NetBwAvg float64 `json:"-" db:"net_bw_avg"` // NetBwAvg as Float64 - NetDataVolTotal float64 `json:"-" db:"net_data_vol_total"` // NetDataVolTotal as Float64 - FileBwAvg float64 `json:"-" db:"file_bw_avg"` // FileBwAvg as Float64 - FileDataVolTotal float64 `json:"-" db:"file_data_vol_total"` // FileDataVolTotal as Float64 -} - -// Non-Swaggered Comment: JobMeta -// Non-Swaggered Comment: When reading from the database or sending data via GraphQL, the start time can be in the much more -// Non-Swaggered Comment: convenient time.Time type. In the `meta.json` files, the start time is encoded as a unix epoch timestamp. -// Non-Swaggered Comment: This is why there is this struct, which contains all fields from the regular job struct, but "overwrites" -// Non-Swaggered Comment: the StartTime field with one of type int64. -// Non-Swaggered Comment: ID *int64 `json:"id,omitempty"` >> never used in the job-archive, only available via REST-API - -// JobMeta model -// @Description Meta data information of a HPC job. -type JobMeta struct { - // The unique identifier of a job in the database - ID *int64 `json:"id,omitempty"` - BaseJob - StartTime int64 `json:"startTime" db:"start_time" example:"1649723812" minimum:"1"` // Start epoch time stamp in seconds (Min > 0) - Statistics map[string]JobStatistics `json:"statistics,omitempty"` // Metric statistics of job -} - -const ( - MonitoringStatusDisabled int32 = 0 - MonitoringStatusRunningOrArchiving int32 = 1 - MonitoringStatusArchivingFailed int32 = 2 - MonitoringStatusArchivingSuccessful int32 = 3 -) - -var JobDefaults BaseJob = BaseJob{ - Exclusive: 1, - MonitoringStatus: MonitoringStatusRunningOrArchiving, -} - -// JobStatistics model -// @Description Specification for job metric statistics. -type JobStatistics struct { - // Metric unit (see schema/unit.schema.json) - Unit string `json:"unit" example:"GHz"` - Avg float64 `json:"avg" example:"2500" minimum:"0"` // Job metric average - Min float64 `json:"min" example:"2000" minimum:"0"` // Job metric minimum - Max float64 `json:"max" example:"3000" minimum:"0"` // Job metric maximum -} - -// Tag model -// @Description Defines a tag using name and type. -type Tag struct { - // The unique DB identifier of a tag - ID int64 `json:"id" db:"id"` - Type string `json:"type" db:"tag_type" example:"Debug"` // Tag Type - Name string `json:"name" db:"tag_name" example:"Testjob"` // Tag Name -} - -// Resource model -// @Description A resource used by a job -type Resource struct { - Hostname string `json:"hostname"` // Name of the host (= node) - HWThreads []int `json:"hwthreads,omitempty"` // List of OS processor ids - Accelerators []string `json:"accelerators,omitempty"` // List of of accelerator device ids - Configuration string `json:"configuration,omitempty"` // The configuration options of the node -} - -type JobState string - -const ( - JobStateRunning JobState = "running" - JobStateCompleted JobState = "completed" - JobStateFailed JobState = "failed" - JobStateCancelled JobState = "cancelled" - JobStateStopped JobState = "stopped" - JobStateTimeout JobState = "timeout" - JobStatePreempted JobState = "preempted" - JobStateOutOfMemory JobState = "out_of_memory" -) - -func (e *JobState) UnmarshalGQL(v interface{}) error { - str, ok := v.(string) - if !ok { - return fmt.Errorf("enums must be strings") - } - - *e = JobState(str) - if !e.Valid() { - return errors.New("invalid job state") - } - - return nil -} - -func (e JobState) MarshalGQL(w io.Writer) { - fmt.Fprintf(w, "\"%s\"", e) -} - -func (e JobState) Valid() bool { - return e == JobStateRunning || - e == JobStateCompleted || - e == JobStateFailed || - e == JobStateCancelled || - e == JobStateStopped || - e == JobStateTimeout || - e == JobStatePreempted || - e == JobStateOutOfMemory -} diff --git a/tools/archive-migration/json.go b/tools/archive-migration/json.go deleted file mode 100644 index b2c281c..0000000 --- a/tools/archive-migration/json.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. -// All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. -package main - -import ( - "encoding/json" - "io" - - "github.com/ClusterCockpit/cc-backend/pkg/schema" -) - -func DecodeJobData(r io.Reader) (*JobData, error) { - var d JobData - if err := json.NewDecoder(r).Decode(&d); err != nil { - return nil, err - } - - return &d, nil -} - -func DecodeJobMeta(r io.Reader) (*JobMeta, error) { - var d JobMeta - if err := json.NewDecoder(r).Decode(&d); err != nil { - return nil, err - } - - return &d, nil -} - -func DecodeCluster(r io.Reader) (*Cluster, error) { - var c Cluster - if err := json.NewDecoder(r).Decode(&c); err != nil { - return nil, err - } - - return &c, nil -} - -func EncodeJobData(w io.Writer, d *schema.JobData) error { - // Sanitize parameters - if err := json.NewEncoder(w).Encode(d); err != nil { - return err - } - - return nil -} - -func EncodeJobMeta(w io.Writer, d *schema.JobMeta) error { - // Sanitize parameters - if err := json.NewEncoder(w).Encode(d); err != nil { - return err - } - - return nil -} - -func EncodeCluster(w io.Writer, c *schema.Cluster) error { - // Sanitize parameters - if err := json.NewEncoder(w).Encode(c); err != nil { - return err - } - - return nil -} diff --git a/tools/archive-migration/main.go b/tools/archive-migration/main.go deleted file mode 100644 index b78e94e..0000000 --- a/tools/archive-migration/main.go +++ /dev/null @@ -1,371 +0,0 @@ -// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. -// All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. -package main - -import ( - "bufio" - "encoding/json" - "errors" - "flag" - "fmt" - "os" - "path/filepath" - "sync" - - "github.com/ClusterCockpit/cc-backend/internal/config" - "github.com/ClusterCockpit/cc-backend/pkg/log" - "github.com/ClusterCockpit/cc-backend/pkg/schema" - ccunits "github.com/ClusterCockpit/cc-units" -) - -const Version = 1 - -var ar FsArchive -var srcPath string -var dstPath string - -func loadJobData(filename string) (*JobData, error) { - - f, err := os.Open(filename) - if err != nil { - return &JobData{}, fmt.Errorf("fsBackend loadJobData()- %v", err) - } - defer f.Close() - - return DecodeJobData(bufio.NewReader(f)) -} - -func ConvertUnitString(us string) schema.Unit { - var nu schema.Unit - - if us == "CPI" || - us == "IPC" || - us == "load" || - us == "" { - nu.Base = us - return nu - } - u := ccunits.NewUnit(us) - p := u.GetPrefix() - if p.Prefix() != "" { - prefix := p.Prefix() - nu.Prefix = prefix - } - m := u.GetMeasure() - d := u.GetUnitDenominator() - if d.Short() != "inval" { - nu.Base = fmt.Sprintf("%s/%s", m.Short(), d.Short()) - } else { - nu.Base = m.Short() - } - - return nu -} - -func deepCopyJobMeta(j *JobMeta) schema.JobMeta { - var jn schema.JobMeta - - //required properties - jn.JobID = j.JobID - jn.User = j.User - jn.Project = j.Project - jn.Cluster = j.Cluster - jn.SubCluster = j.SubCluster - jn.NumNodes = j.NumNodes - jn.Exclusive = j.Exclusive - jn.StartTime = j.StartTime - jn.State = schema.JobState(j.State) - jn.Duration = j.Duration - - for _, ro := range j.Resources { - var rn schema.Resource - rn.Hostname = ro.Hostname - rn.Configuration = ro.Configuration - hwt := make([]int, len(ro.HWThreads)) - if ro.HWThreads != nil { - copy(hwt, ro.HWThreads) - } - rn.HWThreads = hwt - acc := make([]string, len(ro.Accelerators)) - if ro.Accelerators != nil { - copy(acc, ro.Accelerators) - } - rn.Accelerators = acc - jn.Resources = append(jn.Resources, &rn) - } - jn.MetaData = make(map[string]string) - - for k, v := range j.MetaData { - jn.MetaData[k] = v - } - - jn.Statistics = make(map[string]schema.JobStatistics) - for k, v := range j.Statistics { - var sn schema.JobStatistics - sn.Avg = v.Avg - sn.Max = v.Max - sn.Min = v.Min - tmpUnit := ConvertUnitString(v.Unit) - if tmpUnit.Base == "inval" { - sn.Unit = schema.Unit{Base: ""} - } else { - sn.Unit = tmpUnit - } - jn.Statistics[k] = sn - } - - //optional properties - jn.Partition = j.Partition - jn.ArrayJobId = j.ArrayJobId - jn.NumHWThreads = j.NumHWThreads - jn.NumAcc = j.NumAcc - jn.MonitoringStatus = j.MonitoringStatus - jn.SMT = j.SMT - jn.Walltime = j.Walltime - - for _, t := range j.Tags { - jn.Tags = append(jn.Tags, t) - } - - return jn -} - -func deepCopyJobData(d *JobData, cluster string, subCluster string) *schema.JobData { - var dn = make(schema.JobData) - - for k, v := range *d { - // fmt.Printf("Metric %s\n", k) - dn[k] = make(map[schema.MetricScope]*schema.JobMetric) - - for mk, mv := range v { - // fmt.Printf("Scope %s\n", mk) - var mn schema.JobMetric - tmpUnit := ConvertUnitString(mv.Unit) - if tmpUnit.Base == "inval" { - mn.Unit = schema.Unit{Base: ""} - } else { - mn.Unit = tmpUnit - } - - mn.Timestep = mv.Timestep - - for _, v := range mv.Series { - var sn schema.Series - sn.Hostname = v.Hostname - if v.Id != nil { - var id = new(string) - - if mk == schema.MetricScopeAccelerator { - s := GetSubCluster(cluster, subCluster) - var err error - - *id, err = s.Topology.GetAcceleratorID(*v.Id) - if err != nil { - log.Fatal(err) - } - - } else { - *id = fmt.Sprint(*v.Id) - } - sn.Id = id - } - if v.Statistics != nil { - sn.Statistics = schema.MetricStatistics{ - Avg: v.Statistics.Avg, - Min: v.Statistics.Min, - Max: v.Statistics.Max} - } - - sn.Data = make([]schema.Float, len(v.Data)) - copy(sn.Data, v.Data) - mn.Series = append(mn.Series, sn) - } - - dn[k][mk] = &mn - } - // fmt.Printf("FINISH %s\n", k) - } - - return &dn -} - -func deepCopyClusterConfig(co *Cluster) schema.Cluster { - var cn schema.Cluster - - cn.Name = co.Name - for _, sco := range co.SubClusters { - var scn schema.SubCluster - scn.Name = sco.Name - scn.Nodes = sco.Nodes - scn.ProcessorType = sco.ProcessorType - scn.SocketsPerNode = sco.SocketsPerNode - scn.CoresPerSocket = sco.CoresPerSocket - scn.ThreadsPerCore = sco.ThreadsPerCore - scn.FlopRateScalar = schema.MetricValue{ - Unit: schema.Unit{Base: "F/s", Prefix: "G"}, - Value: float64(sco.FlopRateScalar)} - scn.FlopRateSimd = schema.MetricValue{ - Unit: schema.Unit{Base: "F/s", Prefix: "G"}, - Value: float64(sco.FlopRateSimd)} - scn.MemoryBandwidth = schema.MetricValue{ - Unit: schema.Unit{Base: "B/s", Prefix: "G"}, - Value: float64(sco.MemoryBandwidth)} - scn.Topology = *sco.Topology - cn.SubClusters = append(cn.SubClusters, &scn) - } - - for _, mco := range co.MetricConfig { - var mcn schema.MetricConfig - mcn.Name = mco.Name - mcn.Scope = mco.Scope - if mco.Aggregation == "" { - fmt.Println("cluster.json - Property aggregation missing! Please review file!") - mcn.Aggregation = "sum" - } else { - mcn.Aggregation = mco.Aggregation - } - mcn.Timestep = mco.Timestep - tmpUnit := ConvertUnitString(mco.Unit) - if tmpUnit.Base == "inval" { - mcn.Unit = schema.Unit{Base: ""} - } else { - mcn.Unit = tmpUnit - } - mcn.Peak = mco.Peak - mcn.Normal = mco.Normal - mcn.Caution = mco.Caution - mcn.Alert = mco.Alert - mcn.SubClusters = mco.SubClusters - - cn.MetricConfig = append(cn.MetricConfig, &mcn) - } - - return cn -} - -func convertJob(job *JobMeta) { - // check if source data is available, otherwise skip job - src_data_path := getPath(job, srcPath, "data.json") - info, err := os.Stat(src_data_path) - if err != nil { - log.Fatal(err) - } - if info.Size() == 0 { - fmt.Printf("Skip path %s, filesize is 0 Bytes.", src_data_path) - return - } - - path := getPath(job, dstPath, "meta.json") - err = os.MkdirAll(filepath.Dir(path), 0750) - if err != nil { - log.Fatal(err) - } - f, err := os.Create(path) - if err != nil { - log.Fatal(err) - } - - jmn := deepCopyJobMeta(job) - if err = EncodeJobMeta(f, &jmn); err != nil { - log.Fatal(err) - } - if err = f.Close(); err != nil { - log.Fatal(err) - } - - f, err = os.Create(getPath(job, dstPath, "data.json")) - if err != nil { - log.Fatal(err) - } - - var jd *JobData - jd, err = loadJobData(src_data_path) - if err != nil { - log.Fatal(err) - } - jdn := deepCopyJobData(jd, job.Cluster, job.SubCluster) - if err := EncodeJobData(f, jdn); err != nil { - log.Fatal(err) - } - if err := f.Close(); err != nil { - log.Fatal(err) - } -} - -func main() { - var flagLogLevel, flagConfigFile string - var flagLogDateTime, debug bool - - flag.BoolVar(&flagLogDateTime, "logdate", false, "Set this flag to add date and time to log messages") - flag.BoolVar(&debug, "debug", false, "Set this flag to force sequential execution for debugging") - flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug,info,warn (default),err,fatal,crit]`") - flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`") - flag.StringVar(&srcPath, "src", "./var/job-archive", "Specify the source job archive path") - flag.StringVar(&dstPath, "dst", "./var/job-archive-new", "Specify the destination job archive path") - flag.Parse() - - if _, err := os.Stat(filepath.Join(srcPath, "version.txt")); !errors.Is(err, os.ErrNotExist) { - log.Fatal("Archive version exists!") - } - - log.Init(flagLogLevel, flagLogDateTime) - config.Init(flagConfigFile) - srcConfig := fmt.Sprintf("{\"path\": \"%s\"}", srcPath) - err := ar.Init(json.RawMessage(srcConfig)) - if err != nil { - log.Fatal(err) - } - - err = initClusterConfig() - if err != nil { - log.Fatal(err) - } - // setup new job archive - err = os.Mkdir(dstPath, 0750) - if err != nil { - log.Fatal(err) - } - - for _, c := range Clusters { - path := fmt.Sprintf("%s/%s", dstPath, c.Name) - fmt.Println(path) - err = os.Mkdir(path, 0750) - if err != nil { - log.Fatal(err) - } - cn := deepCopyClusterConfig(c) - - f, err := os.Create(fmt.Sprintf("%s/%s/cluster.json", dstPath, c.Name)) - if err != nil { - log.Fatal(err) - } - if err := EncodeCluster(f, &cn); err != nil { - log.Fatal(err) - } - if err := f.Close(); err != nil { - log.Fatal(err) - } - } - - var wg sync.WaitGroup - - for job := range ar.Iter() { - if debug { - fmt.Printf("Job %d\n", job.JobID) - convertJob(job) - } else { - job := job - wg.Add(1) - - go func() { - defer wg.Done() - convertJob(job) - }() - } - } - - wg.Wait() - os.WriteFile(filepath.Join(dstPath, "version.txt"), []byte(fmt.Sprintf("%d", Version)), 0644) -} diff --git a/tools/archive-migration/metrics.go b/tools/archive-migration/metrics.go deleted file mode 100644 index ec5de6f..0000000 --- a/tools/archive-migration/metrics.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. -// All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. -package main - -import ( - "github.com/ClusterCockpit/cc-backend/pkg/schema" -) - -type JobData map[string]map[schema.MetricScope]*JobMetric - -type JobMetric struct { - Unit string `json:"unit"` - Scope schema.MetricScope `json:"scope"` - Timestep int `json:"timestep"` - Series []Series `json:"series"` - StatisticsSeries *StatsSeries `json:"statisticsSeries"` -} - -type Series struct { - Hostname string `json:"hostname"` - Id *int `json:"id,omitempty"` - Statistics *MetricStatistics `json:"statistics"` - Data []schema.Float `json:"data"` -} - -type MetricStatistics struct { - Avg float64 `json:"avg"` - Min float64 `json:"min"` - Max float64 `json:"max"` -} - -type StatsSeries struct { - Mean []Float `json:"mean"` - Min []Float `json:"min"` - Max []Float `json:"max"` - Percentiles map[int][]Float `json:"percentiles,omitempty"` -} - -// type MetricScope string - -// const ( -// MetricScopeInvalid MetricScope = "invalid_scope" - -// MetricScopeNode MetricScope = "node" -// MetricScopeSocket MetricScope = "socket" -// MetricScopeMemoryDomain MetricScope = "memoryDomain" -// MetricScopeCore MetricScope = "core" -// MetricScopeHWThread MetricScope = "hwthread" - -// MetricScopeAccelerator MetricScope = "accelerator" -// ) - -// var metricScopeGranularity map[MetricScope]int = map[MetricScope]int{ -// MetricScopeNode: 10, -// MetricScopeSocket: 5, -// MetricScopeMemoryDomain: 3, -// MetricScopeCore: 2, -// MetricScopeHWThread: 1, - -// MetricScopeAccelerator: 5, // Special/Randomly choosen - -// MetricScopeInvalid: -1, -// } diff --git a/web/frontend/package-lock.json b/web/frontend/package-lock.json index e4c1d1e..e21171f 100644 --- a/web/frontend/package-lock.json +++ b/web/frontend/package-lock.json @@ -1,18 +1,18 @@ { "name": "cc-frontend", - "version": "1.0.1", + "version": "1.0.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "cc-frontend", - "version": "1.0.1", + "version": "1.0.2", "license": "MIT", "dependencies": { "@rollup/plugin-replace": "^5.0.7", "@sveltestrap/sveltestrap": "^6.2.7", - "@urql/svelte": "^4.2.1", - "chart.js": "^4.4.5", + "@urql/svelte": "^4.2.2", + "chart.js": "^4.4.6", "date-fns": "^2.30.0", "graphql": "^16.9.0", "mathjs": "^12.4.3", @@ -25,16 +25,16 @@ "@rollup/plugin-node-resolve": "^15.3.0", "@rollup/plugin-terser": "^0.4.4", "@timohausmann/quadtree-js": "^1.2.6", - "rollup": "^4.24.0", + "rollup": "^4.27.4", "rollup-plugin-css-only": "^4.5.2", "rollup-plugin-svelte": "^7.2.2", "svelte": "^4.2.19" } }, "node_modules/@0no-co/graphql.web": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/@0no-co/graphql.web/-/graphql.web-1.0.8.tgz", - "integrity": "sha512-8BG6woLtDMvXB9Ajb/uE+Zr/U7y4qJ3upXi0JQHZmsKUJa7HjF/gFvmL2f3/mSmfZoQGRr9VoY97LCX2uaFMzA==", + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@0no-co/graphql.web/-/graphql.web-1.0.11.tgz", + "integrity": "sha512-xuSJ9WXwTmtngWkbdEoopMo6F8NLtjy84UNAMsAr5C3/2SgAL/dEU10TMqTIsipqPQ8HA/7WzeqQ9DEQxSvPPA==", "license": "MIT", "peerDependencies": { "graphql": "^14.0.0 || ^15.0.0 || ^16.0.0" @@ -59,9 +59,9 @@ } }, "node_modules/@babel/runtime": { - "version": "7.25.7", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.25.7.tgz", - "integrity": "sha512-FjoyLe754PMiYsFaN5C94ttGiOmBNYTf6pLr4xXHAT5uctHb092PBszndLDR5XA/jghQvn4n7JMHl7dmTgbm9w==", + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.26.0.tgz", + "integrity": "sha512-FDSOghenHTiToteC/QRlv2q3DhPZ/oOXTBoirfWNx1Cx3TMVcGWQtMMmQcSvb/JjpNeGzx8Pq/b4fKEJuWm1sw==", "license": "MIT", "dependencies": { "regenerator-runtime": "^0.14.0" @@ -130,9 +130,9 @@ } }, "node_modules/@kurkle/color": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/@kurkle/color/-/color-0.3.2.tgz", - "integrity": "sha512-fuscdXJ9G1qb7W8VdHi+IwRqij3lBkosAm4ydQtEmbY58OzHXqQhvlxqEkoz0yssNVn38bcpRWgA9PP+OGoisw==", + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/@kurkle/color/-/color-0.3.4.tgz", + "integrity": "sha512-M5UknZPHRu3DEDWoipU6sE8PdkZ6Z/S+v4dD+Ke8IaNlpdSQah50lz1KtcFBa2vsdOnwbbnxJwVM4wty6udA5w==", "license": "MIT" }, "node_modules/@popperjs/core": { @@ -241,14 +241,14 @@ } }, "node_modules/@rollup/pluginutils": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.1.2.tgz", - "integrity": "sha512-/FIdS3PyZ39bjZlwqFnWqCOVnW7o963LtKMwQOD0NhQqw22gSr2YY1afu3FxRip4ZCZNsD5jq6Aaz6QV3D/Njw==", + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.1.3.tgz", + "integrity": "sha512-Pnsb6f32CD2W3uCaLZIzDmeFyQ2b8UWMFI7xtwUezpcGBDVDW6y9XgAWIlARiGAo6eNF5FK5aQTr0LFyNyqq5A==", "license": "MIT", "dependencies": { "@types/estree": "^1.0.0", "estree-walker": "^2.0.2", - "picomatch": "^2.3.1" + "picomatch": "^4.0.2" }, "engines": { "node": ">=14.0.0" @@ -263,9 +263,9 @@ } }, "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.24.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.24.0.tgz", - "integrity": "sha512-Q6HJd7Y6xdB48x8ZNVDOqsbh2uByBhgK8PiQgPhwkIw/HC/YX5Ghq2mQY5sRMZWHb3VsFkWooUVOZHKr7DmDIA==", + "version": "4.27.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.27.4.tgz", + "integrity": "sha512-2Y3JT6f5MrQkICUyRVCw4oa0sutfAsgaSsb0Lmmy1Wi2y7X5vT9Euqw4gOsCyy0YfKURBg35nhUKZS4mDcfULw==", "cpu": [ "arm" ], @@ -277,9 +277,9 @@ ] }, "node_modules/@rollup/rollup-android-arm64": { - "version": "4.24.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.24.0.tgz", - "integrity": "sha512-ijLnS1qFId8xhKjT81uBHuuJp2lU4x2yxa4ctFPtG+MqEE6+C5f/+X/bStmxapgmwLwiL3ih122xv8kVARNAZA==", + "version": "4.27.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.27.4.tgz", + "integrity": "sha512-wzKRQXISyi9UdCVRqEd0H4cMpzvHYt1f/C3CoIjES6cG++RHKhrBj2+29nPF0IB5kpy9MS71vs07fvrNGAl/iA==", "cpu": [ "arm64" ], @@ -291,9 +291,9 @@ ] }, "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.24.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.24.0.tgz", - "integrity": "sha512-bIv+X9xeSs1XCk6DVvkO+S/z8/2AMt/2lMqdQbMrmVpgFvXlmde9mLcbQpztXm1tajC3raFDqegsH18HQPMYtA==", + "version": "4.27.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.27.4.tgz", + "integrity": "sha512-PlNiRQapift4LNS8DPUHuDX/IdXiLjf8mc5vdEmUR0fF/pyy2qWwzdLjB+iZquGr8LuN4LnUoSEvKRwjSVYz3Q==", "cpu": [ "arm64" ], @@ -305,9 +305,9 @@ ] }, "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.24.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.24.0.tgz", - "integrity": "sha512-X6/nOwoFN7RT2svEQWUsW/5C/fYMBe4fnLK9DQk4SX4mgVBiTA9h64kjUYPvGQ0F/9xwJ5U5UfTbl6BEjaQdBQ==", + "version": "4.27.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.27.4.tgz", + "integrity": "sha512-o9bH2dbdgBDJaXWJCDTNDYa171ACUdzpxSZt+u/AAeQ20Nk5x+IhA+zsGmrQtpkLiumRJEYef68gcpn2ooXhSQ==", "cpu": [ "x64" ], @@ -318,10 +318,38 @@ "darwin" ] }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.27.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.27.4.tgz", + "integrity": "sha512-NBI2/i2hT9Q+HySSHTBh52da7isru4aAAo6qC3I7QFVsuhxi2gM8t/EI9EVcILiHLj1vfi+VGGPaLOUENn7pmw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.27.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.27.4.tgz", + "integrity": "sha512-wYcC5ycW2zvqtDYrE7deary2P2UFmSh85PUpAx+dwTCO9uw3sgzD6Gv9n5X4vLaQKsrfTSZZ7Z7uynQozPVvWA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.24.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.24.0.tgz", - "integrity": "sha512-0KXvIJQMOImLCVCz9uvvdPgfyWo93aHHp8ui3FrtOP57svqrF/roSSR5pjqL2hcMp0ljeGlU4q9o/rQaAQ3AYA==", + "version": "4.27.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.27.4.tgz", + "integrity": "sha512-9OwUnK/xKw6DyRlgx8UizeqRFOfi9mf5TYCw1uolDaJSbUmBxP85DE6T4ouCMoN6pXw8ZoTeZCSEfSaYo+/s1w==", "cpu": [ "arm" ], @@ -333,9 +361,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.24.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.24.0.tgz", - "integrity": "sha512-it2BW6kKFVh8xk/BnHfakEeoLPv8STIISekpoF+nBgWM4d55CZKc7T4Dx1pEbTnYm/xEKMgy1MNtYuoA8RFIWw==", + "version": "4.27.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.27.4.tgz", + "integrity": "sha512-Vgdo4fpuphS9V24WOV+KwkCVJ72u7idTgQaBoLRD0UxBAWTF9GWurJO9YD9yh00BzbkhpeXtm6na+MvJU7Z73A==", "cpu": [ "arm" ], @@ -347,9 +375,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.24.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.24.0.tgz", - "integrity": "sha512-i0xTLXjqap2eRfulFVlSnM5dEbTVque/3Pi4g2y7cxrs7+a9De42z4XxKLYJ7+OhE3IgxvfQM7vQc43bwTgPwA==", + "version": "4.27.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.27.4.tgz", + "integrity": "sha512-pleyNgyd1kkBkw2kOqlBx+0atfIIkkExOTiifoODo6qKDSpnc6WzUY5RhHdmTdIJXBdSnh6JknnYTtmQyobrVg==", "cpu": [ "arm64" ], @@ -361,9 +389,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.24.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.24.0.tgz", - "integrity": "sha512-9E6MKUJhDuDh604Qco5yP/3qn3y7SLXYuiC0Rpr89aMScS2UAmK1wHP2b7KAa1nSjWJc/f/Lc0Wl1L47qjiyQw==", + "version": "4.27.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.27.4.tgz", + "integrity": "sha512-caluiUXvUuVyCHr5DxL8ohaaFFzPGmgmMvwmqAITMpV/Q+tPoaHZ/PWa3t8B2WyoRcIIuu1hkaW5KkeTDNSnMA==", "cpu": [ "arm64" ], @@ -375,9 +403,9 @@ ] }, "node_modules/@rollup/rollup-linux-powerpc64le-gnu": { - "version": "4.24.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.24.0.tgz", - "integrity": "sha512-2XFFPJ2XMEiF5Zi2EBf4h73oR1V/lycirxZxHZNc93SqDN/IWhYYSYj8I9381ikUFXZrz2v7r2tOVk2NBwxrWw==", + "version": "4.27.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.27.4.tgz", + "integrity": "sha512-FScrpHrO60hARyHh7s1zHE97u0KlT/RECzCKAdmI+LEoC1eDh/RDji9JgFqyO+wPDb86Oa/sXkily1+oi4FzJQ==", "cpu": [ "ppc64" ], @@ -389,9 +417,9 @@ ] }, "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.24.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.24.0.tgz", - "integrity": "sha512-M3Dg4hlwuntUCdzU7KjYqbbd+BLq3JMAOhCKdBE3TcMGMZbKkDdJ5ivNdehOssMCIokNHFOsv7DO4rlEOfyKpg==", + "version": "4.27.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.27.4.tgz", + "integrity": "sha512-qyyprhyGb7+RBfMPeww9FlHwKkCXdKHeGgSqmIXw9VSUtvyFZ6WZRtnxgbuz76FK7LyoN8t/eINRbPUcvXB5fw==", "cpu": [ "riscv64" ], @@ -403,9 +431,9 @@ ] }, "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.24.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.24.0.tgz", - "integrity": "sha512-mjBaoo4ocxJppTorZVKWFpy1bfFj9FeCMJqzlMQGjpNPY9JwQi7OuS1axzNIk0nMX6jSgy6ZURDZ2w0QW6D56g==", + "version": "4.27.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.27.4.tgz", + "integrity": "sha512-PFz+y2kb6tbh7m3A7nA9++eInGcDVZUACulf/KzDtovvdTizHpZaJty7Gp0lFwSQcrnebHOqxF1MaKZd7psVRg==", "cpu": [ "s390x" ], @@ -417,9 +445,9 @@ ] }, "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.24.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.24.0.tgz", - "integrity": "sha512-ZXFk7M72R0YYFN5q13niV0B7G8/5dcQ9JDp8keJSfr3GoZeXEoMHP/HlvqROA3OMbMdfr19IjCeNAnPUG93b6A==", + "version": "4.27.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.27.4.tgz", + "integrity": "sha512-Ni8mMtfo+o/G7DVtweXXV/Ol2TFf63KYjTtoZ5f078AUgJTmaIJnj4JFU7TK/9SVWTaSJGxPi5zMDgK4w+Ez7Q==", "cpu": [ "x64" ], @@ -431,9 +459,9 @@ ] }, "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.24.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.24.0.tgz", - "integrity": "sha512-w1i+L7kAXZNdYl+vFvzSZy8Y1arS7vMgIy8wusXJzRrPyof5LAb02KGr1PD2EkRcl73kHulIID0M501lN+vobQ==", + "version": "4.27.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.27.4.tgz", + "integrity": "sha512-5AeeAF1PB9TUzD+3cROzFTnAJAcVUGLuR8ng0E0WXGkYhp6RD6L+6szYVX+64Rs0r72019KHZS1ka1q+zU/wUw==", "cpu": [ "x64" ], @@ -445,9 +473,9 @@ ] }, "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.24.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.24.0.tgz", - "integrity": "sha512-VXBrnPWgBpVDCVY6XF3LEW0pOU51KbaHhccHw6AS6vBWIC60eqsH19DAeeObl+g8nKAz04QFdl/Cefta0xQtUQ==", + "version": "4.27.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.27.4.tgz", + "integrity": "sha512-yOpVsA4K5qVwu2CaS3hHxluWIK5HQTjNV4tWjQXluMiiiu4pJj4BN98CvxohNCpcjMeTXk/ZMJBRbgRg8HBB6A==", "cpu": [ "arm64" ], @@ -459,9 +487,9 @@ ] }, "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.24.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.24.0.tgz", - "integrity": "sha512-xrNcGDU0OxVcPTH/8n/ShH4UevZxKIO6HJFK0e15XItZP2UcaiLFd5kiX7hJnqCbSztUF8Qot+JWBC/QXRPYWQ==", + "version": "4.27.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.27.4.tgz", + "integrity": "sha512-KtwEJOaHAVJlxV92rNYiG9JQwQAdhBlrjNRp7P9L8Cb4Rer3in+0A+IPhJC9y68WAi9H0sX4AiG2NTsVlmqJeQ==", "cpu": [ "ia32" ], @@ -473,9 +501,9 @@ ] }, "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.24.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.24.0.tgz", - "integrity": "sha512-fbMkAF7fufku0N2dE5TBXcNlg0pt0cJue4xBRE2Qc5Vqikxr4VCgKj/ht6SMdFcOacVA9rqF70APJ8RN/4vMJw==", + "version": "4.27.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.27.4.tgz", + "integrity": "sha512-3j4jx1TppORdTAoBJRd+/wJRGCPC0ETWkXOecJ6PPZLj6SptXkrXcNqdj0oclbKML6FkQltdz7bBA3rUSirZug==", "cpu": [ "x64" ], @@ -519,9 +547,9 @@ "license": "MIT" }, "node_modules/@urql/core": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/@urql/core/-/core-5.0.6.tgz", - "integrity": "sha512-38rgSDqVNihFDauw1Pm9V7XLWIKuK8V9CKgrUF7/xEKinze8ENKP1ZeBhkG+dxWzJan7CHK+SLl46kAdvZwIlA==", + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/@urql/core/-/core-5.0.8.tgz", + "integrity": "sha512-1GOnUw7/a9bzkcM0+U8U5MmxW2A7FE5YquuEmcJzTtW5tIs2EoS4F2ITpuKBjRBbyRjZgO860nWFPo1m4JImGA==", "license": "MIT", "dependencies": { "@0no-co/graphql.web": "^1.0.5", @@ -529,9 +557,9 @@ } }, "node_modules/@urql/svelte": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/@urql/svelte/-/svelte-4.2.1.tgz", - "integrity": "sha512-tzjt5qElu6EF4ns+AWLUFvvGFH+bDGEgLStHQTBu76puQcMCW374MrjxWM9lKA6lfA7iUyu1KXkIRhxNy09l4Q==", + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/@urql/svelte/-/svelte-4.2.2.tgz", + "integrity": "sha512-6ntLGsWcnNtaMZVmFpePfFTSpYxYpznCAqnuvLDjt7Oa7YqHcFiyPnz7IIsiPD9VE6hZSi0+RwmRk5BMba/teQ==", "license": "MIT", "dependencies": { "@urql/core": "^5.0.0", @@ -543,9 +571,9 @@ } }, "node_modules/acorn": { - "version": "8.13.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.13.0.tgz", - "integrity": "sha512-8zSiw54Oxrdym50NlZ9sUusyO1Z1ZchgRLWRaK6c86XJFClyCgFKetdowBg5bKxyp/u+CDBJG4Mpp0m3HLZl9w==", + "version": "8.14.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz", + "integrity": "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==", "license": "MIT", "bin": { "acorn": "bin/acorn" @@ -597,9 +625,9 @@ "license": "MIT" }, "node_modules/chart.js": { - "version": "4.4.5", - "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.4.5.tgz", - "integrity": "sha512-CVVjg1RYTJV9OCC8WeJPMx8gsV8K6WIyIEQUE3ui4AR9Hfgls9URri6Ja3hyMVBbTF8Q2KFa19PE815gWcWhng==", + "version": "4.4.6", + "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.4.6.tgz", + "integrity": "sha512-8Y406zevUPbbIBA/HRk33khEmQPk5+cxeflWE/2rx1NJsjVWMPw/9mSP9rxHP5eqi6LNoPBVMfZHxbwLSgldYA==", "license": "MIT", "dependencies": { "@kurkle/color": "^0.3.0" @@ -645,9 +673,9 @@ "license": "MIT" }, "node_modules/complex.js": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/complex.js/-/complex.js-2.3.0.tgz", - "integrity": "sha512-wWHzifVdUPbPBhh+ObvpVGIzrAQjTvmnnEJKBfLW5YbyAB6OXQ0r+Q92fByMIrSSlxUuCujqxriJSR6R/kVxPA==", + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/complex.js/-/complex.js-2.4.2.tgz", + "integrity": "sha512-qtx7HRhPGSCBtGiST4/WGHuW+zeaND/6Ld+db6PbrulIB1i2Ev/2UPiqcmpQNPSyfBKraC0EOvOKCB5dGZKt3g==", "license": "MIT", "engines": { "node": "*" @@ -867,9 +895,9 @@ "license": "MIT" }, "node_modules/magic-string": { - "version": "0.30.12", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.12.tgz", - "integrity": "sha512-Ea8I3sQMVXr8JhN4z+H/d8zwo+tYDgHE9+5G4Wnrwhs0gaK9fXTKx0Tw5Xwsd/bCPTTZNRAdpyzvoeORe9LYpw==", + "version": "0.30.14", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.14.tgz", + "integrity": "sha512-5c99P1WKTed11ZC0HMJOj6CDIue6F8ySu+bJL+85q1zBEIY8IklrJ1eiKC2NDRh3Ct3FcvmJPyQHb9erXMTJNw==", "license": "MIT", "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0" @@ -955,21 +983,21 @@ } }, "node_modules/periscopic/node_modules/is-reference": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/is-reference/-/is-reference-3.0.2.tgz", - "integrity": "sha512-v3rht/LgVcsdZa3O2Nqs+NMowLOxeOm7Ay9+/ARQ2F+qEoANRcqrjAZKGN0v8ymUetZGgkp26LTnGT7H0Qo9Pg==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-reference/-/is-reference-3.0.3.tgz", + "integrity": "sha512-ixkJoqQvAP88E6wLydLGGqCJsrFUnqoH6HnaczB8XmDH1oaWU+xxdptvikTgaEhtZ53Ky6YXiBuUI2WXLMCwjw==", "license": "MIT", "dependencies": { - "@types/estree": "*" + "@types/estree": "^1.0.6" } }, "node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", "license": "MIT", "engines": { - "node": ">=8.6" + "node": ">=12" }, "funding": { "url": "https://github.com/sponsors/jonschlinkert" @@ -1020,9 +1048,9 @@ } }, "node_modules/rollup": { - "version": "4.24.0", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.24.0.tgz", - "integrity": "sha512-DOmrlGSXNk1DM0ljiQA+i+o0rSLhtii1je5wgk60j49d1jHT5YYttBv1iWOnYSTG+fZZESUOSNiAl89SIet+Cg==", + "version": "4.27.4", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.27.4.tgz", + "integrity": "sha512-RLKxqHEMjh/RGLsDxAEsaLO3mWgyoU6x9w6n1ikAzet4B3gI2/3yP6PWY2p9QzRTh6MfEIXB3MwsOY0Iv3vNrw==", "devOptional": true, "license": "MIT", "dependencies": { @@ -1036,22 +1064,24 @@ "npm": ">=8.0.0" }, "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.24.0", - "@rollup/rollup-android-arm64": "4.24.0", - "@rollup/rollup-darwin-arm64": "4.24.0", - "@rollup/rollup-darwin-x64": "4.24.0", - "@rollup/rollup-linux-arm-gnueabihf": "4.24.0", - "@rollup/rollup-linux-arm-musleabihf": "4.24.0", - "@rollup/rollup-linux-arm64-gnu": "4.24.0", - "@rollup/rollup-linux-arm64-musl": "4.24.0", - "@rollup/rollup-linux-powerpc64le-gnu": "4.24.0", - "@rollup/rollup-linux-riscv64-gnu": "4.24.0", - "@rollup/rollup-linux-s390x-gnu": "4.24.0", - "@rollup/rollup-linux-x64-gnu": "4.24.0", - "@rollup/rollup-linux-x64-musl": "4.24.0", - "@rollup/rollup-win32-arm64-msvc": "4.24.0", - "@rollup/rollup-win32-ia32-msvc": "4.24.0", - "@rollup/rollup-win32-x64-msvc": "4.24.0", + "@rollup/rollup-android-arm-eabi": "4.27.4", + "@rollup/rollup-android-arm64": "4.27.4", + "@rollup/rollup-darwin-arm64": "4.27.4", + "@rollup/rollup-darwin-x64": "4.27.4", + "@rollup/rollup-freebsd-arm64": "4.27.4", + "@rollup/rollup-freebsd-x64": "4.27.4", + "@rollup/rollup-linux-arm-gnueabihf": "4.27.4", + "@rollup/rollup-linux-arm-musleabihf": "4.27.4", + "@rollup/rollup-linux-arm64-gnu": "4.27.4", + "@rollup/rollup-linux-arm64-musl": "4.27.4", + "@rollup/rollup-linux-powerpc64le-gnu": "4.27.4", + "@rollup/rollup-linux-riscv64-gnu": "4.27.4", + "@rollup/rollup-linux-s390x-gnu": "4.27.4", + "@rollup/rollup-linux-x64-gnu": "4.27.4", + "@rollup/rollup-linux-x64-musl": "4.27.4", + "@rollup/rollup-win32-arm64-msvc": "4.27.4", + "@rollup/rollup-win32-ia32-msvc": "4.27.4", + "@rollup/rollup-win32-x64-msvc": "4.27.4", "fsevents": "~2.3.2" } }, @@ -1103,6 +1133,19 @@ "node": ">= 8.0.0" } }, + "node_modules/rollup-plugin-svelte/node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -1235,12 +1278,12 @@ } }, "node_modules/svelte/node_modules/is-reference": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/is-reference/-/is-reference-3.0.2.tgz", - "integrity": "sha512-v3rht/LgVcsdZa3O2Nqs+NMowLOxeOm7Ay9+/ARQ2F+qEoANRcqrjAZKGN0v8ymUetZGgkp26LTnGT7H0Qo9Pg==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-reference/-/is-reference-3.0.3.tgz", + "integrity": "sha512-ixkJoqQvAP88E6wLydLGGqCJsrFUnqoH6HnaczB8XmDH1oaWU+xxdptvikTgaEhtZ53Ky6YXiBuUI2WXLMCwjw==", "license": "MIT", "dependencies": { - "@types/estree": "*" + "@types/estree": "^1.0.6" } }, "node_modules/terser": { diff --git a/web/frontend/package.json b/web/frontend/package.json index 1079440..389ffe6 100644 --- a/web/frontend/package.json +++ b/web/frontend/package.json @@ -11,7 +11,7 @@ "@rollup/plugin-node-resolve": "^15.3.0", "@rollup/plugin-terser": "^0.4.4", "@timohausmann/quadtree-js": "^1.2.6", - "rollup": "^4.24.0", + "rollup": "^4.27.4", "rollup-plugin-css-only": "^4.5.2", "rollup-plugin-svelte": "^7.2.2", "svelte": "^4.2.19" @@ -19,8 +19,8 @@ "dependencies": { "@rollup/plugin-replace": "^5.0.7", "@sveltestrap/sveltestrap": "^6.2.7", - "@urql/svelte": "^4.2.1", - "chart.js": "^4.4.5", + "@urql/svelte": "^4.2.2", + "chart.js": "^4.4.6", "date-fns": "^2.30.0", "graphql": "^16.9.0", "mathjs": "^12.4.3", diff --git a/web/frontend/src/Analysis.root.svelte b/web/frontend/src/Analysis.root.svelte index c6dc424..5a04465 100644 --- a/web/frontend/src/Analysis.root.svelte +++ b/web/frontend/src/Analysis.root.svelte @@ -422,14 +422,14 @@ {#if groupSelection.key == "user"} {te.id} {:else} {te.id} diff --git a/web/frontend/src/Config.root.svelte b/web/frontend/src/Config.root.svelte index 6dd68f1..dc45491 100644 --- a/web/frontend/src/Config.root.svelte +++ b/web/frontend/src/Config.root.svelte @@ -15,6 +15,7 @@ export let isAdmin; export let isApi; export let username; + export let ncontent; {#if isAdmin == true} @@ -22,7 +23,7 @@ Admin Options - + {/if} diff --git a/web/frontend/src/Job.root.svelte b/web/frontend/src/Job.root.svelte index 04e9cf9..086f25c 100644 --- a/web/frontend/src/Job.root.svelte +++ b/web/frontend/src/Job.root.svelte @@ -139,9 +139,6 @@ return names; }, []) ), - ...(ccconfig[`job_view_polarPlotMetrics:${job.cluster}`] || - ccconfig[`job_view_polarPlotMetrics`] - ), ...(ccconfig[`job_view_nodestats_selectedMetrics:${job.cluster}`] || ccconfig[`job_view_nodestats_selectedMetrics`] ), diff --git a/web/frontend/src/config.entrypoint.js b/web/frontend/src/config.entrypoint.js index 345056b..feb3916 100644 --- a/web/frontend/src/config.entrypoint.js +++ b/web/frontend/src/config.entrypoint.js @@ -6,7 +6,8 @@ new Config({ props: { isAdmin: isAdmin, isApi: isApi, - username: username + username: username, + ncontent: ncontent, }, context: new Map([ ['cc-config', clusterCockpitConfig], diff --git a/web/frontend/src/config/AdminSettings.svelte b/web/frontend/src/config/AdminSettings.svelte index 9d3abf2..f512d40 100644 --- a/web/frontend/src/config/AdminSettings.svelte +++ b/web/frontend/src/config/AdminSettings.svelte @@ -10,6 +10,9 @@ import AddUser from "./admin/AddUser.svelte"; import ShowUsers from "./admin/ShowUsers.svelte"; import Options from "./admin/Options.svelte"; + import NoticeEdit from "./admin/NoticeEdit.svelte"; + + export let ncontent; let users = []; let roles = []; @@ -52,4 +55,5 @@ + diff --git a/web/frontend/src/config/admin/NoticeEdit.svelte b/web/frontend/src/config/admin/NoticeEdit.svelte new file mode 100644 index 0000000..325800b --- /dev/null +++ b/web/frontend/src/config/admin/NoticeEdit.svelte @@ -0,0 +1,78 @@ + + + + + + + + Edit Notice Shown On Homepage +

Empty content ("No Content.") hides notice card on homepage.

+
+ + + + + +
+

+ {#if displayMessage}Update: {message.msg}{/if} +

+ + + diff --git a/web/frontend/src/generic/Filters.svelte b/web/frontend/src/generic/Filters.svelte index 9580f17..481211b 100644 --- a/web/frontend/src/generic/Filters.svelte +++ b/web/frontend/src/generic/Filters.svelte @@ -76,7 +76,7 @@ numHWThreads: filterPresets.numHWThreads || { from: null, to: null }, numAccelerators: filterPresets.numAccelerators || { from: null, to: null }, - stats: [], + stats: filterPresets.stats || [], }; let isClusterOpen = false, @@ -127,27 +127,30 @@ items.push({ jobId: { [filters.jobIdMatch]: filters.jobId } }); if (filters.arrayJobId != null) items.push({ arrayJobId: filters.arrayJobId }); - if (filters.numNodes.from != null || filters.numNodes.to != null) + if (filters.numNodes.from != null || filters.numNodes.to != null) { items.push({ numNodes: { from: filters.numNodes.from, to: filters.numNodes.to }, }); - if (filters.numHWThreads.from != null || filters.numHWThreads.to != null) + isNodesModified = true; + } + if (filters.numHWThreads.from != null || filters.numHWThreads.to != null) { items.push({ numHWThreads: { from: filters.numHWThreads.from, to: filters.numHWThreads.to, }, }); - if ( - filters.numAccelerators.from != null || - filters.numAccelerators.to != null - ) + isHwthreadsModified = true; + } + if (filters.numAccelerators.from != null || filters.numAccelerators.to != null) { items.push({ numAccelerators: { from: filters.numAccelerators.from, to: filters.numAccelerators.to, }, }); + isAccsModified = true; + } if (filters.user) items.push({ user: { [filters.userMatch]: filters.user } }); if (filters.project) @@ -197,10 +200,10 @@ opts.push(`energy=${filters.energy.from}-${filters.energy.to}`); if (filters.numNodes.from && filters.numNodes.to) opts.push(`numNodes=${filters.numNodes.from}-${filters.numNodes.to}`); + if (filters.numHWThreads.from && filters.numHWThreads.to) + opts.push(`numHWThreads=${filters.numHWThreads.from}-${filters.numHWThreads.to}`); if (filters.numAccelerators.from && filters.numAccelerators.to) - opts.push( - `numAccelerators=${filters.numAccelerators.from}-${filters.numAccelerators.to}`, - ); + opts.push(`numAccelerators=${filters.numAccelerators.from}-${filters.numAccelerators.to}`); if (filters.user.length != 0) if (filters.userMatch != "in") { opts.push(`user=${filters.user}`); @@ -214,7 +217,10 @@ if (filters.arrayJobId) opts.push(`arrayJobId=${filters.arrayJobId}`); if (filters.project && filters.projectMatch != "contains") opts.push(`projectMatch=${filters.projectMatch}`); - + if (filters.stats.length != 0) + for (let stat of filters.stats) { + opts.push(`stat=${stat.field}-${stat.from}-${stat.to}`); + } if (opts.length == 0 && window.location.search.length <= 1) return; let newurl = `${window.location.pathname}?${opts.join("&")}`; @@ -364,8 +370,7 @@ {#if (isNodesModified || isHwthreadsModified) && isAccsModified}, {/if} {#if isAccsModified} - Accelerators: {filters.numAccelerators.from} - {filters - .numAccelerators.to} + Accelerators: {filters.numAccelerators.from} - {filters.numAccelerators.to} {/if} {/if} @@ -385,7 +390,7 @@ {#if filters.stats.length > 0} (isStatsOpen = true)}> {filters.stats - .map((stat) => `${stat.text}: ${stat.from} - ${stat.to}`) + .map((stat) => `${stat.field}: ${stat.from} - ${stat.to}`) .join(", ")} {/if} diff --git a/web/frontend/src/generic/JobList.svelte b/web/frontend/src/generic/JobList.svelte index c4aacd2..89b8fad 100644 --- a/web/frontend/src/generic/JobList.svelte +++ b/web/frontend/src/generic/JobList.svelte @@ -30,6 +30,10 @@ initialized = getContext("initialized"), globalMetrics = getContext("globalMetrics"); + const equalsCheck = (a, b) => { + return JSON.stringify(a) === JSON.stringify(b); + } + export let sorting = { field: "startTime", type: "col", order: "DESC" }; export let matchedJobs = 0; export let metrics = ccconfig.plot_list_selectedMetrics; @@ -40,6 +44,8 @@ let page = 1; let paging = { itemsPerPage, page }; let filter = []; + let lastFilter = []; + let lastSorting = null; let triggerMetricRefresh = false; function getUnit(m) { @@ -105,9 +111,34 @@ variables: { paging, sorting, filter }, }); - let jobs = [] + $: if (!usePaging && sorting) { + // console.log('Reset Paging ...') + paging = { itemsPerPage: 10, page: 1 } + }; + + let jobs = []; $: if ($initialized && $jobsStore.data) { - jobs = [...$jobsStore.data.jobs.items] + if (usePaging) { + jobs = [...$jobsStore.data.jobs.items] + } else { // Prevents jump to table head in continiuous mode, only if no change in sort or filter + if (equalsCheck(filter, lastFilter) && equalsCheck(sorting, lastSorting)) { + // console.log('Both Equal: Continuous Addition ... Set None') + jobs = jobs.concat([...$jobsStore.data.jobs.items]) + } else if (equalsCheck(filter, lastFilter)) { + // console.log('Filter Equal: Continuous Reset ... Set lastSorting') + lastSorting = { ...sorting } + jobs = [...$jobsStore.data.jobs.items] + } else if (equalsCheck(sorting, lastSorting)) { + // console.log('Sorting Equal: Continuous Reset ... Set lastFilter') + lastFilter = [ ...filter ] + jobs = [...$jobsStore.data.jobs.items] + } else { + // console.log('None Equal: Continuous Reset ... Set lastBoth') + lastSorting = { ...sorting } + lastFilter = [ ...filter ] + jobs = [...$jobsStore.data.jobs.items] + } + } } $: matchedJobs = $jobsStore.data != null ? $jobsStore.data.jobs.count : -1; @@ -170,7 +201,6 @@ } if (!usePaging) { - let scrollMultiplier = 1 window.addEventListener('scroll', () => { let { scrollTop, @@ -181,8 +211,7 @@ // Add 100 px offset to trigger load earlier if (scrollTop + clientHeight >= scrollHeight - 100 && $jobsStore.data != null && $jobsStore.data.jobs.hasNextPage) { let pendingPaging = { ...paging } - scrollMultiplier += 1 - pendingPaging.itemsPerPage = itemsPerPage * scrollMultiplier + pendingPaging.page += 1 paging = pendingPaging }; }); diff --git a/web/frontend/src/generic/filters/JobStates.svelte b/web/frontend/src/generic/filters/JobStates.svelte index 9255003..d903abc 100644 --- a/web/frontend/src/generic/filters/JobStates.svelte +++ b/web/frontend/src/generic/filters/JobStates.svelte @@ -77,6 +77,13 @@ dispatch("set-filter", { states }); }}>Close & Apply + + + { displayCheck ? 'Copied!' : 'Copy Job ID to Clipboard' } + {#if job.metaData?.jobName} {#if job.metaData?.jobName.length <= 25} diff --git a/web/frontend/src/generic/joblist/JobListRow.svelte b/web/frontend/src/generic/joblist/JobListRow.svelte index 0900730..82cf2ed 100644 --- a/web/frontend/src/generic/joblist/JobListRow.svelte +++ b/web/frontend/src/generic/joblist/JobListRow.svelte @@ -37,6 +37,7 @@ : ["node"]; let selectedResolution = resampleDefault; let zoomStates = {}; + let thresholdStates = {}; const cluster = getContext("clusters").find((c) => c.name == job.cluster); const client = getContextClient(); @@ -80,6 +81,13 @@ zoomStates[metric] = {...detail.lastZoomState} } + if ( // States have to differ, causes deathloop if just set + detail?.lastThreshold && + thresholdStates[metric] !== detail.lastThreshold + ) { // Handle to correctly reset on summed metric scope change + thresholdStates[metric] = detail.lastThreshold; + } + if (detail?.newRes) { // Triggers GQL selectedResolution = detail.newRes } @@ -191,6 +199,7 @@ numhwthreads={job.numHWThreads} numaccs={job.numAcc} zoomState={zoomStates[metric.data.name] || null} + thresholdState={thresholdStates[metric.data.name] || null} /> {:else if metric.disabled == true && metric.data} 0) divisor = subClusterTopology.accelerators.length / numaccs; - else if (numhwthreads > 0) divisor = subClusterTopology.node.length / numhwthreads; + else if (numhwthreads > 0) divisor = subClusterTopology.core.length / numhwthreads; } - else if (scope == 'socket') divisor = subClusterTopology.socket.length; - else if (scope == "core") divisor = subClusterTopology.core.length; - else if (scope == "accelerator") - divisor = subClusterTopology.accelerators.length; - else if (scope == "hwthread") divisor = subClusterTopology.node.length; + else if (scope == 'node') divisor = 1; // Use as configured for nodes + else if (scope == 'socket') divisor = subClusterTopology.socket.length; + else if (scope == "memoryDomain") divisor = subClusterTopology.memoryDomain.length; + else if (scope == "core") divisor = subClusterTopology.core.length; + else if (scope == "hwthread") divisor = subClusterTopology.core.length; // alt. name for core + else if (scope == "accelerator") divisor = subClusterTopology.accelerators.length; else { - // console.log('TODO: how to calc thresholds for ', scope) - return null; + console.log('Unknown scope, return default aggregation thresholds ', scope) + divisor = 1; } return { @@ -130,6 +137,7 @@ export let numhwthreads = 0; export let numaccs = 0; export let zoomState = null; + export let thresholdState = null; if (useStatsSeries == null) useStatsSeries = statisticsSeries != null; if (useStatsSeries == false && series == null) useStatsSeries = true; @@ -149,7 +157,7 @@ caution: "rgba(255, 128, 0, 0.3)", alert: "rgba(255, 0, 0, 0.3)", }; - const thresholds = findThresholds( + const thresholds = findJobAggregationThresholds( subClusterTopology, metricConfig, scope, @@ -468,12 +476,14 @@ // console.log('Dispatch Zoom with Res from / to', timestep, closest) dispatch('zoom', { newRes: closest, - lastZoomState: u?.scales + lastZoomState: u?.scales, + lastThreshold: thresholds?.normal }); } } else { dispatch('zoom', { - lastZoomState: u?.scales + lastZoomState: u?.scales, + lastThreshold: thresholds?.normal }); }; }; @@ -498,16 +508,19 @@ let timeoutId = null; function render(ren_width, ren_height) { - if (!uplot) { // Init uPlot + if (!uplot) { opts.width = ren_width; opts.height = ren_height; - if (zoomState) { + if (zoomState && metricConfig?.aggregation == "avg") { opts.scales = {...zoomState} + } else if (zoomState && metricConfig?.aggregation == "sum") { + // Allow Zoom In === Ymin changed + if (zoomState.y.min !== 0) { // scope change?: only use zoomState if thresholds match + if ((thresholdState === thresholds?.normal)) { opts.scales = {...zoomState} }; + } // else: reset scaling to default } - // console.log('Init Sizes ...', { width: opts.width, height: opts.height }) uplot = new uPlot(opts, plotData, plotWrapper); - } else { // Update size - // console.log('Update uPlot ...', { width: ren_width, height: ren_height }) + } else { uplot.setSize({ width: ren_width, height: ren_height }); } } diff --git a/web/frontend/src/generic/plots/Polar.svelte b/web/frontend/src/generic/plots/Polar.svelte index b55e4f0..7e6957c 100644 --- a/web/frontend/src/generic/plots/Polar.svelte +++ b/web/frontend/src/generic/plots/Polar.svelte @@ -45,7 +45,7 @@ if (footprintData) { return footprintData.filter(fpd => { if (!jobMetrics.find(m => m.name == fpd.name && m.scope == "node" || fpd.impact == 4)) { - console.warn(`PolarPlot: No metric data (or config) for '${fpd.name}'`) + console.warn(`PolarPlot: No metric data for '${fpd.name}'`) return false } return true @@ -72,6 +72,7 @@ const getMetricConfig = getContext("getMetricConfig"); const getValuesForStatGeneric = (getStat) => labels.map(name => { + // TODO: Requires Scaling if Shared Job const peak = getMetricConfig(cluster, subCluster, name).peak const metric = jobMetrics.find(m => m.name == name && m.scope == "node") const value = getStat(metric.metric) / peak @@ -79,6 +80,7 @@ }) const getValuesForStatFootprint = (getStat) => labels.map(name => { + // FootprintData 'Peak' is pre-scaled for Shared Jobs in JobSummary Component const peak = footprintData.find(fpd => fpd.name === name).peak const metric = jobMetrics.find(m => m.name == name && m.scope == "node") const value = getStat(metric.metric) / peak @@ -86,14 +88,21 @@ }) function getMax(metric) { - let max = 0 + let max = metric.series[0].statistics.max; for (let series of metric.series) max = Math.max(max, series.statistics.max) return max } + function getMin(metric) { + let min = metric.series[0].statistics.min; + for (let series of metric.series) + min = Math.min(min, series.statistics.min) + return min + } + function getAvg(metric) { - let avg = 0 + let avg = 0; for (let series of metric.series) avg += series.statistics.avg return avg / metric.series.length @@ -104,6 +113,8 @@ return getValuesForStatGeneric(getAvg) } else if (type === 'max') { return getValuesForStatGeneric(getMax) + } else if (type === 'min') { + return getValuesForStatGeneric(getMin) } console.log('Unknown Type For Polar Data') return [] @@ -114,6 +125,8 @@ return getValuesForStatFootprint(getAvg) } else if (type === 'max') { return getValuesForStatFootprint(getMax) + } else if (type === 'min') { + return getValuesForStatFootprint(getMin) } console.log('Unknown Type For Polar Data') return [] @@ -124,25 +137,36 @@ datasets: [ { label: 'Max', - data: footprintData ? loadDataForFootprint('max') : loadDataGeneric('max'), // + data: footprintData ? loadDataForFootprint('max') : loadDataGeneric('max'), // Node Scope Only fill: 1, - backgroundColor: 'rgba(0, 102, 255, 0.25)', - borderColor: 'rgb(0, 102, 255)', - pointBackgroundColor: 'rgb(0, 102, 255)', + backgroundColor: 'rgba(0, 0, 255, 0.25)', + borderColor: 'rgb(0, 0, 255)', + pointBackgroundColor: 'rgb(0, 0, 255)', pointBorderColor: '#fff', pointHoverBackgroundColor: '#fff', - pointHoverBorderColor: 'rgb(0, 102, 255)' + pointHoverBorderColor: 'rgb(0, 0, 255)' }, { label: 'Avg', - data: footprintData ? loadDataForFootprint('avg') : loadDataGeneric('avg'), // getValuesForStat(getAvg) - fill: true, - backgroundColor: 'rgba(255, 153, 0, 0.25)', - borderColor: 'rgb(255, 153, 0)', - pointBackgroundColor: 'rgb(255, 153, 0)', + data: footprintData ? loadDataForFootprint('avg') : loadDataGeneric('avg'), // Node Scope Only + fill: 2, + backgroundColor: 'rgba(255, 210, 0, 0.25)', + borderColor: 'rgb(255, 210, 0)', + pointBackgroundColor: 'rgb(255, 210, 0)', pointBorderColor: '#fff', pointHoverBackgroundColor: '#fff', - pointHoverBorderColor: 'rgb(255, 153, 0)' + pointHoverBorderColor: 'rgb(255, 210, 0)' + }, + { + label: 'Min', + data: footprintData ? loadDataForFootprint('min') : loadDataGeneric('min'), // Node Scope Only + fill: true, + backgroundColor: 'rgba(255, 0, 0, 0.25)', + borderColor: 'rgb(255, 0, 0)', + pointBackgroundColor: 'rgb(255, 0, 0)', + pointBorderColor: '#fff', + pointHoverBackgroundColor: '#fff', + pointHoverBorderColor: 'rgb(255, 0, 0)' } ] } diff --git a/web/frontend/src/generic/utils.js b/web/frontend/src/generic/utils.js index 0e06c0f..d90a88d 100644 --- a/web/frontend/src/generic/utils.js +++ b/web/frontend/src/generic/utils.js @@ -6,6 +6,7 @@ import { } from "@urql/svelte"; import { setContext, getContext, hasContext, onDestroy, tick } from "svelte"; import { readable } from "svelte/store"; +import { round } from "mathjs"; /* * Call this function only at component initialization time! @@ -318,23 +319,34 @@ export function checkMetricsDisabled(ma, c, s) { // [m]etric[a]rray, [c]luster, return result } -export function getStatsItems() { +export function getStatsItems(presetStats = []) { // console.time('stats') const globalMetrics = getContext("globalMetrics") const result = globalMetrics.map((gm) => { if (gm?.footprint) { - // console.time('deep') const mc = getMetricConfigDeep(gm.name, null, null) - // console.timeEnd('deep') if (mc) { - return { - field: gm.name + '_' + gm.footprint, - text: gm.name + ' (' + gm.footprint + ')', - metric: gm.name, - from: 0, - to: mc.peak, - peak: mc.peak, - enabled: false + const presetEntry = presetStats.find((s) => s?.field === (gm.name + '_' + gm.footprint)) + if (presetEntry) { + return { + field: gm.name + '_' + gm.footprint, + text: gm.name + ' (' + gm.footprint + ')', + metric: gm.name, + from: presetEntry.from, + to: presetEntry.to, + peak: mc.peak, + enabled: true + } + } else { + return { + field: gm.name + '_' + gm.footprint, + text: gm.name + ' (' + gm.footprint + ')', + metric: gm.name, + from: 0, + to: mc.peak, + peak: mc.peak, + enabled: false + } } } } @@ -344,6 +356,38 @@ export function getStatsItems() { return [...result]; }; +export function findJobFootprintThresholds(job, stat, metricConfig) { + if (!job || !metricConfig || !stat) { + console.warn("Argument missing for findJobThresholds!"); + return null; + } + // metricConfig is on subCluster-Level + const defaultThresholds = { + peak: metricConfig.peak, + normal: metricConfig.normal, + caution: metricConfig.caution, + alert: metricConfig.alert + }; + /* + Footprints should be comparable: + Always use unchanged single node thresholds for exclusive jobs and "avg" Footprints. + For shared jobs, scale thresholds by the fraction of the job's HWThreads to the node's HWThreads. + 'stat' is one of: avg, min, max + */ + if (job.exclusive === 1 || stat === "avg") { + return defaultThresholds + } else { + const topol = getContext("getHardwareTopology")(job.cluster, job.subCluster) + const jobFraction = job.numHWThreads / topol.node.length; + return { + peak: round(defaultThresholds.peak * jobFraction, 0), + normal: round(defaultThresholds.normal * jobFraction, 0), + caution: round(defaultThresholds.caution * jobFraction, 0), + alert: round(defaultThresholds.alert * jobFraction, 0), + }; + } +} + export function getSortItems() { //console.time('sort') const globalMetrics = getContext("globalMetrics") diff --git a/web/frontend/src/job/JobSummary.svelte b/web/frontend/src/job/JobSummary.svelte index f2295f5..9954e41 100644 --- a/web/frontend/src/job/JobSummary.svelte +++ b/web/frontend/src/job/JobSummary.svelte @@ -8,44 +8,6 @@ - `height String?`: Height of the card [Default: '310px'] --> - - {{end}} \ No newline at end of file diff --git a/web/templates/login.tmpl b/web/templates/login.tmpl index f10e064..cd13926 100644 --- a/web/templates/login.tmpl +++ b/web/templates/login.tmpl @@ -41,6 +41,7 @@ {{- if .Infos.hasOpenIDConnect}} OpenID Connect Login {{end}} + diff --git a/web/web.go b/web/web.go index 45ca9e3..1cfa176 100644 --- a/web/web.go +++ b/web/web.go @@ -99,6 +99,7 @@ type Page struct { Infos map[string]interface{} // For generic use (e.g. username for /monitoring/user/, job id for /monitoring/job/) Config map[string]interface{} // UI settings for the currently logged in user (e.g. line width, ...) Resampling *schema.ResampleConfig // If not nil, defines resampling trigger and resolutions + Redirect string // The originally requested URL, for intermediate login handling } func RenderTemplate(rw http.ResponseWriter, file string, page *Page) {