diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index db99fb2..6c2fc9b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: - name: Install Go uses: actions/setup-go@v4 with: - go-version: 1.19.x + go-version: 1.20.x - name: Checkout code uses: actions/checkout@v3 - name: Build, Vet & Test diff --git a/api/swagger.json b/api/swagger.json index 6c3bc5c..7f5eaf7 100644 --- a/api/swagger.json +++ b/api/swagger.json @@ -17,6 +17,63 @@ "host": "localhost:8080", "basePath": "/api", "paths": { + "/clusters/": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Get a list of all cluster configs. Specific cluster can be requested using query parameter.", + "produces": [ + "application/json" + ], + "tags": [ + "Cluster query" + ], + "summary": "Lists all cluster configs", + "parameters": [ + { + "type": "string", + "description": "Job Cluster", + "name": "cluster", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Array of clusters", + "schema": { + "$ref": "#/definitions/api.GetClustersApiResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + } + } + } + }, "/jobs/": { "get": { "security": [ @@ -327,6 +384,76 @@ } } }, + "/jobs/edit_meta/{id}": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Edit key value pairs in job metadata json\nIf a key already exists its content will be overwritten", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Job add and modify" + ], + "summary": "Edit meta-data json", + "parameters": [ + { + "type": "integer", + "description": "Job Database ID", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "Kay value pair to add", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/api.EditMetaRequest" + } + } + ], + "responses": { + "200": { + "description": "Updated job resource", + "schema": { + "$ref": "#/definitions/schema.Job" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "404": { + "description": "Job does not exist", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + } + } + } + }, "/jobs/start_job/": { "post": { "security": [ @@ -624,6 +751,80 @@ } }, "/jobs/{id}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Job to get is specified by database ID\nReturns full job resource information according to 'JobMeta' scheme and all metrics according to 'JobData'.", + "produces": [ + "application/json" + ], + "tags": [ + "Job query" + ], + "summary": "Get job meta and optional all metric data", + "parameters": [ + { + "type": "integer", + "description": "Database ID of Job", + "name": "id", + "in": "path", + "required": true + }, + { + "type": "boolean", + "description": "Include all available metrics", + "name": "all-metrics", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Job resource", + "schema": { + "$ref": "#/definitions/api.GetJobApiResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "404": { + "description": "Resource not found", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "422": { + "description": "Unprocessable Entity: finding job failed: sql: no rows in result set", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + } + } + }, "post": { "security": [ { @@ -640,7 +841,7 @@ "tags": [ "Job query" ], - "summary": "Get complete job meta and metric data", + "summary": "Get job meta and configurable metric data", "parameters": [ { "type": "integer", @@ -1114,6 +1315,19 @@ } } }, + "api.EditMetaRequest": { + "type": "object", + "properties": { + "key": { + "type": "string", + "example": "jobScript" + }, + "value": { + "type": "string", + "example": "bash script" + } + } + }, "api.ErrorResponse": { "type": "object", "properties": { @@ -1127,6 +1341,18 @@ } } }, + "api.GetClustersApiResponse": { + "type": "object", + "properties": { + "clusters": { + "description": "Array of clusters", + "type": "array", + "items": { + "$ref": "#/definitions/schema.Cluster" + } + } + } + }, "api.GetJobApiResponse": { "type": "object", "properties": { @@ -1222,6 +1448,40 @@ } } }, + "schema.Accelerator": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "type": { + "type": "string" + } + } + }, + "schema.Cluster": { + "type": "object", + "properties": { + "metricConfig": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.MetricConfig" + } + }, + "name": { + "type": "string" + }, + "subClusters": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.SubCluster" + } + } + } + }, "schema.Job": { "description": "Information of a HPC job.", "type": "object", @@ -1252,6 +1512,10 @@ "minimum": 0, "example": 1 }, + "flopsAnyAvg": { + "description": "FlopsAnyAvg as Float64", + "type": "number" + }, "id": { "description": "The unique identifier of a job in the database", "type": "integer" @@ -1278,6 +1542,18 @@ ], "example": "completed" }, + "loadAvg": { + "description": "LoadAvg as Float64", + "type": "number" + }, + "memBwAvg": { + "description": "MemBwAvg as Float64", + "type": "number" + }, + "memUsedMax": { + "description": "MemUsedMax as Float64", + "type": "number" + }, "metaData": { "description": "Additional information about the job", "type": "object", @@ -1604,6 +1880,44 @@ } } }, + "schema.MetricConfig": { + "type": "object", + "properties": { + "aggregation": { + "type": "string" + }, + "alert": { + "type": "number" + }, + "caution": { + "type": "number" + }, + "name": { + "type": "string" + }, + "normal": { + "type": "number" + }, + "peak": { + "type": "number" + }, + "scope": { + "$ref": "#/definitions/schema.MetricScope" + }, + "subClusters": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.SubClusterConfig" + } + }, + "timestep": { + "type": "integer" + }, + "unit": { + "$ref": "#/definitions/schema.Unit" + } + } + }, "schema.MetricScope": { "type": "string", "enum": [ @@ -1639,6 +1953,17 @@ } } }, + "schema.MetricValue": { + "type": "object", + "properties": { + "unit": { + "$ref": "#/definitions/schema.Unit" + }, + "value": { + "type": "number" + } + } + }, "schema.Resource": { "description": "A resource used by a job", "type": "object", @@ -1719,6 +2044,64 @@ } } }, + "schema.SubCluster": { + "type": "object", + "properties": { + "coresPerSocket": { + "type": "integer" + }, + "flopRateScalar": { + "$ref": "#/definitions/schema.MetricValue" + }, + "flopRateSimd": { + "$ref": "#/definitions/schema.MetricValue" + }, + "memoryBandwidth": { + "$ref": "#/definitions/schema.MetricValue" + }, + "name": { + "type": "string" + }, + "nodes": { + "type": "string" + }, + "processorType": { + "type": "string" + }, + "socketsPerNode": { + "type": "integer" + }, + "threadsPerCore": { + "type": "integer" + }, + "topology": { + "$ref": "#/definitions/schema.Topology" + } + } + }, + "schema.SubClusterConfig": { + "type": "object", + "properties": { + "alert": { + "type": "number" + }, + "caution": { + "type": "number" + }, + "name": { + "type": "string" + }, + "normal": { + "type": "number" + }, + "peak": { + "type": "number" + }, + "remove": { + "type": "boolean" + } + } + }, "schema.Tag": { "description": "Defines a tag using name and type.", "type": "object", @@ -1739,6 +2122,59 @@ } } }, + "schema.Topology": { + "type": "object", + "properties": { + "accelerators": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.Accelerator" + } + }, + "core": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer" + } + } + }, + "die": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer" + } + } + }, + "memoryDomain": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer" + } + } + }, + "node": { + "type": "array", + "items": { + "type": "integer" + } + }, + "socket": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer" + } + } + } + } + }, "schema.Unit": { "type": "object", "properties": { diff --git a/api/swagger.yaml b/api/swagger.yaml index cf3b3e3..f47ac3f 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -50,6 +50,15 @@ definitions: msg: type: string type: object + api.EditMetaRequest: + properties: + key: + example: jobScript + type: string + value: + example: bash script + type: string + type: object api.ErrorResponse: properties: error: @@ -59,6 +68,14 @@ definitions: description: Statustext of Errorcode type: string type: object + api.GetClustersApiResponse: + properties: + clusters: + description: Array of clusters + items: + $ref: '#/definitions/schema.Cluster' + type: array + type: object api.GetJobApiResponse: properties: data: @@ -124,6 +141,28 @@ definitions: - jobState - stopTime type: object + schema.Accelerator: + properties: + id: + type: string + model: + type: string + type: + type: string + type: object + schema.Cluster: + properties: + metricConfig: + items: + $ref: '#/definitions/schema.MetricConfig' + type: array + name: + type: string + subClusters: + items: + $ref: '#/definitions/schema.SubCluster' + type: array + type: object schema.Job: description: Information of a HPC job. properties: @@ -150,6 +189,9 @@ definitions: maximum: 2 minimum: 0 type: integer + flopsAnyAvg: + description: FlopsAnyAvg as Float64 + type: number id: description: The unique identifier of a job in the database type: integer @@ -169,6 +211,15 @@ definitions: - timeout - out_of_memory example: completed + loadAvg: + description: LoadAvg as Float64 + type: number + memBwAvg: + description: MemBwAvg as Float64 + type: number + memUsedMax: + description: MemUsedMax as Float64 + type: number metaData: additionalProperties: type: string @@ -427,6 +478,31 @@ definitions: unit: $ref: '#/definitions/schema.Unit' type: object + schema.MetricConfig: + properties: + aggregation: + type: string + alert: + type: number + caution: + type: number + name: + type: string + normal: + type: number + peak: + type: number + scope: + $ref: '#/definitions/schema.MetricScope' + subClusters: + items: + $ref: '#/definitions/schema.SubClusterConfig' + type: array + timestep: + type: integer + unit: + $ref: '#/definitions/schema.Unit' + type: object schema.MetricScope: enum: - invalid_scope @@ -454,6 +530,13 @@ definitions: min: type: number type: object + schema.MetricValue: + properties: + unit: + $ref: '#/definitions/schema.Unit' + value: + type: number + type: object schema.Resource: description: A resource used by a job properties: @@ -508,6 +591,44 @@ definitions: type: array type: object type: object + schema.SubCluster: + properties: + coresPerSocket: + type: integer + flopRateScalar: + $ref: '#/definitions/schema.MetricValue' + flopRateSimd: + $ref: '#/definitions/schema.MetricValue' + memoryBandwidth: + $ref: '#/definitions/schema.MetricValue' + name: + type: string + nodes: + type: string + processorType: + type: string + socketsPerNode: + type: integer + threadsPerCore: + type: integer + topology: + $ref: '#/definitions/schema.Topology' + type: object + schema.SubClusterConfig: + properties: + alert: + type: number + caution: + type: number + name: + type: string + normal: + type: number + peak: + type: number + remove: + type: boolean + type: object schema.Tag: description: Defines a tag using name and type. properties: @@ -523,6 +644,41 @@ definitions: example: Debug type: string type: object + schema.Topology: + properties: + accelerators: + items: + $ref: '#/definitions/schema.Accelerator' + type: array + core: + items: + items: + type: integer + type: array + type: array + die: + items: + items: + type: integer + type: array + type: array + memoryDomain: + items: + items: + type: integer + type: array + type: array + node: + items: + type: integer + type: array + socket: + items: + items: + type: integer + type: array + type: array + type: object schema.Unit: properties: base: @@ -543,6 +699,43 @@ info: title: ClusterCockpit REST API version: 1.0.0 paths: + /clusters/: + get: + description: Get a list of all cluster configs. Specific cluster can be requested + using query parameter. + parameters: + - description: Job Cluster + in: query + name: cluster + type: string + produces: + - application/json + responses: + "200": + description: Array of clusters + schema: + $ref: '#/definitions/api.GetClustersApiResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/api.ErrorResponse' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/api.ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/api.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/api.ErrorResponse' + security: + - ApiKeyAuth: [] + summary: Lists all cluster configs + tags: + - Cluster query /jobs/: get: description: |- @@ -609,6 +802,57 @@ paths: tags: - Job query /jobs/{id}: + get: + description: |- + Job to get is specified by database ID + Returns full job resource information according to 'JobMeta' scheme and all metrics according to 'JobData'. + parameters: + - description: Database ID of Job + in: path + name: id + required: true + type: integer + - description: Include all available metrics + in: query + name: all-metrics + type: boolean + produces: + - application/json + responses: + "200": + description: Job resource + schema: + $ref: '#/definitions/api.GetJobApiResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/api.ErrorResponse' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/api.ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/api.ErrorResponse' + "404": + description: Resource not found + schema: + $ref: '#/definitions/api.ErrorResponse' + "422": + description: 'Unprocessable Entity: finding job failed: sql: no rows in + result set' + schema: + $ref: '#/definitions/api.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/api.ErrorResponse' + security: + - ApiKeyAuth: [] + summary: Get job meta and optional all metric data + tags: + - Job query post: consumes: - application/json @@ -663,7 +907,7 @@ paths: $ref: '#/definitions/api.ErrorResponse' security: - ApiKeyAuth: [] - summary: Get complete job meta and metric data + summary: Get job meta and configurable metric data tags: - Job query /jobs/delete_job/: @@ -810,6 +1054,53 @@ paths: summary: Remove a job from the sql database tags: - Job remove + /jobs/edit_meta/{id}: + post: + consumes: + - application/json + description: |- + Edit key value pairs in job metadata json + If a key already exists its content will be overwritten + parameters: + - description: Job Database ID + in: path + name: id + required: true + type: integer + - description: Kay value pair to add + in: body + name: request + required: true + schema: + $ref: '#/definitions/api.EditMetaRequest' + produces: + - application/json + responses: + "200": + description: Updated job resource + schema: + $ref: '#/definitions/schema.Job' + "400": + description: Bad Request + schema: + $ref: '#/definitions/api.ErrorResponse' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/api.ErrorResponse' + "404": + description: Job does not exist + schema: + $ref: '#/definitions/api.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/api.ErrorResponse' + security: + - ApiKeyAuth: [] + summary: Edit meta-data json + tags: + - Job add and modify /jobs/start_job/: post: consumes: diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index e956503..991fe6b 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -134,7 +134,7 @@ func initEnv() { } func main() { - var flagReinitDB, flagInit, flagServer, flagSyncLDAP, flagGops, flagMigrateDB, flagDev, flagVersion, flagLogDateTime bool + var flagReinitDB, flagInit, flagServer, flagSyncLDAP, flagGops, flagMigrateDB, flagRevertDB, flagForceDB, flagDev, flagVersion, flagLogDateTime bool var flagNewUser, flagDelUser, flagGenJWT, flagConfigFile, flagImportJob, flagLogLevel string flag.BoolVar(&flagInit, "init", false, "Setup var directory, initialize swlite database file, config.json and .env") flag.BoolVar(&flagReinitDB, "init-db", false, "Go through job-archive and re-initialize the 'job', 'tag', and 'jobtag' tables (all running jobs will be lost!)") @@ -144,6 +144,8 @@ func main() { flag.BoolVar(&flagDev, "dev", false, "Enable development components: GraphQL Playground and Swagger UI") flag.BoolVar(&flagVersion, "version", false, "Show version information and exit") flag.BoolVar(&flagMigrateDB, "migrate-db", false, "Migrate database to supported version and exit") + flag.BoolVar(&flagRevertDB, "revert-db", false, "Migrate database to previous version and exit") + flag.BoolVar(&flagForceDB, "force-db", false, "Force database version, clear dirty flag and exit") flag.BoolVar(&flagLogDateTime, "logdate", false, "Set this flag to add date and time to log messages") flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`") flag.StringVar(&flagNewUser, "add-user", "", "Add a new user. Argument format: `:[admin,support,manager,api,user]:`") @@ -205,6 +207,22 @@ func main() { os.Exit(0) } + if flagRevertDB { + err := repository.RevertDB(config.Keys.DBDriver, config.Keys.DB) + if err != nil { + log.Fatal(err) + } + os.Exit(0) + } + + if flagForceDB { + err := repository.ForceDB(config.Keys.DBDriver, config.Keys.DB) + if err != nil { + log.Fatal(err) + } + os.Exit(0) + } + repository.Connect(config.Keys.DBDriver, config.Keys.DB) db := repository.GetConnection() diff --git a/go.mod b/go.mod index facfa00..4c57c15 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,13 @@ module github.com/ClusterCockpit/cc-backend go 1.18 require ( - github.com/99designs/gqlgen v0.17.40 + github.com/99designs/gqlgen v0.17.45 github.com/ClusterCockpit/cc-units v0.4.0 github.com/Masterminds/squirrel v1.5.3 github.com/go-co-op/gocron v1.25.0 github.com/go-ldap/ldap/v3 v3.4.4 github.com/go-sql-driver/mysql v1.7.0 - github.com/golang-jwt/jwt/v4 v4.5.0 + github.com/golang-jwt/jwt/v5 v5.2.1 github.com/golang-migrate/migrate/v4 v4.15.2 github.com/google/gops v0.3.27 github.com/gorilla/handlers v1.5.1 @@ -23,9 +23,9 @@ require ( github.com/qustavo/sqlhooks/v2 v2.1.0 github.com/santhosh-tekuri/jsonschema/v5 v5.2.0 github.com/swaggo/http-swagger v1.3.3 - github.com/swaggo/swag v1.16.2 - github.com/vektah/gqlparser/v2 v2.5.10 - golang.org/x/crypto v0.16.0 + github.com/swaggo/swag v1.16.3 + github.com/vektah/gqlparser/v2 v2.5.11 + golang.org/x/crypto v0.21.0 golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea ) @@ -37,21 +37,21 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/containerd v1.6.18 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/deepmap/oapi-codegen v1.12.4 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect - github.com/go-openapi/jsonpointer v0.20.0 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/spec v0.20.9 // indirect - github.com/go-openapi/swag v0.22.4 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/spec v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/google/uuid v1.4.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/gorilla/securecookie v1.1.1 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/golang-lru/v2 v2.0.3 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect @@ -72,17 +72,17 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sosodev/duration v1.2.0 // indirect github.com/swaggo/files v1.0.0 // indirect - github.com/urfave/cli/v2 v2.25.7 // indirect - github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + github.com/urfave/cli/v2 v2.27.1 // indirect + github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect go.uber.org/atomic v1.10.0 // indirect - golang.org/x/mod v0.14.0 // indirect - golang.org/x/net v0.19.0 // indirect + golang.org/x/mod v0.16.0 // indirect + golang.org/x/net v0.22.0 // indirect golang.org/x/oauth2 v0.5.0 // indirect - golang.org/x/sys v0.15.0 // indirect + golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.16.0 // indirect + golang.org/x/tools v0.19.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect diff --git a/go.sum b/go.sum index abd7f71..86b2d68 100644 --- a/go.sum +++ b/go.sum @@ -50,8 +50,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= -github.com/99designs/gqlgen v0.17.40 h1:/l8JcEVQ93wqIfmH9VS1jsAkwm6eAF1NwQn3N+SDqBY= -github.com/99designs/gqlgen v0.17.40/go.mod h1:b62q1USk82GYIVjC60h02YguAZLqYZtvWml8KkhJps4= +github.com/99designs/gqlgen v0.17.45 h1:bH0AH67vIJo8JKNKPJP+pOPpQhZeuVRQLf53dKIpDik= +github.com/99designs/gqlgen v0.17.45/go.mod h1:Bas0XQ+Jiu/Xm5E33jC8sES3G+iC2esHBMXcq0fUPs0= github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= @@ -113,6 +113,7 @@ github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:m github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/goquery v1.9.1 h1:mTL6XjbJTZdpfL+Gwl5U2h1l9yEkJjhmlTeV9VPW7UI= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= @@ -130,6 +131,7 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= +github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/arrow v0.0.0-20210818145353-234c94e4ce64/go.mod h1:2qMFB56yOP3KzkB3PbYZ4AlUFg3a88F67TIx5lB/WwY= github.com/apache/arrow/go/arrow v0.0.0-20211013220434-5962184e7a30/go.mod h1:Q7yQnSMnLvcXlZ8RV+jwz/6y1rQTqbX6C82SndT52Zs= @@ -352,8 +354,8 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= -github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -472,28 +474,24 @@ github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+ github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= -github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/spec v0.20.9 h1:xnlYNQAwKd2VQRRfwTEI0DcK+2cbuvI/0c7jx3gA8/8= -github.com/go-openapi/spec v0.20.9/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA= +github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY= +github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk= github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= -github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= @@ -547,8 +545,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.1.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= -github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang-migrate/migrate/v4 v4.15.2 h1:vU+M05vs6jWHKDdmE1Ecwj0BznygFc4QsdRe2E/L7kc= github.com/golang-migrate/migrate/v4 v4.15.2/go.mod h1:f2toGLkYqD3JH+Todi4aZ2ZdbeUNx4sIwiOK96rE9Lw= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= @@ -643,8 +641,8 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= -github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -701,8 +699,8 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru/v2 v2.0.3 h1:kmRrRLlInXvng0SmLxmQpQkpbYAvcXm7NPDrgxJa9mE= -github.com/hashicorp/golang-lru/v2 v2.0.3/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= @@ -1065,7 +1063,7 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= @@ -1132,8 +1130,6 @@ github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -1141,18 +1137,15 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/swaggo/files v1.0.0 h1:1gGXVIeUFCS/dta17rnP0iOpr6CXFwKD7EO5ID233e4= github.com/swaggo/files v1.0.0/go.mod h1:N59U6URJLyU1PQgFqPM7wXLMhJx7QAolnvfQkqO13kc= github.com/swaggo/http-swagger v1.3.3 h1:Hu5Z0L9ssyBLofaama21iYaF2VbWyA8jdohaaCGpHsc= github.com/swaggo/http-swagger v1.3.3/go.mod h1:sE+4PjD89IxMPm77FnkDz0sdO+p5lbXzrVWT6OTVVGo= -github.com/swaggo/swag v1.16.2 h1:28Pp+8DkQoV+HLzLx8RGJZXNGKbFqnuvSbAAtoxiY04= -github.com/swaggo/swag v1.16.2/go.mod h1:6YzXnDcpr0767iOejs318CwYkCQqyGer6BizOg03f+E= +github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg= +github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -1167,10 +1160,10 @@ github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= -github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= -github.com/vektah/gqlparser/v2 v2.5.10 h1:6zSM4azXC9u4Nxy5YmdmGu4uKamfwsdKTwp5zsEealU= -github.com/vektah/gqlparser/v2 v2.5.10/go.mod h1:1rCcfwB2ekJofmluGWXMSEnPMZgbxzwj6FaZ/4OT8Cc= +github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho= +github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/vektah/gqlparser/v2 v2.5.11 h1:JJxLtXIoN7+3x6MBdtIP59TP1RANnY7pXOaDnADQSf8= +github.com/vektah/gqlparser/v2 v2.5.11/go.mod h1:1rCcfwB2ekJofmluGWXMSEnPMZgbxzwj6FaZ/4OT8Cc= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= @@ -1190,8 +1183,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1: github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= -github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw= +github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1286,8 +1279,8 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= -golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1338,8 +1331,8 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1409,8 +1402,8 @@ golang.org/x/net v0.0.0-20220111093109-d55c255bac03/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180227000427-d7d64896b5ff/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1444,7 +1437,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sys v0.0.0-20180224232135-f6cff0780e54/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1568,8 +1561,8 @@ golang.org/x/sys v0.0.0-20220317061510-51cd9980dadf/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1676,8 +1669,8 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= -golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1852,8 +1845,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/api/docs.go b/internal/api/docs.go index bf70cdb..e5ec50b 100644 --- a/internal/api/docs.go +++ b/internal/api/docs.go @@ -1,5 +1,4 @@ -// Code generated by swaggo/swag. DO NOT EDIT. - +// Package api Code generated by swaggo/swag. DO NOT EDIT package api import "github.com/swaggo/swag" @@ -24,6 +23,63 @@ const docTemplate = `{ "host": "{{.Host}}", "basePath": "{{.BasePath}}", "paths": { + "/clusters/": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Get a list of all cluster configs. Specific cluster can be requested using query parameter.", + "produces": [ + "application/json" + ], + "tags": [ + "Cluster query" + ], + "summary": "Lists all cluster configs", + "parameters": [ + { + "type": "string", + "description": "Job Cluster", + "name": "cluster", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Array of clusters", + "schema": { + "$ref": "#/definitions/api.GetClustersApiResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + } + } + } + }, "/jobs/": { "get": { "security": [ @@ -334,6 +390,76 @@ const docTemplate = `{ } } }, + "/jobs/edit_meta/{id}": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Edit key value pairs in job metadata json\nIf a key already exists its content will be overwritten", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Job add and modify" + ], + "summary": "Edit meta-data json", + "parameters": [ + { + "type": "integer", + "description": "Job Database ID", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "Kay value pair to add", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/api.EditMetaRequest" + } + } + ], + "responses": { + "200": { + "description": "Updated job resource", + "schema": { + "$ref": "#/definitions/schema.Job" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "404": { + "description": "Job does not exist", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + } + } + } + }, "/jobs/start_job/": { "post": { "security": [ @@ -631,6 +757,80 @@ const docTemplate = `{ } }, "/jobs/{id}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Job to get is specified by database ID\nReturns full job resource information according to 'JobMeta' scheme and all metrics according to 'JobData'.", + "produces": [ + "application/json" + ], + "tags": [ + "Job query" + ], + "summary": "Get job meta and optional all metric data", + "parameters": [ + { + "type": "integer", + "description": "Database ID of Job", + "name": "id", + "in": "path", + "required": true + }, + { + "type": "boolean", + "description": "Include all available metrics", + "name": "all-metrics", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Job resource", + "schema": { + "$ref": "#/definitions/api.GetJobApiResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "404": { + "description": "Resource not found", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "422": { + "description": "Unprocessable Entity: finding job failed: sql: no rows in result set", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + } + } + }, "post": { "security": [ { @@ -647,7 +847,7 @@ const docTemplate = `{ "tags": [ "Job query" ], - "summary": "Get complete job meta and metric data", + "summary": "Get job meta and configurable metric data", "parameters": [ { "type": "integer", @@ -1121,6 +1321,19 @@ const docTemplate = `{ } } }, + "api.EditMetaRequest": { + "type": "object", + "properties": { + "key": { + "type": "string", + "example": "jobScript" + }, + "value": { + "type": "string", + "example": "bash script" + } + } + }, "api.ErrorResponse": { "type": "object", "properties": { @@ -1134,6 +1347,18 @@ const docTemplate = `{ } } }, + "api.GetClustersApiResponse": { + "type": "object", + "properties": { + "clusters": { + "description": "Array of clusters", + "type": "array", + "items": { + "$ref": "#/definitions/schema.Cluster" + } + } + } + }, "api.GetJobApiResponse": { "type": "object", "properties": { @@ -1229,6 +1454,40 @@ const docTemplate = `{ } } }, + "schema.Accelerator": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "type": { + "type": "string" + } + } + }, + "schema.Cluster": { + "type": "object", + "properties": { + "metricConfig": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.MetricConfig" + } + }, + "name": { + "type": "string" + }, + "subClusters": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.SubCluster" + } + } + } + }, "schema.Job": { "description": "Information of a HPC job.", "type": "object", @@ -1259,6 +1518,10 @@ const docTemplate = `{ "minimum": 0, "example": 1 }, + "flopsAnyAvg": { + "description": "FlopsAnyAvg as Float64", + "type": "number" + }, "id": { "description": "The unique identifier of a job in the database", "type": "integer" @@ -1285,6 +1548,18 @@ const docTemplate = `{ ], "example": "completed" }, + "loadAvg": { + "description": "LoadAvg as Float64", + "type": "number" + }, + "memBwAvg": { + "description": "MemBwAvg as Float64", + "type": "number" + }, + "memUsedMax": { + "description": "MemUsedMax as Float64", + "type": "number" + }, "metaData": { "description": "Additional information about the job", "type": "object", @@ -1611,6 +1886,44 @@ const docTemplate = `{ } } }, + "schema.MetricConfig": { + "type": "object", + "properties": { + "aggregation": { + "type": "string" + }, + "alert": { + "type": "number" + }, + "caution": { + "type": "number" + }, + "name": { + "type": "string" + }, + "normal": { + "type": "number" + }, + "peak": { + "type": "number" + }, + "scope": { + "$ref": "#/definitions/schema.MetricScope" + }, + "subClusters": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.SubClusterConfig" + } + }, + "timestep": { + "type": "integer" + }, + "unit": { + "$ref": "#/definitions/schema.Unit" + } + } + }, "schema.MetricScope": { "type": "string", "enum": [ @@ -1646,6 +1959,17 @@ const docTemplate = `{ } } }, + "schema.MetricValue": { + "type": "object", + "properties": { + "unit": { + "$ref": "#/definitions/schema.Unit" + }, + "value": { + "type": "number" + } + } + }, "schema.Resource": { "description": "A resource used by a job", "type": "object", @@ -1726,6 +2050,64 @@ const docTemplate = `{ } } }, + "schema.SubCluster": { + "type": "object", + "properties": { + "coresPerSocket": { + "type": "integer" + }, + "flopRateScalar": { + "$ref": "#/definitions/schema.MetricValue" + }, + "flopRateSimd": { + "$ref": "#/definitions/schema.MetricValue" + }, + "memoryBandwidth": { + "$ref": "#/definitions/schema.MetricValue" + }, + "name": { + "type": "string" + }, + "nodes": { + "type": "string" + }, + "processorType": { + "type": "string" + }, + "socketsPerNode": { + "type": "integer" + }, + "threadsPerCore": { + "type": "integer" + }, + "topology": { + "$ref": "#/definitions/schema.Topology" + } + } + }, + "schema.SubClusterConfig": { + "type": "object", + "properties": { + "alert": { + "type": "number" + }, + "caution": { + "type": "number" + }, + "name": { + "type": "string" + }, + "normal": { + "type": "number" + }, + "peak": { + "type": "number" + }, + "remove": { + "type": "boolean" + } + } + }, "schema.Tag": { "description": "Defines a tag using name and type.", "type": "object", @@ -1746,6 +2128,59 @@ const docTemplate = `{ } } }, + "schema.Topology": { + "type": "object", + "properties": { + "accelerators": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.Accelerator" + } + }, + "core": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer" + } + } + }, + "die": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer" + } + } + }, + "memoryDomain": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer" + } + } + }, + "node": { + "type": "array", + "items": { + "type": "integer" + } + }, + "socket": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer" + } + } + } + } + }, "schema.Unit": { "type": "object", "properties": { diff --git a/internal/api/rest.go b/internal/api/rest.go index 11682ee..dadabcd 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -70,12 +70,16 @@ func (api *RestApi) MountRoutes(r *mux.Router) { r.HandleFunc("/jobs/", api.getJobs).Methods(http.MethodGet) r.HandleFunc("/jobs/{id}", api.getJobById).Methods(http.MethodPost) + r.HandleFunc("/jobs/{id}", api.getCompleteJobById).Methods(http.MethodGet) r.HandleFunc("/jobs/tag_job/{id}", api.tagJob).Methods(http.MethodPost, http.MethodPatch) + r.HandleFunc("/jobs/edit_meta/{id}", api.editMeta).Methods(http.MethodPost, http.MethodPatch) r.HandleFunc("/jobs/metrics/{id}", api.getJobMetrics).Methods(http.MethodGet) r.HandleFunc("/jobs/delete_job/", api.deleteJobByRequest).Methods(http.MethodDelete) r.HandleFunc("/jobs/delete_job/{id}", api.deleteJobById).Methods(http.MethodDelete) r.HandleFunc("/jobs/delete_job_before/{ts}", api.deleteJobBefore).Methods(http.MethodDelete) + r.HandleFunc("/clusters/", api.getClusters).Methods(http.MethodGet) + if api.MachineStateDir != "" { r.HandleFunc("/machine_state/{cluster}/{host}", api.getMachineState).Methods(http.MethodGet) r.HandleFunc("/machine_state/{cluster}/{host}", api.putMachineState).Methods(http.MethodPut, http.MethodPost) @@ -110,12 +114,11 @@ type UpdateUserApiResponse struct { // StopJobApiRequest model type StopJobApiRequest struct { - // Stop Time of job as epoch + JobId *int64 `json:"jobId" example:"123000"` + Cluster *string `json:"cluster" example:"fritz"` + StartTime *int64 `json:"startTime" example:"1649723812"` + State schema.JobState `json:"jobState" validate:"required" example:"completed"` StopTime int64 `json:"stopTime" validate:"required" example:"1649763839"` - State schema.JobState `json:"jobState" validate:"required" example:"completed"` // Final job state - JobId *int64 `json:"jobId" example:"123000"` // Cluster Job ID of job - Cluster *string `json:"cluster" example:"fritz"` // Cluster of job - StartTime *int64 `json:"startTime" example:"1649723812"` // Start Time of job as epoch } // DeleteJobApiRequest model @@ -132,6 +135,11 @@ type GetJobsApiResponse struct { Page int `json:"page"` // Page id returned } +// GetClustersApiResponse model +type GetClustersApiResponse struct { + Clusters []*schema.Cluster `json:"clusters"` // Array of clusters +} + // ErrorResponse model type ErrorResponse struct { // Statustext of Errorcode @@ -146,6 +154,12 @@ type ApiTag struct { Name string `json:"name" example:"Testjob"` // Tag Name } +// ApiMeta model +type EditMetaRequest struct { + Key string `json:"key" example:"jobScript"` + Value string `json:"value" example:"bash script"` +} + type TagJobApiRequest []*ApiTag type GetJobApiRequest []string @@ -155,10 +169,15 @@ type GetJobApiResponse struct { Data []*JobMetricWithName } +type GetCompleteJobApiResponse struct { + Meta *schema.Job + Data schema.JobData +} + type JobMetricWithName struct { + Metric *schema.JobMetric `json:"metric"` Name string `json:"name"` Scope schema.MetricScope `json:"scope"` - Metric *schema.JobMetric `json:"metric"` } type ApiReturnedUser struct { @@ -223,6 +242,55 @@ func securedCheck(r *http.Request) error { return nil } +// getClusters godoc +// @summary Lists all cluster configs +// @tags Cluster query +// @description Get a list of all cluster configs. Specific cluster can be requested using query parameter. +// @produce json +// @param cluster query string false "Job Cluster" +// @success 200 {object} api.GetClustersApiResponse "Array of clusters" +// @failure 400 {object} api.ErrorResponse "Bad Request" +// @failure 401 {object} api.ErrorResponse "Unauthorized" +// @failure 403 {object} api.ErrorResponse "Forbidden" +// @failure 500 {object} api.ErrorResponse "Internal Server Error" +// @security ApiKeyAuth +// @router /clusters/ [get] +func (api *RestApi) getClusters(rw http.ResponseWriter, r *http.Request) { + if user := repository.GetUserFromContext(r.Context()); user != nil && + !user.HasRole(schema.RoleApi) { + + handleError(fmt.Errorf("missing role: %v", schema.GetRoleString(schema.RoleApi)), http.StatusForbidden, rw) + return + } + + rw.Header().Add("Content-Type", "application/json") + bw := bufio.NewWriter(rw) + defer bw.Flush() + + var clusters []*schema.Cluster + + if r.URL.Query().Has("cluster") { + name := r.URL.Query().Get("cluster") + cluster := archive.GetCluster(name) + if cluster == nil { + handleError(fmt.Errorf("unknown cluster: %s", name), http.StatusBadRequest, rw) + return + } + clusters = append(clusters, cluster) + } else { + clusters = archive.Clusters + } + + payload := GetClustersApiResponse{ + Clusters: clusters, + } + + if err := json.NewEncoder(bw).Encode(payload); err != nil { + handleError(err, http.StatusInternalServerError, rw) + return + } +} + // getJobs godoc // @summary Lists all jobs // @tags Job query @@ -243,7 +311,6 @@ func securedCheck(r *http.Request) error { // @security ApiKeyAuth // @router /jobs/ [get] func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { - if user := repository.GetUserFromContext(r.Context()); user != nil && !user.HasRole(schema.RoleApi) { @@ -342,10 +409,8 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { if res.MonitoringStatus == schema.MonitoringStatusArchivingSuccessful { res.Statistics, err = archive.GetStatistics(job) if err != nil { - if err != nil { - handleError(err, http.StatusInternalServerError, rw) - return - } + handleError(err, http.StatusInternalServerError, rw) + return } } @@ -370,14 +435,107 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { } // getJobById godoc -// @summary Get complete job meta and metric data +// @summary Get job meta and optional all metric data +// @tags Job query +// @description Job to get is specified by database ID +// @description Returns full job resource information according to 'JobMeta' scheme and all metrics according to 'JobData'. +// @produce json +// @param id path int true "Database ID of Job" +// @param all-metrics query bool false "Include all available metrics" +// @success 200 {object} api.GetJobApiResponse "Job resource" +// @failure 400 {object} api.ErrorResponse "Bad Request" +// @failure 401 {object} api.ErrorResponse "Unauthorized" +// @failure 403 {object} api.ErrorResponse "Forbidden" +// @failure 404 {object} api.ErrorResponse "Resource not found" +// @failure 422 {object} api.ErrorResponse "Unprocessable Entity: finding job failed: sql: no rows in result set" +// @failure 500 {object} api.ErrorResponse "Internal Server Error" +// @security ApiKeyAuth +// @router /jobs/{id} [get] +func (api *RestApi) getCompleteJobById(rw http.ResponseWriter, r *http.Request) { + if user := repository.GetUserFromContext(r.Context()); user != nil && + !user.HasRole(schema.RoleApi) { + + handleError(fmt.Errorf("missing role: %v", + schema.GetRoleString(schema.RoleApi)), http.StatusForbidden, rw) + return + } + + // Fetch job from db + id, ok := mux.Vars(r)["id"] + var job *schema.Job + var err error + if ok { + id, e := strconv.ParseInt(id, 10, 64) + if e != nil { + handleError(fmt.Errorf("integer expected in path for id: %w", e), http.StatusBadRequest, rw) + return + } + + job, err = api.JobRepository.FindById(id) + } else { + handleError(errors.New("the parameter 'id' is required"), http.StatusBadRequest, rw) + return + } + if err != nil { + handleError(fmt.Errorf("finding job failed: %w", err), http.StatusUnprocessableEntity, rw) + return + } + + job.Tags, err = api.JobRepository.GetTags(&job.ID) + if err != nil { + handleError(err, http.StatusInternalServerError, rw) + return + + } + if _, err = api.JobRepository.FetchMetadata(job); err != nil { + + handleError(err, http.StatusInternalServerError, rw) + return + } + + var scopes []schema.MetricScope + + if job.NumNodes == 1 { + scopes = []schema.MetricScope{"core"} + } else { + scopes = []schema.MetricScope{"node"} + } + + var data schema.JobData + + if r.URL.Query().Get("all-metrics") == "true" { + data, err = metricdata.LoadData(job, nil, scopes, r.Context()) + if err != nil { + log.Warn("Error while loading job data") + return + } + } + + log.Debugf("/api/job/%s: get job %d", id, job.JobID) + rw.Header().Add("Content-Type", "application/json") + bw := bufio.NewWriter(rw) + defer bw.Flush() + + payload := GetCompleteJobApiResponse{ + Meta: job, + Data: data, + } + + if err := json.NewEncoder(bw).Encode(payload); err != nil { + handleError(err, http.StatusInternalServerError, rw) + return + } +} + +// getJobById godoc +// @summary Get job meta and configurable metric data // @tags Job query // @description Job to get is specified by database ID // @description Returns full job resource information according to 'JobMeta' scheme and all metrics according to 'JobData'. // @accept json // @produce json -// @param id path int true "Database ID of Job" -// @param request body api.GetJobApiRequest true "Array of metric names" +// @param id path int true "Database ID of Job" +// @param request body api.GetJobApiRequest true "Array of metric names" // @success 200 {object} api.GetJobApiResponse "Job resource" // @failure 400 {object} api.ErrorResponse "Bad Request" // @failure 401 {object} api.ErrorResponse "Unauthorized" @@ -417,6 +575,18 @@ func (api *RestApi) getJobById(rw http.ResponseWriter, r *http.Request) { return } + job.Tags, err = api.JobRepository.GetTags(&job.ID) + if err != nil { + handleError(err, http.StatusInternalServerError, rw) + return + + } + if _, err = api.JobRepository.FetchMetadata(job); err != nil { + + handleError(err, http.StatusInternalServerError, rw) + return + } + var metrics GetJobApiRequest if err = decode(r.Body, &metrics); err != nil { http.Error(rw, err.Error(), http.StatusBadRequest) @@ -464,6 +634,57 @@ func (api *RestApi) getJobById(rw http.ResponseWriter, r *http.Request) { } } +// editMeta godoc +// @summary Edit meta-data json +// @tags Job add and modify +// @description Edit key value pairs in job metadata json +// @description If a key already exists its content will be overwritten +// @accept json +// @produce json +// @param id path int true "Job Database ID" +// @param request body api.EditMetaRequest true "Kay value pair to add" +// @success 200 {object} schema.Job "Updated job resource" +// @failure 400 {object} api.ErrorResponse "Bad Request" +// @failure 401 {object} api.ErrorResponse "Unauthorized" +// @failure 404 {object} api.ErrorResponse "Job does not exist" +// @failure 500 {object} api.ErrorResponse "Internal Server Error" +// @security ApiKeyAuth +// @router /jobs/edit_meta/{id} [post] +func (api *RestApi) editMeta(rw http.ResponseWriter, r *http.Request) { + if user := repository.GetUserFromContext(r.Context()); user != nil && + !user.HasRole(schema.RoleApi) { + handleError(fmt.Errorf("missing role: %v", schema.GetRoleString(schema.RoleApi)), http.StatusForbidden, rw) + return + } + + iid, err := strconv.ParseInt(mux.Vars(r)["id"], 10, 64) + if err != nil { + http.Error(rw, err.Error(), http.StatusBadRequest) + return + } + + job, err := api.JobRepository.FindById(iid) + if err != nil { + http.Error(rw, err.Error(), http.StatusNotFound) + return + } + + var req EditMetaRequest + if err := decode(r.Body, &req); err != nil { + http.Error(rw, err.Error(), http.StatusBadRequest) + return + } + + if err := api.JobRepository.UpdateMetadata(job, req.Key, req.Value); err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) + return + } + + rw.Header().Add("Content-Type", "application/json") + rw.WriteHeader(http.StatusOK) + json.NewEncoder(rw).Encode(job) +} + // tagJob godoc // @summary Adds one or more tags to a job // @tags Job add and modify @@ -873,7 +1094,6 @@ func (api *RestApi) deleteJobBefore(rw http.ResponseWriter, r *http.Request) { } func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Job, req StopJobApiRequest) { - // Sanity checks if job == nil || job.StartTime.Unix() >= req.StopTime || job.State != schema.JobStateRunning { handleError(errors.New("stopTime must be larger than startTime and only running jobs can be stopped"), http.StatusBadRequest, rw) @@ -1015,12 +1235,13 @@ func (api *RestApi) createUser(rw http.ResponseWriter, r *http.Request) { Password: password, Email: email, Projects: []string{project}, - Roles: []string{role}}); err != nil { + Roles: []string{role}, + }); err != nil { http.Error(rw, err.Error(), http.StatusUnprocessableEntity) return } - rw.Write([]byte(fmt.Sprintf("User %v successfully created!\n", username))) + fmt.Fprintf(rw, "User %v successfully created!\n", username) } // deleteUser godoc diff --git a/internal/auth/jwt.go b/internal/auth/jwt.go index 83bfee3..c1af752 100644 --- a/internal/auth/jwt.go +++ b/internal/auth/jwt.go @@ -17,7 +17,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/log" "github.com/ClusterCockpit/cc-backend/pkg/schema" - "github.com/golang-jwt/jwt/v4" + "github.com/golang-jwt/jwt/v5" ) type JWTAuthenticator struct { @@ -49,8 +49,8 @@ func (ja *JWTAuthenticator) Init() error { func (ja *JWTAuthenticator) AuthViaJWT( rw http.ResponseWriter, - r *http.Request) (*schema.User, error) { - + r *http.Request, +) (*schema.User, error) { rawtoken := r.Header.Get("X-Auth-Token") if rawtoken == "" { rawtoken = r.Header.Get("Authorization") @@ -73,9 +73,9 @@ func (ja *JWTAuthenticator) AuthViaJWT( log.Warn("Error while parsing JWT token") return nil, err } - if err := token.Claims.Valid(); err != nil { + if !token.Valid { log.Warn("jwt token claims are not valid") - return nil, err + return nil, errors.New("jwt token claims are not valid") } // Token is valid, extract payload @@ -88,7 +88,6 @@ func (ja *JWTAuthenticator) AuthViaJWT( if config.Keys.JwtConfig.ValidateUser { ur := repository.GetUserRepository() user, err := ur.GetUser(sub) - // Deny any logins for unknown usernames if err != nil { log.Warn("Could not find user from JWT in internal database.") @@ -117,7 +116,6 @@ func (ja *JWTAuthenticator) AuthViaJWT( // Generate a new JWT that can be used for authentication func (ja *JWTAuthenticator) ProvideJWT(user *schema.User) (string, error) { - if ja.privateKey == nil { return "", errors.New("environment variable 'JWT_PRIVATE_KEY' not set") } diff --git a/internal/auth/jwtCookieSession.go b/internal/auth/jwtCookieSession.go index 3748836..01f5746 100644 --- a/internal/auth/jwtCookieSession.go +++ b/internal/auth/jwtCookieSession.go @@ -17,7 +17,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/log" "github.com/ClusterCockpit/cc-backend/pkg/schema" - "github.com/golang-jwt/jwt/v4" + "github.com/golang-jwt/jwt/v5" ) type JWTCookieSessionAuthenticator struct { @@ -90,8 +90,8 @@ func (ja *JWTCookieSessionAuthenticator) CanLogin( user *schema.User, username string, rw http.ResponseWriter, - r *http.Request) (*schema.User, bool) { - + r *http.Request, +) (*schema.User, bool) { jc := config.Keys.JwtConfig cookieName := "" if jc.CookieName != "" { @@ -113,8 +113,8 @@ func (ja *JWTCookieSessionAuthenticator) CanLogin( func (ja *JWTCookieSessionAuthenticator) Login( user *schema.User, rw http.ResponseWriter, - r *http.Request) (*schema.User, error) { - + r *http.Request, +) (*schema.User, error) { jc := config.Keys.JwtConfig jwtCookie, err := r.Cookie(jc.CookieName) var rawtoken string @@ -144,10 +144,9 @@ func (ja *JWTCookieSessionAuthenticator) Login( return nil, err } - // Check token validity and extract paypload - if err := token.Claims.Valid(); err != nil { + if !token.Valid { log.Warn("jwt token claims are not valid") - return nil, err + return nil, errors.New("jwt token claims are not valid") } claims := token.Claims.(jwt.MapClaims) diff --git a/internal/auth/jwtSession.go b/internal/auth/jwtSession.go index 286bb82..541e31e 100644 --- a/internal/auth/jwtSession.go +++ b/internal/auth/jwtSession.go @@ -17,7 +17,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/log" "github.com/ClusterCockpit/cc-backend/pkg/schema" - "github.com/golang-jwt/jwt/v4" + "github.com/golang-jwt/jwt/v5" ) type JWTSessionAuthenticator struct { @@ -44,8 +44,8 @@ func (ja *JWTSessionAuthenticator) CanLogin( user *schema.User, username string, rw http.ResponseWriter, - r *http.Request) (*schema.User, bool) { - + r *http.Request, +) (*schema.User, bool) { return user, r.Header.Get("Authorization") != "" || r.URL.Query().Get("login-token") != "" } @@ -53,8 +53,8 @@ func (ja *JWTSessionAuthenticator) CanLogin( func (ja *JWTSessionAuthenticator) Login( user *schema.User, rw http.ResponseWriter, - r *http.Request) (*schema.User, error) { - + r *http.Request, +) (*schema.User, error) { rawtoken := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ") if rawtoken == "" { rawtoken = r.URL.Query().Get("login-token") @@ -71,9 +71,9 @@ func (ja *JWTSessionAuthenticator) Login( return nil, err } - if err = token.Claims.Valid(); err != nil { + if !token.Valid { log.Warn("jwt token claims are not valid") - return nil, err + return nil, errors.New("jwt token claims are not valid") } claims := token.Claims.(jwt.MapClaims) diff --git a/internal/metricdata/cc-metric-store.go b/internal/metricdata/cc-metric-store.go index 4e69e47..9038af4 100644 --- a/internal/metricdata/cc-metric-store.go +++ b/internal/metricdata/cc-metric-store.go @@ -32,32 +32,32 @@ type CCMetricStoreConfig struct { } type CCMetricStore struct { + here2there map[string]string + there2here map[string]string + client http.Client jwt string url string queryEndpoint string - client http.Client - here2there map[string]string - there2here map[string]string } type ApiQueryRequest struct { Cluster string `json:"cluster"` + Queries []ApiQuery `json:"queries"` + ForAllNodes []string `json:"for-all-nodes"` From int64 `json:"from"` To int64 `json:"to"` WithStats bool `json:"with-stats"` WithData bool `json:"with-data"` - Queries []ApiQuery `json:"queries"` - ForAllNodes []string `json:"for-all-nodes"` } type ApiQuery struct { + Type *string `json:"type,omitempty"` + SubType *string `json:"subtype,omitempty"` Metric string `json:"metric"` Hostname string `json:"host"` - Aggregate bool `json:"aggreg"` - Type *string `json:"type,omitempty"` TypeIds []string `json:"type-ids,omitempty"` - SubType *string `json:"subtype,omitempty"` SubTypeIds []string `json:"subtype-ids,omitempty"` + Aggregate bool `json:"aggreg"` } type ApiQueryResponse struct { @@ -67,16 +67,15 @@ type ApiQueryResponse struct { type ApiMetricData struct { Error *string `json:"error"` + Data []schema.Float `json:"data"` From int64 `json:"from"` To int64 `json:"to"` - Data []schema.Float `json:"data"` Avg schema.Float `json:"avg"` Min schema.Float `json:"min"` Max schema.Float `json:"max"` } func (ccms *CCMetricStore) Init(rawConfig json.RawMessage) error { - var config CCMetricStoreConfig if err := json.Unmarshal(rawConfig, &config); err != nil { log.Warn("Error while unmarshaling raw json config") @@ -122,8 +121,8 @@ func (ccms *CCMetricStore) toLocalName(metric string) string { func (ccms *CCMetricStore) doRequest( ctx context.Context, - body *ApiQueryRequest) (*ApiQueryResponse, error) { - + body *ApiQueryRequest, +) (*ApiQueryResponse, error) { buf := &bytes.Buffer{} if err := json.NewEncoder(buf).Encode(body); err != nil { log.Warn("Error while encoding request body") @@ -162,8 +161,8 @@ func (ccms *CCMetricStore) LoadData( job *schema.Job, metrics []string, scopes []schema.MetricScope, - ctx context.Context) (schema.JobData, error) { - + ctx context.Context, +) (schema.JobData, error) { queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes) if err != nil { log.Warn("Error while building queries") @@ -186,7 +185,7 @@ func (ccms *CCMetricStore) LoadData( } var errors []string - var jobData schema.JobData = make(schema.JobData) + jobData := make(schema.JobData) for i, row := range resBody.Results { query := req.Queries[i] metric := ccms.toLocalName(query.Metric) @@ -267,8 +266,8 @@ var ( func (ccms *CCMetricStore) buildQueries( job *schema.Job, metrics []string, - scopes []schema.MetricScope) ([]ApiQuery, []schema.MetricScope, error) { - + scopes []schema.MetricScope, +) ([]ApiQuery, []schema.MetricScope, error) { queries := make([]ApiQuery, 0, len(metrics)*len(scopes)*len(job.Resources)) assignedScope := []schema.MetricScope{} @@ -509,8 +508,8 @@ func (ccms *CCMetricStore) buildQueries( func (ccms *CCMetricStore) LoadStats( job *schema.Job, metrics []string, - ctx context.Context) (map[string]map[string]schema.MetricStatistics, error) { - + ctx context.Context, +) (map[string]map[string]schema.MetricStatistics, error) { queries, _, err := ccms.buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}) // #166 Add scope shere for analysis view accelerator normalization? if err != nil { log.Warn("Error while building query") @@ -571,8 +570,8 @@ func (ccms *CCMetricStore) LoadNodeData( metrics, nodes []string, scopes []schema.MetricScope, from, to time.Time, - ctx context.Context) (map[string]map[string][]*schema.JobMetric, error) { - + ctx context.Context, +) (map[string]map[string][]*schema.JobMetric, error) { req := ApiQueryRequest{ Cluster: cluster, From: from.Unix(), @@ -657,7 +656,6 @@ func (ccms *CCMetricStore) LoadNodeData( } func intToStringSlice(is []int) []string { - ss := make([]string, len(is)) for i, x := range is { ss[i] = strconv.Itoa(x) diff --git a/internal/repository/job.go b/internal/repository/job.go index e1a997a..329c0ba 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -16,6 +16,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/graph/model" "github.com/ClusterCockpit/cc-backend/internal/metricdata" + "github.com/ClusterCockpit/cc-backend/pkg/archive" "github.com/ClusterCockpit/cc-backend/pkg/log" "github.com/ClusterCockpit/cc-backend/pkg/lrucache" "github.com/ClusterCockpit/cc-backend/pkg/schema" @@ -29,13 +30,11 @@ var ( ) type JobRepository struct { - DB *sqlx.DB - driver string - - stmtCache *sq.StmtCache - cache *lrucache.Cache - + DB *sqlx.DB + stmtCache *sq.StmtCache + cache *lrucache.Cache archiveChannel chan *schema.Job + driver string archivePending sync.WaitGroup } @@ -212,7 +211,7 @@ func (r *JobRepository) UpdateMetadata(job *schema.Job, key, val string) (err er } r.cache.Put(cachekey, job.MetaData, len(job.RawMetaData), 24*time.Hour) - return nil + return archive.UpdateMetadata(job, job.MetaData) } // Find executes a SQL query to find a specific batch job. @@ -223,8 +222,8 @@ func (r *JobRepository) UpdateMetadata(job *schema.Job, key, val string) (err er func (r *JobRepository) Find( jobId *int64, cluster *string, - startTime *int64) (*schema.Job, error) { - + startTime *int64, +) (*schema.Job, error) { start := time.Now() q := sq.Select(jobColumns...).From("job"). Where("job.job_id = ?", *jobId) @@ -248,8 +247,8 @@ func (r *JobRepository) Find( func (r *JobRepository) FindAll( jobId *int64, cluster *string, - startTime *int64) ([]*schema.Job, error) { - + startTime *int64, +) ([]*schema.Job, error) { start := time.Now() q := sq.Select(jobColumns...).From("job"). Where("job.job_id = ?", *jobId) @@ -292,7 +291,8 @@ func (r *JobRepository) FindById(jobId int64) (*schema.Job, error) { func (r *JobRepository) FindConcurrentJobs( ctx context.Context, - job *schema.Job) (*model.JobLinkResultList, error) { + job *schema.Job, +) (*model.JobLinkResultList, error) { if job == nil { return nil, nil } @@ -420,8 +420,8 @@ func (r *JobRepository) Stop( jobId int64, duration int32, state schema.JobState, - monitoringStatus int32) (err error) { - + monitoringStatus int32, +) (err error) { stmt := sq.Update("job"). Set("job_state", state). Set("duration", duration). @@ -434,11 +434,14 @@ func (r *JobRepository) Stop( func (r *JobRepository) DeleteJobsBefore(startTime int64) (int, error) { var cnt int - qs := fmt.Sprintf("SELECT count(*) FROM job WHERE job.start_time < %d", startTime) - err := r.DB.Get(&cnt, qs) //ignore error as it will also occur in delete statement - _, err = r.DB.Exec(`DELETE FROM job WHERE job.start_time < ?`, startTime) + q := sq.Select("count(*)").From("job").Where("job.start_time < ?", startTime) + q.RunWith(r.DB).QueryRow().Scan(cnt) + qd := sq.Delete("job").Where("job.start_time < ?", startTime) + _, err := qd.RunWith(r.DB).Exec() + if err != nil { - log.Errorf(" DeleteJobsBefore(%d): error %#v", startTime, err) + s, _, _ := qd.ToSql() + log.Errorf(" DeleteJobsBefore(%d) with %s: error %#v", startTime, s, err) } else { log.Debugf("DeleteJobsBefore(%d): Deleted %d jobs", startTime, cnt) } @@ -446,9 +449,12 @@ func (r *JobRepository) DeleteJobsBefore(startTime int64) (int, error) { } func (r *JobRepository) DeleteJobById(id int64) error { - _, err := r.DB.Exec(`DELETE FROM job WHERE job.id = ?`, id) + qd := sq.Delete("job").Where("job.id = ?", id) + _, err := qd.RunWith(r.DB).Exec() + if err != nil { - log.Errorf("DeleteJobById(%d): error %#v", id, err) + s, _, _ := qd.ToSql() + log.Errorf("DeleteJobById(%d) with %s : error %#v", id, s, err) } else { log.Debugf("DeleteJobById(%d): Success", id) } @@ -468,8 +474,8 @@ func (r *JobRepository) UpdateMonitoringStatus(job int64, monitoringStatus int32 func (r *JobRepository) MarkArchived( jobId int64, monitoringStatus int32, - metricStats map[string]schema.JobStatistics) error { - + metricStats map[string]schema.JobStatistics, +) error { stmt := sq.Update("job"). Set("monitoring_status", monitoringStatus). Where("job.id = ?", jobId) @@ -578,8 +584,10 @@ func (r *JobRepository) FindUserOrProjectOrJobname(user *schema.User, searchterm } } -var ErrNotFound = errors.New("no such jobname, project or user") -var ErrForbidden = errors.New("not authorized") +var ( + ErrNotFound = errors.New("no such jobname, project or user") + ErrForbidden = errors.New("not authorized") +) func (r *JobRepository) FindColumnValue(user *schema.User, searchterm string, table string, selectColumn string, whereColumn string, isLike bool) (result string, err error) { compareStr := " = ?" @@ -663,7 +671,6 @@ func (r *JobRepository) Partitions(cluster string) ([]string, error) { // AllocatedNodes returns a map of all subclusters to a map of hostnames to the amount of jobs running on that host. // Hosts with zero jobs running on them will not show up! func (r *JobRepository) AllocatedNodes(cluster string) (map[string]map[string]int, error) { - start := time.Now() subclusters := make(map[string]map[string]int) rows, err := sq.Select("resources", "subcluster").From("job"). @@ -706,7 +713,6 @@ func (r *JobRepository) AllocatedNodes(cluster string) (map[string]map[string]in } func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error { - start := time.Now() res, err := sq.Update("job"). Set("monitoring_status", schema.MonitoringStatusArchivingFailed). @@ -735,7 +741,6 @@ func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error { } func (r *JobRepository) FindJobsBetween(startTimeBegin int64, startTimeEnd int64) ([]*schema.Job, error) { - var query sq.SelectBuilder if startTimeBegin == startTimeEnd || startTimeBegin > startTimeEnd { diff --git a/internal/repository/migration.go b/internal/repository/migration.go index 0f37d0a..38a88f7 100644 --- a/internal/repository/migration.go +++ b/internal/repository/migration.go @@ -16,7 +16,7 @@ import ( "github.com/golang-migrate/migrate/v4/source/iofs" ) -const Version uint = 6 +const Version uint = 7 //go:embed migrations/* var migrationFiles embed.FS @@ -57,7 +57,7 @@ func checkDBVersion(backend string, db *sql.DB) error { log.Fatalf("unsupported database backend: %s", backend) } - v, _, err := m.Version() + v, dirty, err := m.Version() if err != nil { if err == migrate.ErrNilVersion { log.Warn("Legacy database without version or missing database file!") @@ -68,18 +68,18 @@ func checkDBVersion(backend string, db *sql.DB) error { if v < Version { return fmt.Errorf("unsupported database version %d, need %d.\nPlease backup your database file and run cc-backend -migrate-db", v, Version) + } else if v > Version { + return fmt.Errorf("unsupported database version %d, need %d.\nPlease refer to documentation how to downgrade db with external migrate tool", v, Version) } - if v > Version { - return fmt.Errorf("unsupported database version %d, need %d.\nPlease refer to documentation how to downgrade db with external migrate tool", v, Version) + if dirty { + return fmt.Errorf("last migration to version %d has failed, please fix the db manually and force version with -force-db flag", Version) } return nil } -func MigrateDB(backend string, db string) error { - var m *migrate.Migrate - +func getMigrateInstance(backend string, db string) (m *migrate.Migrate, err error) { switch backend { case "sqlite3": d, err := iofs.New(migrationFiles, "migrations/sqlite3") @@ -89,22 +89,31 @@ func MigrateDB(backend string, db string) error { m, err = migrate.NewWithSourceInstance("iofs", d, fmt.Sprintf("sqlite3://%s?_foreign_keys=on", db)) if err != nil { - return err + return m, err } case "mysql": d, err := iofs.New(migrationFiles, "migrations/mysql") if err != nil { - return err + return m, err } m, err = migrate.NewWithSourceInstance("iofs", d, fmt.Sprintf("mysql://%s?multiStatements=true", db)) if err != nil { - return err + return m, err } default: log.Fatalf("unsupported database backend: %s", backend) } + return m, nil +} + +func MigrateDB(backend string, db string) error { + m, err := getMigrateInstance(backend, db) + if err != nil { + return err + } + if err := m.Up(); err != nil { if err == migrate.ErrNoChange { log.Info("DB already up to date!") @@ -116,3 +125,35 @@ func MigrateDB(backend string, db string) error { m.Close() return nil } + +func RevertDB(backend string, db string) error { + m, err := getMigrateInstance(backend, db) + if err != nil { + return err + } + + if err := m.Migrate(Version - 1); err != nil { + if err == migrate.ErrNoChange { + log.Info("DB already up to date!") + } else { + return err + } + } + + m.Close() + return nil +} + +func ForceDB(backend string, db string) error { + m, err := getMigrateInstance(backend, db) + if err != nil { + return err + } + + if err := m.Force(int(Version)); err != nil { + return err + } + + m.Close() + return nil +} diff --git a/internal/repository/migrations/mysql/07_fix-tag-id.down.sql b/internal/repository/migrations/mysql/07_fix-tag-id.down.sql new file mode 100644 index 0000000..9f9959a --- /dev/null +++ b/internal/repository/migrations/mysql/07_fix-tag-id.down.sql @@ -0,0 +1,3 @@ +SET FOREIGN_KEY_CHECKS = 0; +ALTER TABLE tag MODIFY id INTEGER; +SET FOREIGN_KEY_CHECKS = 1; diff --git a/internal/repository/migrations/mysql/07_fix-tag-id.up.sql b/internal/repository/migrations/mysql/07_fix-tag-id.up.sql new file mode 100644 index 0000000..1abc4b3 --- /dev/null +++ b/internal/repository/migrations/mysql/07_fix-tag-id.up.sql @@ -0,0 +1,3 @@ +SET FOREIGN_KEY_CHECKS = 0; +ALTER TABLE tag MODIFY id INTEGER AUTO_INCREMENT; +SET FOREIGN_KEY_CHECKS = 1; diff --git a/internal/repository/migrations/sqlite3/07_fix-tag-id.down.sql b/internal/repository/migrations/sqlite3/07_fix-tag-id.down.sql new file mode 100644 index 0000000..e69de29 diff --git a/internal/repository/migrations/sqlite3/07_fix-tag-id.up.sql b/internal/repository/migrations/sqlite3/07_fix-tag-id.up.sql new file mode 100644 index 0000000..e69de29 diff --git a/internal/repository/tags.go b/internal/repository/tags.go index 52bc836..e97bc65 100644 --- a/internal/repository/tags.go +++ b/internal/repository/tags.go @@ -15,8 +15,11 @@ import ( // Add the tag with id `tagId` to the job with the database id `jobId`. func (r *JobRepository) AddTag(job int64, tag int64) ([]*schema.Tag, error) { - if _, err := r.stmtCache.Exec(`INSERT INTO jobtag (job_id, tag_id) VALUES ($1, $2)`, job, tag); err != nil { - log.Error("Error while running query") + q := sq.Insert("jobtag").Columns("job_id", "tag_id").Values(job, tag) + + if _, err := q.RunWith(r.stmtCache).Exec(); err != nil { + s, _, _ := q.ToSql() + log.Errorf("Error adding tag with %s: %v", s, err) return nil, err } @@ -37,8 +40,11 @@ func (r *JobRepository) AddTag(job int64, tag int64) ([]*schema.Tag, error) { // Removes a tag from a job func (r *JobRepository) RemoveTag(job, tag int64) ([]*schema.Tag, error) { - if _, err := r.stmtCache.Exec("DELETE FROM jobtag WHERE jobtag.job_id = $1 AND jobtag.tag_id = $2", job, tag); err != nil { - log.Error("Error while running query") + q := sq.Delete("jobtag").Where("jobtag.job_id = ?", job).Where("jobtag.tag_id = ?", tag) + + if _, err := q.RunWith(r.stmtCache).Exec(); err != nil { + s, _, _ := q.ToSql() + log.Errorf("Error adding tag with %s: %v", s, err) return nil, err } @@ -59,8 +65,12 @@ func (r *JobRepository) RemoveTag(job, tag int64) ([]*schema.Tag, error) { // CreateTag creates a new tag with the specified type and name and returns its database id. func (r *JobRepository) CreateTag(tagType string, tagName string) (tagId int64, err error) { - res, err := r.stmtCache.Exec("INSERT INTO tag (tag_type, tag_name) VALUES ($1, $2)", tagType, tagName) + q := sq.Insert("tag").Columns("tag_type", "tag_name").Values(tagType, tagName) + + res, err := q.RunWith(r.stmtCache).Exec() if err != nil { + s, _, _ := q.ToSql() + log.Errorf("Error inserting tag with %s: %v", s, err) return 0, err } @@ -154,7 +164,8 @@ func (r *JobRepository) GetTags(job *int64) ([]*schema.Tag, error) { rows, err := q.RunWith(r.stmtCache).Query() if err != nil { - log.Error("Error while running query") + s, _, _ := q.ToSql() + log.Errorf("Error get tags with %s: %v", s, err) return nil, err } diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go index beeb24d..7e7b6e8 100644 --- a/pkg/archive/archive.go +++ b/pkg/archive/archive.go @@ -52,9 +52,11 @@ type JobContainer struct { Data *schema.JobData } -var cache *lrucache.Cache = lrucache.New(128 * 1024 * 1024) -var ar ArchiveBackend -var useArchive bool +var ( + cache *lrucache.Cache = lrucache.New(128 * 1024 * 1024) + ar ArchiveBackend + useArchive bool +) func Init(rawConfig json.RawMessage, disableArchive bool) error { useArchive = !disableArchive @@ -95,8 +97,8 @@ func GetHandle() ArchiveBackend { func LoadAveragesFromArchive( job *schema.Job, metrics []string, - data [][]schema.Float) error { - + data [][]schema.Float, +) error { metaFile, err := ar.LoadJobMeta(job) if err != nil { log.Warn("Error while loading job metadata from archiveBackend") @@ -115,7 +117,6 @@ func LoadAveragesFromArchive( } func GetStatistics(job *schema.Job) (map[string]schema.JobStatistics, error) { - metaFile, err := ar.LoadJobMeta(job) if err != nil { log.Warn("Error while loading job metadata from archiveBackend") @@ -125,10 +126,29 @@ func GetStatistics(job *schema.Job) (map[string]schema.JobStatistics, error) { return metaFile.Statistics, nil } +// If the job is archived, find its `meta.json` file and override the Metadata +// in that JSON file. If the job is not archived, nothing is done. +func UpdateMetadata(job *schema.Job, metadata map[string]string) error { + if job.State == schema.JobStateRunning || !useArchive { + return nil + } + + jobMeta, err := ar.LoadJobMeta(job) + if err != nil { + log.Warn("Error while loading job metadata from archiveBackend") + return err + } + + for k, v := range metadata { + jobMeta.MetaData[k] = v + } + + return ar.StoreJobMeta(jobMeta) +} + // If the job is archived, find its `meta.json` file and override the tags list // in that JSON file. If the job is not archived, nothing is done. func UpdateTags(job *schema.Job, tags []*schema.Tag) error { - if job.State == schema.JobStateRunning || !useArchive { return nil } diff --git a/tools/gen-keypair/main.go b/tools/gen-keypair/main.go index f8c66fe..ff9c5c3 100644 --- a/tools/gen-keypair/main.go +++ b/tools/gen-keypair/main.go @@ -1,4 +1,4 @@ -// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. // All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. @@ -20,7 +20,8 @@ func main() { os.Exit(1) } - fmt.Fprintf(os.Stdout, "JWT_PUBLIC_KEY=%#v\nJWT_PRIVATE_KEY=%#v\n", + fmt.Fprintf(os.Stdout, "ED25519 PUBLIC_KEY=%#v\nED25519 PRIVATE_KEY=%#v\n", base64.StdEncoding.EncodeToString(pub), base64.StdEncoding.EncodeToString(priv)) + fmt.Println("This is NO JWT token. You can generate JWT tokens with cc-backend. Use this keypair for signing and validation of JWT tokens in ClusterCockpit.") }