From e8d7722c42e0f1f7d7a412fc8bed26dde6b0f93b Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Mon, 19 Sep 2022 16:16:05 +0200 Subject: [PATCH] Add json schema validation for config file --- configs/config.schema.json | 351 +++++++++++++++++++++++++++++++++ internal/config/config.go | 31 +-- internal/config/config_test.go | 27 +++ pkg/schema/validate.go | 3 + 4 files changed, 398 insertions(+), 14 deletions(-) create mode 100644 configs/config.schema.json create mode 100644 internal/config/config_test.go diff --git a/configs/config.schema.json b/configs/config.schema.json new file mode 100644 index 0000000..876a4ee --- /dev/null +++ b/configs/config.schema.json @@ -0,0 +1,351 @@ +{ + "$schema": "http://json-schema.org/draft/2020-12/schema", + "title": "cc-backend configuration file schema", + "type": "object", + "properties":{ + "addr": { + "description": "Address where the http (or https) server will listen on (for example: 'localhost:80').", + "type": "string" + }, + "user": { + "description": "Drop root permissions once .env was read and the port was taken. Only applicable if using privileged port.", + "type": "string" + }, + "group": { + "description": "Drop root permissions once .env was read and the port was taken. Only applicable if using privileged port.", + "type": "string" + }, + "disable-authentication": { + "description": "Disable authentication (for everything: API, Web-UI, ...).", + "type": "boolean" + }, + "embed-static-files": { + "description": "If all files in `web/frontend/public` should be served from within the binary itself (they are embedded) or not.", + "type": "boolean" + }, + "static-files": { + "description": "Folder where static assets can be found, if embed-static-files is false.", + "type": "string" + }, + "db-driver": { + "description": "sqlite3 or mysql (mysql will work for mariadb as well).", + "type": "string", + "enum": [ + "sqlite3", + "mysql" + ] + }, + "db": { + "description": "For sqlite3 a filename, for mysql a DSN in this format: https://github.com/go-sql-driver/mysql#dsn-data-source-name (Without query parameters!).", + "type": "string" + }, + "job-archive": { + "description": "Path to the job-archive.", + "type": "string" + }, + "disable-archive": { + "description": "Keep all metric data in the metric data repositories, do not write to the job-archive.", + "type": "boolean" + }, + "validate": { + "description": "Validate all input json documents against json schema.", + "type": "boolean" + }, + "session-max-age": { + "description": "Specifies for how long a session shall be valid as a string parsable by time.ParseDuration(). If 0 or empty, the session/token does not expire!", + "type": "string" + }, + "jwt-max-age": { + "description": "Specifies for how long a JWT token shall be valid as a string parsable by time.ParseDuration(). If 0 or empty, the session/token does not expire!", + "type": "string" + }, + "https-cert-file": { + "description": "Filepath to SSL certificate. If also https-key-file is set use HTTPS using those certificates.", + "type": "string" + }, + "https-key-file": { + "description": "Filepath to SSL key file. If also https-cert-file is set use HTTPS using those certificates.", + "type": "string" + }, + "redirect-http-to": { + "description": "If not the empty string and addr does not end in :80, redirect every request incoming at port 80 to that url.", + "type": "string" + }, + "stop-jobs-exceeding-walltime": { + "description": "If not zero, automatically mark jobs as stopped running X seconds longer than their walltime. Only applies if walltime is set for job.", + "type": "integer" + }, + "": { + "description": "", + "type": "string" + }, + "ldap": { + "description": "For LDAP Authentication and user synchronisation.", + "type": "object", + "properties": { + "url": { + "description": "URL of LDAP directory server.", + "type": "string" + }, + "user_base": { + "description": "Base DN of user tree root.", + "type": "string" + }, + "search_dn": { + "description": "DN for authenticating LDAP admin account with general read rights.", + "type": "string" + }, + "user_bind": { + "description": "Expression used to authenticate users via LDAP bind. Must contain uid={username}.", + "type": "string" + }, + "user_filter": { + "description": "Filter to extract users for syncing.", + "type": "string" + }, + "sync_interval": { + "description": "Interval used for syncing local user table with LDAP directory. Parsed using time.ParseDuration.", + "type": "string" + }, + "sync_del_old_users": { + "description": "Delete obsolete users in database.", + "type": "boolean" + } + }, + "required": [ + "url", + "user_base", + "search_dn", + "user_bind", + "user_filter" + ] + }, + "clusters": { + "description": "Configuration for the clusters to be displayed.", + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "description": "The name of the cluster.", + "type": "string" + }, + "metricDataRepository": { + "description": "Type of the metric data repository for this cluster", + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "influxdb-v2", + "prometheus", + "cc-metric-store" + ] + }, + "url": { + "type": "string" + }, + "token": { + "type": "string" + } + }, + "required": [ + "kind", + "url" + ] + }, + "filterRanges": { + "description": "This option controls the slider ranges for the UI controls of numNodes, duration, and startTime.", + "type": "object", + "properties": { + "numNodes": { + "description": "UI slider range for number of nodes", + "type": "object", + "properties": { + "from": { + "type": "integer" + }, + "to": { + "type": "integer" + } + }, + "required": [ + "from", + "to" + ] + }, + "duration": { + "description": "UI slider range for duration", + "type": "object", + "properties": { + "from": { + "type": "integer" + }, + "to": { + "type": "integer" + } + }, + "required": [ + "from", + "to" + ] + }, + "startTime": { + "description": "UI slider range for start time", + "type": "object", + "properties": { + "from": { + "type": "string", + "format": "date-time" + }, + "to": { + "type": "null" + } + }, + "required": [ + "from", + "to" + ] + } + }, + "required": [ + "numNodes", + "duration", + "startTime" + ] + } + }, + "required": [ + "name", + "metricDataRepository", + "filterRanges" + ], + "minItems": 1 + } + }, + "ui-defaults": { + "description": "Default configuration for web UI", + "type": "object", + "properties": { + "plot_general_colorBackground": { + "description": "Color plot background according to job average threshold limits", + "type": "boolean" + }, + "plot_general_lineWidth": { + "description": "Initial linewidth", + "type": "integer" + }, + "plot_list_jobsPerPage": { + "description": "Jobs shown per page in job lists", + "type": "integer" + }, + "plot_list_hideShortRunningJobs": { + "description": "Do not show running jobs shorter than X seconds", + "type": "integer" + }, + "plot_view_plotsPerRow": { + "description": "Number of plots per row in single job view", + "type": "integer" + }, + "plot_view_showPolarplot": { + "description": "Option to toggle polar plot in single job view", + "type": "boolean" + }, + "plot_view_showRoofline": { + "description": "Option to toggle roofline plot in single job view", + "type": "boolean" + }, + "plot_view_showStatTable": { + "description": "Option to toggle the node statistic table in single job view", + "type": "boolean" + }, + "system_view_selectedMetric": { + "description": "Initial metric shown in system view", + "type": "string" + }, + "analysis_view_histogramMetrics": { + "description": "Metrics to show as job count histograms in analysis view", + "type": "array", + "items": { + "type": "string", + "minItems": 1 + } + }, + "analysis_view_scatterPlotMetrics": { + "description": "Initial scatter plto configuration in analysis view", + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string", + "minItems": 2, + "maxItems": 2 + }, + "minItems": 1 + } + }, + "job_view_nodestats_selectedMetrics": { + "description": "Initial metrics shown in node statistics table of single job view", + "type": "array", + "items": { + "type": "string", + "minItems": 1 + } + }, + "job_view_polarPlotMetrics": { + "description": "Metrics shown in polar plot of single job view", + "type": "array", + "items": { + "type": "string", + "minItems": 1 + } + }, + "job_view_selectedMetrics": { + "description": "", + "type": "array", + "items": { + "type": "string", + "minItems": 1 + } + }, + "plot_general_colorscheme": { + "description": "Initial color scheme", + "type": "array", + "items": { + "type": "string", + "minItems": 1 + } + }, + "plot_list_selectedMetrics": { + "description": "Initial metric plots shown in jobs lists", + "type": "array", + "items": { + "type": "string", + "minItems": 1 + } + } + }, + "required": [ + "plot_general_colorBackground", + "plot_general_lineWidth", + "plot_list_jobsPerPage", + "plot_view_plotsPerRow", + "plot_view_showPolarplot", + "plot_view_showRoofline", + "plot_view_showStatTable", + "system_view_selectedMetric", + "analysis_view_histogramMetrics", + "analysis_view_scatterPlotMetrics", + "job_view_nodestats_selectedMetrics", + "job_view_polarPlotMetrics", + "job_view_selectedMetrics", + "plot_general_colorscheme", + "plot_list_selectedMetrics", + "plot_list_hideShortRunningJobs" + ] + } + }, + "required": [ + "clusters" + ] +} diff --git a/internal/config/config.go b/internal/config/config.go index d764939..2ee882e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -5,6 +5,7 @@ package config import ( + "bytes" "encoding/json" "log" "os" @@ -13,16 +14,17 @@ import ( ) var Keys schema.ProgramConfig = schema.ProgramConfig{ - Addr: ":8080", - DisableAuthentication: false, - EmbedStaticFiles: true, - DBDriver: "sqlite3", - DB: "./var/job.db", - Archive: json.RawMessage(`{\"kind\":\"file\",\"path\":\"./var/job-archive\"}`), - DisableArchive: false, - Validate: false, - LdapConfig: nil, - SessionMaxAge: "168h", + Addr: ":8080", + DisableAuthentication: false, + EmbedStaticFiles: true, + DBDriver: "sqlite3", + DB: "./var/job.db", + Archive: json.RawMessage(`{\"kind\":\"file\",\"path\":\"./var/job-archive\"}`), + DisableArchive: false, + Validate: false, + LdapConfig: nil, + SessionMaxAge: "168h", + StopJobsExceedingWalltime: 0, UiDefaults: map[string]interface{}{ "analysis_view_histogramMetrics": []string{"flops_any", "mem_bw", "mem_used"}, "analysis_view_scatterPlotMetrics": [][]string{{"flops_any", "mem_bw"}, {"flops_any", "cpu_load"}, {"cpu_load", "mem_bw"}}, @@ -41,22 +43,23 @@ var Keys schema.ProgramConfig = schema.ProgramConfig{ "plot_view_showStatTable": true, "system_view_selectedMetric": "cpu_load", }, - StopJobsExceedingWalltime: 0, } func Init(flagConfigFile string) { - f, err := os.Open(flagConfigFile) + raw, err := os.ReadFile(flagConfigFile) if err != nil { if !os.IsNotExist(err) { log.Fatal(err) } } else { - dec := json.NewDecoder(f) + if err := schema.Validate(schema.Config, bytes.NewReader(raw)); err != nil { + log.Fatal(err) + } + dec := json.NewDecoder(bytes.NewReader(raw)) dec.DisallowUnknownFields() if err := dec.Decode(&Keys); err != nil { log.Fatal(err) } - f.Close() if Keys.Clusters == nil || len(Keys.Clusters) < 1 { log.Fatal("At least one cluster required in config!") diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..667eb3f --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,27 @@ +// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +package config + +import ( + "testing" + + _ "github.com/santhosh-tekuri/jsonschema/v5/httploader" +) + +func TestInit(t *testing.T) { + fp := "../../configs/config.json" + Init(fp) + if Keys.Addr != "0.0.0.0:443" { + t.Errorf("wrong addr\ngot: %s \nwant: 0.0.0.0:443", Keys.Addr) + } +} + +func TestInitMinimal(t *testing.T) { + fp := "../../docs/config.json" + Init(fp) + if Keys.Addr != "0.0.0.0:8080" { + t.Errorf("wrong addr\ngot: %s \nwant: 0.0.0.0:8080", Keys.Addr) + } +} diff --git a/pkg/schema/validate.go b/pkg/schema/validate.go index 37174f2..528dbfd 100644 --- a/pkg/schema/validate.go +++ b/pkg/schema/validate.go @@ -18,6 +18,7 @@ type Kind int const ( Meta Kind = iota + 1 Data + Config ClusterCfg ) @@ -31,6 +32,8 @@ func Validate(k Kind, r io.Reader) (err error) { s, err = jsonschema.Compile("https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-data.schema.json") case ClusterCfg: s, err = jsonschema.Compile("https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/cluster.schema.json") + case Config: + s, err = jsonschema.Compile("../../configs/config.schema.json") default: return fmt.Errorf("unkown schema kind ") }