Refactor package structure

Builds but not tested

pkg/archive/archive.go

@@ -0,0 +1,117 @@
+// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package archive
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/ClusterCockpit/cc-backend/internal/config"
+	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+)
+
+type ArchiveBackend interface {
+	Init(rawConfig json.RawMessage) error
+
+	// replaces previous loadMetaJson
+	LoadJobMeta(job *schema.Job) (schema.JobMeta, error)
+
+	// replaces previous loadFromArchive
+	LoadJobData(job *schema.Job) (schema.JobData, error)
+
+	LoadClusterCfg(name string) (model.Cluster, error)
+
+	StoreMeta(jobMeta *schema.JobMeta) error
+
+	Import(jobMeta *schema.JobMeta, jobData *schema.JobData) error
+
+	Iter() <-chan *schema.JobMeta
+}
+
+var ar ArchiveBackend
+
+func Init() error {
+	if config.Keys.Archive != nil {
+		var kind struct {
+			Kind string `json:"kind"`
+		}
+		if err := json.Unmarshal(config.Keys.Archive, &kind); err != nil {
+			return err
+		}
+
+		switch kind.Kind {
+		case "file":
+			ar = &FsArchive{}
+		// case "s3":
+		// 	ar = &S3Archive{}
+		default:
+			return fmt.Errorf("unkown archive backend '%s''", kind.Kind)
+		}
+
+		if err := ar.Init(config.Keys.Archive); err != nil {
+			return err
+		}
+	}
+	return initClusterConfig()
+}
+
+func GetHandle() ArchiveBackend {
+	return ar
+}
+
+// Helper to metricdata.LoadAverages().
+func LoadAveragesFromArchive(job *schema.Job, metrics []string, data [][]schema.Float) error {
+	metaFile, err := ar.LoadJobMeta(job)
+	if err != nil {
+		return err
+	}
+
+	for i, m := range metrics {
+		if stat, ok := metaFile.Statistics[m]; ok {
+			data[i] = append(data[i], schema.Float(stat.Avg))
+		} else {
+			data[i] = append(data[i], schema.NaN)
+		}
+	}
+
+	return nil
+}
+
+func GetStatistics(job *schema.Job) (map[string]schema.JobStatistics, error) {
+	metaFile, err := ar.LoadJobMeta(job)
+	if err != nil {
+		return nil, err
+	}
+
+	return metaFile.Statistics, nil
+}
+
+func Import(job *schema.JobMeta, jobData *schema.JobData) error {
+	return ar.Import(job, jobData)
+}
+
+// If the job is archived, find its `meta.json` file and override the tags list
+// in that JSON file. If the job is not archived, nothing is done.
+func UpdateTags(job *schema.Job, tags []*schema.Tag) error {
+	if job.State == schema.JobStateRunning {
+		return nil
+	}
+
+	jobMeta, err := ar.LoadJobMeta(job)
+	if err != nil {
+		return err
+	}
+
+	jobMeta.Tags = make([]*schema.Tag, 0)
+	for _, tag := range tags {
+		jobMeta.Tags = append(jobMeta.Tags, &schema.Tag{
+			Name: tag.Name,
+			Type: tag.Type,
+		})
+	}
+
+	return ar.StoreMeta(&jobMeta)
+}

pkg/archive/clusterConfig.go

@@ -0,0 +1,167 @@
+// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package archive
+
+import (
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/ClusterCockpit/cc-backend/internal/config"
+	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
+	"github.com/ClusterCockpit/cc-backend/pkg/lrucache"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+)
+
+var cache *lrucache.Cache = lrucache.New(1024)
+
+var Clusters []*model.Cluster
+var nodeLists map[string]map[string]NodeList
+
+func initClusterConfig() error {
+	Clusters = []*model.Cluster{}
+	nodeLists = map[string]map[string]NodeList{}
+
+	for _, c := range config.Keys.Clusters {
+
+		cluster, err := ar.LoadClusterCfg(c.Name)
+		if err != nil {
+			return err
+		}
+
+		if len(cluster.Name) == 0 || len(cluster.MetricConfig) == 0 || len(cluster.SubClusters) == 0 {
+			return errors.New("cluster.name, cluster.metricConfig and cluster.SubClusters should not be empty")
+		}
+
+		for _, mc := range cluster.MetricConfig {
+			if len(mc.Name) == 0 {
+				return errors.New("cluster.metricConfig.name should not be empty")
+			}
+			if mc.Timestep < 1 {
+				return errors.New("cluster.metricConfig.timestep should not be smaller than one")
+			}
+
+			// For backwards compability...
+			if mc.Scope == "" {
+				mc.Scope = schema.MetricScopeNode
+			}
+			if !mc.Scope.Valid() {
+				return errors.New("cluster.metricConfig.scope must be a valid scope ('node', 'scocket', ...)")
+			}
+		}
+
+		if cluster.FilterRanges.StartTime.To.IsZero() {
+			cluster.FilterRanges.StartTime.To = time.Unix(0, 0)
+		}
+
+		Clusters = append(Clusters, &cluster)
+
+		nodeLists[cluster.Name] = make(map[string]NodeList)
+		for _, sc := range cluster.SubClusters {
+			if sc.Nodes == "" {
+				continue
+			}
+
+			nl, err := ParseNodeList(sc.Nodes)
+			if err != nil {
+				return fmt.Errorf("in %s/cluster.json: %w", cluster.Name, err)
+			}
+			nodeLists[cluster.Name][sc.Name] = nl
+		}
+	}
+
+	return nil
+}
+
+func GetCluster(cluster string) *model.Cluster {
+	for _, c := range Clusters {
+		if c.Name == cluster {
+			return c
+		}
+	}
+	return nil
+}
+
+func GetSubCluster(cluster, subcluster string) *model.SubCluster {
+	for _, c := range Clusters {
+		if c.Name == cluster {
+			for _, p := range c.SubClusters {
+				if p.Name == subcluster {
+					return p
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func GetMetricConfig(cluster, metric string) *model.MetricConfig {
+	for _, c := range Clusters {
+		if c.Name == cluster {
+			for _, m := range c.MetricConfig {
+				if m.Name == metric {
+					return m
+				}
+			}
+		}
+	}
+	return nil
+}
+
+// AssignSubCluster sets the `job.subcluster` property of the job based
+// on its cluster and resources.
+func AssignSubCluster(job *schema.BaseJob) error {
+	cluster := GetCluster(job.Cluster)
+	if cluster == nil {
+		return fmt.Errorf("unkown cluster: %#v", job.Cluster)
+	}
+
+	if job.SubCluster != "" {
+		for _, sc := range cluster.SubClusters {
+			if sc.Name == job.SubCluster {
+				return nil
+			}
+		}
+		return fmt.Errorf("already assigned subcluster %#v unkown (cluster: %#v)", job.SubCluster, job.Cluster)
+	}
+
+	if len(job.Resources) == 0 {
+		return fmt.Errorf("job without any resources/hosts")
+	}
+
+	host0 := job.Resources[0].Hostname
+	for sc, nl := range nodeLists[job.Cluster] {
+		if nl != nil && nl.Contains(host0) {
+			job.SubCluster = sc
+			return nil
+		}
+	}
+
+	if cluster.SubClusters[0].Nodes == "" {
+		job.SubCluster = cluster.SubClusters[0].Name
+		return nil
+	}
+
+	return fmt.Errorf("no subcluster found for cluster %#v and host %#v", job.Cluster, host0)
+}
+
+func GetSubClusterByNode(cluster, hostname string) (string, error) {
+	for sc, nl := range nodeLists[cluster] {
+		if nl != nil && nl.Contains(hostname) {
+			return sc, nil
+		}
+	}
+
+	c := GetCluster(cluster)
+	if c == nil {
+		return "", fmt.Errorf("unkown cluster: %#v", cluster)
+	}
+
+	if c.SubClusters[0].Nodes == "" {
+		return c.SubClusters[0].Name, nil
+	}
+
+	return "", fmt.Errorf("no subcluster found for cluster %#v and host %#v", cluster, hostname)
+}

pkg/archive/fsBackend.go

@@ -0,0 +1,200 @@
+// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package archive
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+)
+
+type FsArchiveConfig struct {
+	Path string `json:"filePath"`
+}
+
+type FsArchive struct {
+	path string
+}
+
+// For a given job, return the path of the `data.json`/`meta.json` file.
+// TODO: Implement Issue ClusterCockpit/ClusterCockpit#97
+func getPath(job *schema.Job, rootPath string, file string) string {
+	lvl1, lvl2 := fmt.Sprintf("%d", job.JobID/1000), fmt.Sprintf("%03d", job.JobID%1000)
+	return filepath.Join(
+		rootPath,
+		job.Cluster,
+		lvl1, lvl2,
+		strconv.FormatInt(job.StartTime.Unix(), 10), file)
+}
+
+func loadJobMeta(filename string) (schema.JobMeta, error) {
+
+	f, err := os.Open(filename)
+	if err != nil {
+		return schema.JobMeta{}, err
+	}
+	defer f.Close()
+
+	return DecodeJobMeta(bufio.NewReader(f))
+}
+
+func (fsa *FsArchive) Init(rawConfig json.RawMessage) error {
+	var config FsArchiveConfig
+	if err := json.Unmarshal(rawConfig, &config); err != nil {
+		return err
+	}
+
+	fsa.path = config.Path
+	return nil
+}
+
+func (fsa *FsArchive) LoadJobData(job *schema.Job) (schema.JobData, error) {
+	filename := getPath(job, fsa.path, "data.json")
+
+	f, err := os.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	return DecodeJobData(bufio.NewReader(f))
+}
+
+func (fsa *FsArchive) LoadJobMeta(job *schema.Job) (schema.JobMeta, error) {
+	filename := getPath(job, fsa.path, "meta.json")
+
+	f, err := os.Open(filename)
+	if err != nil {
+		return schema.JobMeta{}, err
+	}
+	defer f.Close()
+
+	return DecodeJobMeta(bufio.NewReader(f))
+}
+
+func (fsa *FsArchive) LoadClusterCfg(name string) (model.Cluster, error) {
+	f, err := os.Open(filepath.Join(fsa.path, name, "cluster.json"))
+	if err != nil {
+		return model.Cluster{}, err
+	}
+	defer f.Close()
+
+	return DecodeCluster(bufio.NewReader(f))
+}
+
+func (fsa *FsArchive) Iter() <-chan *schema.JobMeta {
+	ch := make(chan *schema.JobMeta)
+	go func() {
+		clustersDir, err := os.ReadDir(fsa.path)
+		if err != nil {
+			log.Fatalf("Reading clusters failed: %s", err.Error())
+		}
+
+		for _, clusterDir := range clustersDir {
+			lvl1Dirs, err := os.ReadDir(filepath.Join(fsa.path, clusterDir.Name()))
+			if err != nil {
+				log.Fatalf("Reading jobs failed: %s", err.Error())
+			}
+
+			for _, lvl1Dir := range lvl1Dirs {
+				if !lvl1Dir.IsDir() {
+					// Could be the cluster.json file
+					continue
+				}
+
+				lvl2Dirs, err := os.ReadDir(filepath.Join(fsa.path, clusterDir.Name(), lvl1Dir.Name()))
+				if err != nil {
+					log.Fatalf("Reading jobs failed: %s", err.Error())
+				}
+
+				for _, lvl2Dir := range lvl2Dirs {
+					dirpath := filepath.Join(fsa.path, clusterDir.Name(), lvl1Dir.Name(), lvl2Dir.Name())
+					startTimeDirs, err := os.ReadDir(dirpath)
+					if err != nil {
+						log.Fatalf("Reading jobs failed: %s", err.Error())
+					}
+
+					// For compability with the old job-archive directory structure where
+					// there was no start time directory.
+					for _, startTimeDir := range startTimeDirs {
+						if startTimeDir.IsDir() {
+							job, err := loadJobMeta(filepath.Join(dirpath, startTimeDir.Name()))
+
+							if err != nil {
+								log.Errorf("in %s: %s", filepath.Join(dirpath, startTimeDir.Name()), err.Error())
+							} else {
+								ch <- &job
+							}
+						}
+					}
+				}
+			}
+		}
+	}()
+	return ch
+}
+
+func (fsa *FsArchive) StoreMeta(jobMeta *schema.JobMeta) error {
+
+	job := schema.Job{
+		BaseJob:       jobMeta.BaseJob,
+		StartTime:     time.Unix(jobMeta.StartTime, 0),
+		StartTimeUnix: jobMeta.StartTime,
+	}
+	f, err := os.Create(getPath(&job, fsa.path, "meta.json"))
+	if err != nil {
+		return err
+	}
+	if err := EncodeJobMeta(f, jobMeta); err != nil {
+		return err
+	}
+	if err := f.Close(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (fsa *FsArchive) Import(jobMeta *schema.JobMeta, jobData *schema.JobData) error {
+
+	job := schema.Job{
+		BaseJob:       jobMeta.BaseJob,
+		StartTime:     time.Unix(jobMeta.StartTime, 0),
+		StartTimeUnix: jobMeta.StartTime,
+	}
+	dir := getPath(&job, fsa.path, "")
+	if err := os.MkdirAll(dir, 0777); err != nil {
+		return err
+	}
+
+	f, err := os.Create(path.Join(dir, "meta.json"))
+	if err != nil {
+		return err
+	}
+	if err := EncodeJobMeta(f, jobMeta); err != nil {
+		return err
+	}
+	if err := f.Close(); err != nil {
+		return err
+	}
+
+	f, err = os.Create(path.Join(dir, "data.json"))
+	if err != nil {
+		return err
+	}
+	if err := EncodeJobData(f, jobData); err != nil {
+		return err
+	}
+	return f.Close()
+}

pkg/archive/json.go

@@ -0,0 +1,64 @@
+// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package archive
+
+import (
+	"encoding/json"
+	"io"
+
+	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+)
+
+func DecodeJobData(r io.Reader) (schema.JobData, error) {
+	var d schema.JobData
+	if err := json.NewDecoder(r).Decode(&d); err != nil {
+		return d, err
+	}
+
+	// Sanitize parameters
+
+	return d, nil
+}
+
+func DecodeJobMeta(r io.Reader) (schema.JobMeta, error) {
+	var d schema.JobMeta
+	if err := json.NewDecoder(r).Decode(&d); err != nil {
+		return d, err
+	}
+
+	// Sanitize parameters
+
+	return d, nil
+}
+
+func DecodeCluster(r io.Reader) (model.Cluster, error) {
+	var c model.Cluster
+	if err := json.NewDecoder(r).Decode(&c); err != nil {
+		return c, err
+	}
+
+	// Sanitize parameters
+
+	return c, nil
+}
+
+func EncodeJobData(w io.Writer, d *schema.JobData) error {
+	// Sanitize parameters
+	if err := json.NewEncoder(w).Encode(d); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func EncodeJobMeta(w io.Writer, d *schema.JobMeta) error {
+	// Sanitize parameters
+	if err := json.NewEncoder(w).Encode(d); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/archive/nodelist.go

@@ -0,0 +1,175 @@
+// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package archive
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+)
+
+type NodeList [][]interface {
+	consume(input string) (next string, ok bool)
+}
+
+func (nl *NodeList) Contains(name string) bool {
+	var ok bool
+	for _, term := range *nl {
+		str := name
+		for _, expr := range term {
+			str, ok = expr.consume(str)
+			if !ok {
+				break
+			}
+		}
+
+		if ok && str == "" {
+			return true
+		}
+	}
+
+	return false
+}
+
+type NLExprString string
+
+func (nle NLExprString) consume(input string) (next string, ok bool) {
+	str := string(nle)
+	if strings.HasPrefix(input, str) {
+		return strings.TrimPrefix(input, str), true
+	}
+	return "", false
+}
+
+type NLExprIntRanges []NLExprIntRange
+
+func (nles NLExprIntRanges) consume(input string) (next string, ok bool) {
+	for _, nle := range nles {
+		if next, ok := nle.consume(input); ok {
+			return next, ok
+		}
+	}
+	return "", false
+}
+
+type NLExprIntRange struct {
+	start, end int64
+	zeroPadded bool
+	digits     int
+}
+
+func (nle NLExprIntRange) consume(input string) (next string, ok bool) {
+	if !nle.zeroPadded || nle.digits < 1 {
+		log.Error("node list: only zero-padded ranges are allowed")
+		return "", false
+	}
+
+	if len(input) < nle.digits {
+		return "", false
+	}
+
+	numerals, rest := input[:nle.digits], input[nle.digits:]
+	for len(numerals) > 1 && numerals[0] == '0' {
+		numerals = numerals[1:]
+	}
+
+	x, err := strconv.ParseInt(numerals, 10, 32)
+	if err != nil {
+		return "", false
+	}
+
+	if nle.start <= x && x <= nle.end {
+		return rest, true
+	}
+
+	return "", false
+}
+
+func ParseNodeList(raw string) (NodeList, error) {
+	isLetter := func(r byte) bool { return ('a' <= r && r <= 'z') || ('A' <= r && r <= 'Z') }
+	isDigit := func(r byte) bool { return '0' <= r && r <= '9' }
+
+	rawterms := []string{}
+	prevterm := 0
+	for i := 0; i < len(raw); i++ {
+		if raw[i] == '[' {
+			for i < len(raw) && raw[i] != ']' {
+				i++
+			}
+			if i == len(raw) {
+				return nil, fmt.Errorf("node list: unclosed '['")
+			}
+		} else if raw[i] == ',' {
+			rawterms = append(rawterms, raw[prevterm:i])
+			prevterm = i + 1
+		}
+	}
+	if prevterm != len(raw) {
+		rawterms = append(rawterms, raw[prevterm:])
+	}
+
+	nl := NodeList{}
+	for _, rawterm := range rawterms {
+		exprs := []interface {
+			consume(input string) (next string, ok bool)
+		}{}
+		for i := 0; i < len(rawterm); i++ {
+			c := rawterm[i]
+			if isLetter(c) || isDigit(c) {
+				j := i
+				for j < len(rawterm) && (isLetter(rawterm[j]) || isDigit(rawterm[j])) {
+					j++
+				}
+				exprs = append(exprs, NLExprString(rawterm[i:j]))
+				i = j - 1
+			} else if c == '[' {
+				end := strings.Index(rawterm[i:], "]")
+				if end == -1 {
+					return nil, fmt.Errorf("node list: unclosed '['")
+				}
+
+				parts := strings.Split(rawterm[i+1:i+end], ",")
+				nles := NLExprIntRanges{}
+				for _, part := range parts {
+					minus := strings.Index(part, "-")
+					if minus == -1 {
+						return nil, fmt.Errorf("node list: no '-' found inside '[...]'")
+					}
+
+					s1, s2 := part[0:minus], part[minus+1:]
+					if len(s1) != len(s2) || len(s1) == 0 {
+						return nil, fmt.Errorf("node list: %#v and %#v are not of equal length or of length zero", s1, s2)
+					}
+
+					x1, err := strconv.ParseInt(s1, 10, 32)
+					if err != nil {
+						return nil, fmt.Errorf("node list: %w", err)
+					}
+					x2, err := strconv.ParseInt(s2, 10, 32)
+					if err != nil {
+						return nil, fmt.Errorf("node list: %w", err)
+					}
+
+					nles = append(nles, NLExprIntRange{
+						start:      x1,
+						end:        x2,
+						digits:     len(s1),
+						zeroPadded: true,
+					})
+				}
+
+				exprs = append(exprs, nles)
+				i += end
+			} else {
+				return nil, fmt.Errorf("node list: invalid character: %#v", rune(c))
+			}
+		}
+		nl = append(nl, exprs)
+	}
+
+	return nl, nil
+}

pkg/archive/nodelist_test.go

@@ -0,0 +1,59 @@
+// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package archive
+
+import (
+	"testing"
+)
+
+func TestNodeList(t *testing.T) {
+	nl, err := ParseNodeList("hallo,wel123t,emmy[01-99],fritz[005-500],woody[100-200]")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if nl.Contains("hello") || nl.Contains("woody") {
+		t.Fail()
+	}
+
+	if nl.Contains("fritz1") || nl.Contains("fritz9") || nl.Contains("fritz004") || nl.Contains("woody201") {
+		t.Fail()
+	}
+
+	if !nl.Contains("hallo") || !nl.Contains("wel123t") {
+		t.Fail()
+	}
+
+	if !nl.Contains("emmy01") || !nl.Contains("emmy42") || !nl.Contains("emmy99") {
+		t.Fail()
+	}
+
+	if !nl.Contains("woody100") || !nl.Contains("woody199") {
+		t.Fail()
+	}
+}
+
+func TestNodeListCommasInBrackets(t *testing.T) {
+	nl, err := ParseNodeList("a[1000-2000,2010-2090,3000-5000]")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if nl.Contains("hello") || nl.Contains("woody") {
+		t.Fatal("1")
+	}
+
+	if nl.Contains("a0") || nl.Contains("a0000") || nl.Contains("a5001") || nl.Contains("a2005") {
+		t.Fatal("2")
+	}
+
+	if !nl.Contains("a1001") || !nl.Contains("a2000") {
+		t.Fatal("3")
+	}
+
+	if !nl.Contains("a2042") || !nl.Contains("a4321") || !nl.Contains("a3000") {
+		t.Fatal("4")
+	}
+}

pkg/archive/s3Backend.go

@@ -0,0 +1,13 @@
+// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package archive
+
+type S3ArchiveConfig struct {
+	Path string `json:"filePath"`
+}
+
+type S3Archive struct {
+	path string
+}

pkg/archive/validate.go

@@ -0,0 +1,45 @@
+// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package archive
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/santhosh-tekuri/jsonschema"
+)
+
+type Kind int
+
+const (
+	Meta Kind = iota + 1
+	Data
+	Cluster
+)
+
+func Validate(k Kind, v io.Reader) (err error) {
+	var s *jsonschema.Schema
+
+	switch k {
+	case Meta:
+		s, err = jsonschema.Compile("https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-meta.schema.json")
+	case Data:
+		s, err = jsonschema.Compile("https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-data.schema.json")
+	case Cluster:
+		s, err = jsonschema.Compile("https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/cluster.schema.json")
+	default:
+		return fmt.Errorf("unkown schema kind ")
+	}
+
+	if err != nil {
+		return err
+	}
+
+	if err = s.Validate(v); err != nil {
+		return err
+	}
+
+	return nil
+}