diff --git a/internal/fleet/discovery.go b/internal/fleet/discovery.go new file mode 100644 index 00000000..8826c055 --- /dev/null +++ b/internal/fleet/discovery.go @@ -0,0 +1,246 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package fleet + +import ( + "context" + "encoding/json" + "sort" + "sync" + "time" + + "github.com/ClusterCockpit/cc-backend/internal/repository" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage" +) + +// infraBucket is the reserved subject token for the cluster-independent bucket; +// cluster-scope buckets use the cluster name. +const infraBucket = "infra" + +// DefaultDiscoveryPrefix is the NATS subject prefix under which discovery +// rosters are published. A subscriber listens on +// "..". +const DefaultDiscoveryPrefix = "cc.fleet.discovery" + +// ProviderInfo is one entry in a discovery roster: enough for a consumer to +// locate a peer, and nothing more. +// +// SECURITY: this is the entire on-wire shape, published over NATS which has no +// application-layer auth (see the package doc and internal/api/nats.go). It +// deliberately carries NO instance_id (the registration credential), NO config +// content, and NO config_revision. Meta is whatever the producer put in its +// registration MetaData — operators MUST NOT register secrets there, since it +// is broadcast to every subscriber. +type ProviderInfo struct { + Type ServiceType `json:"type"` + Hostname string `json:"hostname"` + State string `json:"state"` + Meta map[string]string `json:"meta,omitempty"` +} + +// roster is one published message: the set of providers relevant to a given +// consumer type within a given bucket (a cluster name, or "infra"). +type roster struct { + subject string + bucket string + consumer ServiceType + providers []ProviderInfo +} + +// FleetPublisher builds per-consumer discovery rosters from the active-service +// roster and publishes them over NATS. The publish function is injected so the +// component has no hard NATS dependency and is unit-testable without a broker; +// the wiring layer supplies one backed by nats.GetClient().Publish. +// +// Delivery model: server-tailored per-consumer subject. For each bucket and +// each consumer service type, the server publishes the pre-filtered set of +// providers that type should discover (see RelevantProviders). A subscriber +// reads exactly one subject and gets a ready-to-use list. +type FleetPublisher struct { + repo *repository.FleetRepository + publish func(subject string, data []byte) error + prefix string + + stop chan struct{} + stopOnce sync.Once +} + +// NewFleetPublisher returns a publisher that sources the roster from the +// singleton FleetRepository and emits via publish. An empty prefix defaults to +// DefaultDiscoveryPrefix. +func NewFleetPublisher(publish func(subject string, data []byte) error, prefix string) *FleetPublisher { + if prefix == "" { + prefix = DefaultDiscoveryPrefix + } + return &FleetPublisher{ + repo: repository.GetFleetRepository(), + publish: publish, + prefix: prefix, + stop: make(chan struct{}), + } +} + +// PublishRosters builds and publishes every discovery roster once. Publishing +// is best-effort: an encode/publish failure for one subject is logged and the +// rest still go out; the first error is returned. +func (p *FleetPublisher) PublishRosters() error { + active, err := p.repo.ListActive() + if err != nil { + return err + } + + var firstErr error + for _, r := range buildRosters(active, p.prefix) { + data, encErr := encodeRoster(r) + if encErr != nil { + cclog.Errorf("fleet: encoding discovery roster for %q failed: %v", r.subject, encErr) + if firstErr == nil { + firstErr = encErr + } + continue + } + if pubErr := p.publish(r.subject, data); pubErr != nil { + cclog.Errorf("fleet: publishing discovery roster to %q failed: %v", r.subject, pubErr) + if firstErr == nil { + firstErr = pubErr + } + } + } + return firstErr +} + +// Start publishes rosters once immediately, then re-publishes every interval +// until ctx is cancelled or Shutdown is called. Periodic re-publish lets a +// service that subscribes late converge to the current roster (NATS core +// pub/sub is fire-and-forget). Mirrors Registry.StartSweep's lifecycle. +func (p *FleetPublisher) Start(ctx context.Context, interval time.Duration) { + if err := p.PublishRosters(); err != nil { + cclog.Errorf("fleet: initial discovery publish failed: %v", err) + } + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-p.stop: + return + case <-ticker.C: + if err := p.PublishRosters(); err != nil { + cclog.Errorf("fleet: periodic discovery publish failed: %v", err) + } + } + } + }() +} + +// Shutdown stops the periodic publisher. Safe to call multiple times. +func (p *FleetPublisher) Shutdown() { + p.stopOnce.Do(func() { close(p.stop) }) +} + +// buildRosters is the pure core: given the active services, produce one roster +// per (bucket, consumer-type) where the consumer type has any relevant +// providers. Buckets are every cluster present plus the reserved "infra" +// bucket. A cluster-scope consumer on cluster C sees relevant providers in C +// plus all infra-scope providers; the infra bucket sees relevant providers +// everywhere. +func buildRosters(active []*repository.ServiceDB, prefix string) []roster { + byCluster := make(map[string][]*repository.ServiceDB) + var infra []*repository.ServiceDB + for _, s := range active { + if s.Scope == ScopeInfra { + infra = append(infra, s) + } else { + byCluster[s.Cluster] = append(byCluster[s.Cluster], s) + } + } + + buckets := make([]string, 0, len(byCluster)+1) + for c := range byCluster { + buckets = append(buckets, c) + } + sort.Strings(buckets) + buckets = append(buckets, infraBucket) + + rosters := make([]roster, 0) + for _, bucket := range buckets { + var candidates []*repository.ServiceDB + if bucket == infraBucket { + candidates = active // relevant providers everywhere + } else { + candidates = append(candidates, byCluster[bucket]...) + candidates = append(candidates, infra...) + } + + for _, consumer := range AllServiceTypes { + rel := RelevantProviders(consumer) + if len(rel) == 0 { + continue + } + relSet := make(map[ServiceType]struct{}, len(rel)) + for _, t := range rel { + relSet[t] = struct{}{} + } + + providers := make([]ProviderInfo, 0) + for _, s := range candidates { + if _, ok := relSet[ServiceType(s.ServiceType)]; ok { + providers = append(providers, toProviderInfo(s)) + } + } + sort.Slice(providers, func(i, j int) bool { + if providers[i].Type != providers[j].Type { + return providers[i].Type < providers[j].Type + } + return providers[i].Hostname < providers[j].Hostname + }) + + rosters = append(rosters, roster{ + subject: prefix + "." + bucket + "." + string(consumer), + bucket: bucket, + consumer: consumer, + providers: providers, + }) + } + } + return rosters +} + +func toProviderInfo(s *repository.ServiceDB) ProviderInfo { + meta, err := unmarshalMeta(s.MetaData) + if err != nil { + cclog.Warnf("fleet: ignoring unparseable meta_data for %s/%s: %v", s.Hostname, s.ServiceType, err) + meta = nil + } + return ProviderInfo{ + Type: ServiceType(s.ServiceType), + Hostname: s.Hostname, + State: s.State, + Meta: meta, + } +} + +// encodeRoster serializes a roster as an InfluxDB line-protocol event, matching +// the decode path in internal/api/nats.go: measurement "fleetdiscovery", tags +// cluster/type, and a JSON array of providers in the "event" field. +func encodeRoster(r roster) ([]byte, error) { + payload, err := json.Marshal(r.providers) + if err != nil { + return nil, err + } + tags := map[string]string{ + "cluster": r.bucket, + "type": string(r.consumer), + } + msg, err := lp.NewEvent("fleetdiscovery", tags, nil, string(payload), time.Now()) + if err != nil { + return nil, err + } + return []byte(msg.ToLineProtocol(nil)), nil +} diff --git a/internal/fleet/discovery_test.go b/internal/fleet/discovery_test.go new file mode 100644 index 00000000..e4439e8d --- /dev/null +++ b/internal/fleet/discovery_test.go @@ -0,0 +1,209 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package fleet + +import ( + "database/sql" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/ClusterCockpit/cc-backend/internal/repository" + "github.com/ClusterCockpit/cc-lib/v2/receivers" + influx "github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol" +) + +func mkSvc(scope, cluster, stype, host, state, metaJSON string) *repository.ServiceDB { + meta := sql.NullString{} + if metaJSON != "" { + meta = sql.NullString{String: metaJSON, Valid: true} + } + return &repository.ServiceDB{ + Scope: scope, Cluster: cluster, ServiceType: stype, Hostname: host, + State: state, InstanceID: "iid-" + host, MetaData: meta, + } +} + +func findRoster(rosters []roster, subject string) *roster { + for i := range rosters { + if rosters[i].subject == subject { + return &rosters[i] + } + } + return nil +} + +func TestBuildRosters(t *testing.T) { + active := []*repository.ServiceDB{ + mkSvc(ScopeInfra, "", "ccb", "mgmt01", "active", `{"endpoint":"https://mgmt:8080"}`), + mkSvc(ScopeCluster, "fritz", "ccb", "f-ccb", "active", ""), // a cluster-local backend + mkSvc(ScopeInfra, "", "ccms", "store01", "active", ""), + mkSvc(ScopeCluster, "fritz", "ccmc", "f-node01", "active", ""), + mkSvc(ScopeCluster, "alex", "ccmc", "a-node01", "active", ""), + } + + rosters := buildRosters(active, DefaultDiscoveryPrefix) + + t.Run("cluster consumer sees own-cluster + infra providers", func(t *testing.T) { + r := findRoster(rosters, "cc.fleet.discovery.fritz.ccmc") + if r == nil { + t.Fatal("missing fritz.ccmc roster") + } + // ccmc -> {ccb}: the fritz-local ccb AND the infra ccb, sorted by host. + if len(r.providers) != 2 { + t.Fatalf("want 2 providers, got %+v", r.providers) + } + if r.providers[0].Hostname != "f-ccb" || r.providers[1].Hostname != "mgmt01" { + t.Fatalf("unexpected providers/order: %+v", r.providers) + } + for _, p := range r.providers { + if p.Type != ServiceTypeBackend { + t.Fatalf("only ccb is relevant to ccmc, got %q", p.Type) + } + } + }) + + t.Run("cluster filter excludes other clusters", func(t *testing.T) { + r := findRoster(rosters, "cc.fleet.discovery.alex.ccmc") + if r == nil { + t.Fatal("missing alex.ccmc roster") + } + // alex sees only the infra ccb, never fritz's f-ccb. + if len(r.providers) != 1 || r.providers[0].Hostname != "mgmt01" { + t.Fatalf("cluster filter leaked: %+v", r.providers) + } + }) + + t.Run("infra bucket exists and carries relevant providers", func(t *testing.T) { + r := findRoster(rosters, "cc.fleet.discovery.infra.ccms") + if r == nil { + t.Fatal("missing infra.ccms roster") + } + // ccms -> {ccb}: sees both ccb instances anywhere. + if len(r.providers) != 2 { + t.Fatalf("want 2 ccb providers in infra bucket, got %+v", r.providers) + } + }) + + t.Run("meta is carried through", func(t *testing.T) { + r := findRoster(rosters, "cc.fleet.discovery.fritz.ccmc") + var mgmt *ProviderInfo + for i := range r.providers { + if r.providers[i].Hostname == "mgmt01" { + mgmt = &r.providers[i] + } + } + if mgmt == nil || mgmt.Meta["endpoint"] != "https://mgmt:8080" { + t.Fatalf("endpoint meta not carried: %+v", mgmt) + } + }) + + t.Run("consumer with no relevant providers is not published", func(t *testing.T) { + if r := findRoster(rosters, "cc.fleet.discovery.fritz.ccb"); r != nil { + t.Fatalf("ccb has no relevant providers; should emit no roster, got %+v", r.providers) + } + }) +} + +// capturePub records published messages instead of sending them to a broker. +type capturePub struct{ msgs map[string][]byte } + +func (c *capturePub) publish(subject string, data []byte) error { + c.msgs[subject] = data + return nil +} + +func TestPublishRostersRoundTripNoLeak(t *testing.T) { + setupDB(t) + + infraReg := NewInfraRegistry() + reg := NewRegistry(time.Hour) + + // Register + activate a global backend, a metric store, and a cluster collector. + rb, err := infraReg.Register(InfraRegistrationRequest{ + Hostname: "mgmt01", ServiceType: ServiceTypeBackend, + MetaData: map[string]string{"endpoint": "https://mgmt:8080"}, + }) + if err != nil { + t.Fatal(err) + } + rs, err := infraReg.Register(InfraRegistrationRequest{Hostname: "store01", ServiceType: ServiceTypeMetricStore}) + if err != nil { + t.Fatal(err) + } + rc, err := reg.Register(RegistrationRequest{Cluster: "fritz", Hostname: "f-node01", ServiceType: ServiceTypeCollector}) + if err != nil { + t.Fatal(err) + } + instanceIDs := []string{rb.InstanceID, rs.InstanceID, rc.InstanceID} + + // Only active services are advertised. + for _, id := range instanceIDs { + if err := infraReg.Heartbeat(id, time.Unix(1000, 0)); err != nil { + t.Fatal(err) + } + } + + cap := &capturePub{msgs: make(map[string][]byte)} + pub := NewFleetPublisher(cap.publish, "") + if err := pub.PublishRosters(); err != nil { + t.Fatal(err) + } + if len(cap.msgs) == 0 { + t.Fatal("no rosters published") + } + + // SECURITY: the registration credential must never appear on the wire. + for subject, data := range cap.msgs { + for _, id := range instanceIDs { + if strings.Contains(string(data), id) { + t.Fatalf("instance_id %q leaked into %q", id, subject) + } + } + } + + // Round-trip: the fritz collector's roster decodes via the same line-protocol + // path as internal/api/nats.go and contains the backend. + data, ok := cap.msgs["cc.fleet.discovery.fritz.ccmc"] + if !ok { + t.Fatal("missing fritz.ccmc roster") + } + providers := decodeRoster(t, data) + found := false + for _, p := range providers { + if p.Type == ServiceTypeBackend && p.Hostname == "mgmt01" { + found = true + if p.Meta["endpoint"] != "https://mgmt:8080" { + t.Errorf("endpoint meta lost in round-trip: %+v", p) + } + } + } + if !found { + t.Fatalf("ccb/mgmt01 not in decoded roster: %+v", providers) + } +} + +func decodeRoster(t *testing.T, data []byte) []ProviderInfo { + t.Helper() + d := influx.NewDecoderWithBytes(data) + if !d.Next() { + t.Fatal("no line-protocol message decoded") + } + m, err := receivers.DecodeInfluxMessage(d) + if err != nil { + t.Fatal(err) + } + ev, ok := m.GetEventValue() + if !ok { + t.Fatal("message has no event field") + } + var providers []ProviderInfo + if err := json.Unmarshal([]byte(ev), &providers); err != nil { + t.Fatal(err) + } + return providers +}