feat(fleet): add NATS discovery roster publisher

Let fleet members find their peers without any central lookup call.
FleetPublisher reads the active-service roster and publishes, per bucket
(each cluster plus the reserved "infra" bucket) and per consumer service
type, the pre-filtered set of providers that type should discover. A
subscriber listens on exactly one subject,
cc.fleet.discovery.<cluster|infra>.<own-service-type>, and gets a
ready-to-use list. Rosters are re-published periodically because NATS
core pub/sub is fire-and-forget, so a service that subscribes late still
converges.

The publish function is injected rather than taken from the NATS client
directly, so the component has no hard broker dependency and is unit
testable without one; the wiring layer supplies nats.GetClient().Publish.

ProviderInfo is the entire on-wire shape and deliberately carries no
instance_id (the registration credential), no config content and no
config_revision: the discovery subjects have no application-layer auth,
so anything published there is readable by every subscriber.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

internal/fleet/discovery.go

@@ -0,0 +1,246 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package fleet
+
+import (
+	"context"
+	"encoding/json"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/ClusterCockpit/cc-backend/internal/repository"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage"
+)
+
+// infraBucket is the reserved subject token for the cluster-independent bucket;
+// cluster-scope buckets use the cluster name.
+const infraBucket = "infra"
+
+// DefaultDiscoveryPrefix is the NATS subject prefix under which discovery
+// rosters are published. A subscriber listens on
+// "<prefix>.<cluster-or-infra>.<own-service-type>".
+const DefaultDiscoveryPrefix = "cc.fleet.discovery"
+
+// ProviderInfo is one entry in a discovery roster: enough for a consumer to
+// locate a peer, and nothing more.
+//
+// SECURITY: this is the entire on-wire shape, published over NATS which has no
+// application-layer auth (see the package doc and internal/api/nats.go). It
+// deliberately carries NO instance_id (the registration credential), NO config
+// content, and NO config_revision. Meta is whatever the producer put in its
+// registration MetaData — operators MUST NOT register secrets there, since it
+// is broadcast to every subscriber.
+type ProviderInfo struct {
+	Type     ServiceType       `json:"type"`
+	Hostname string            `json:"hostname"`
+	State    string            `json:"state"`
+	Meta     map[string]string `json:"meta,omitempty"`
+}
+
+// roster is one published message: the set of providers relevant to a given
+// consumer type within a given bucket (a cluster name, or "infra").
+type roster struct {
+	subject   string
+	bucket    string
+	consumer  ServiceType
+	providers []ProviderInfo
+}
+
+// FleetPublisher builds per-consumer discovery rosters from the active-service
+// roster and publishes them over NATS. The publish function is injected so the
+// component has no hard NATS dependency and is unit-testable without a broker;
+// the wiring layer supplies one backed by nats.GetClient().Publish.
+//
+// Delivery model: server-tailored per-consumer subject. For each bucket and
+// each consumer service type, the server publishes the pre-filtered set of
+// providers that type should discover (see RelevantProviders). A subscriber
+// reads exactly one subject and gets a ready-to-use list.
+type FleetPublisher struct {
+	repo    *repository.FleetRepository
+	publish func(subject string, data []byte) error
+	prefix  string
+
+	stop     chan struct{}
+	stopOnce sync.Once
+}
+
+// NewFleetPublisher returns a publisher that sources the roster from the
+// singleton FleetRepository and emits via publish. An empty prefix defaults to
+// DefaultDiscoveryPrefix.
+func NewFleetPublisher(publish func(subject string, data []byte) error, prefix string) *FleetPublisher {
+	if prefix == "" {
+		prefix = DefaultDiscoveryPrefix
+	}
+	return &FleetPublisher{
+		repo:    repository.GetFleetRepository(),
+		publish: publish,
+		prefix:  prefix,
+		stop:    make(chan struct{}),
+	}
+}
+
+// PublishRosters builds and publishes every discovery roster once. Publishing
+// is best-effort: an encode/publish failure for one subject is logged and the
+// rest still go out; the first error is returned.
+func (p *FleetPublisher) PublishRosters() error {
+	active, err := p.repo.ListActive()
+	if err != nil {
+		return err
+	}
+
+	var firstErr error
+	for _, r := range buildRosters(active, p.prefix) {
+		data, encErr := encodeRoster(r)
+		if encErr != nil {
+			cclog.Errorf("fleet: encoding discovery roster for %q failed: %v", r.subject, encErr)
+			if firstErr == nil {
+				firstErr = encErr
+			}
+			continue
+		}
+		if pubErr := p.publish(r.subject, data); pubErr != nil {
+			cclog.Errorf("fleet: publishing discovery roster to %q failed: %v", r.subject, pubErr)
+			if firstErr == nil {
+				firstErr = pubErr
+			}
+		}
+	}
+	return firstErr
+}
+
+// Start publishes rosters once immediately, then re-publishes every interval
+// until ctx is cancelled or Shutdown is called. Periodic re-publish lets a
+// service that subscribes late converge to the current roster (NATS core
+// pub/sub is fire-and-forget). Mirrors Registry.StartSweep's lifecycle.
+func (p *FleetPublisher) Start(ctx context.Context, interval time.Duration) {
+	if err := p.PublishRosters(); err != nil {
+		cclog.Errorf("fleet: initial discovery publish failed: %v", err)
+	}
+	go func() {
+		ticker := time.NewTicker(interval)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-p.stop:
+				return
+			case <-ticker.C:
+				if err := p.PublishRosters(); err != nil {
+					cclog.Errorf("fleet: periodic discovery publish failed: %v", err)
+				}
+			}
+		}
+	}()
+}
+
+// Shutdown stops the periodic publisher. Safe to call multiple times.
+func (p *FleetPublisher) Shutdown() {
+	p.stopOnce.Do(func() { close(p.stop) })
+}
+
+// buildRosters is the pure core: given the active services, produce one roster
+// per (bucket, consumer-type) where the consumer type has any relevant
+// providers. Buckets are every cluster present plus the reserved "infra"
+// bucket. A cluster-scope consumer on cluster C sees relevant providers in C
+// plus all infra-scope providers; the infra bucket sees relevant providers
+// everywhere.
+func buildRosters(active []*repository.ServiceDB, prefix string) []roster {
+	byCluster := make(map[string][]*repository.ServiceDB)
+	var infra []*repository.ServiceDB
+	for _, s := range active {
+		if s.Scope == ScopeInfra {
+			infra = append(infra, s)
+		} else {
+			byCluster[s.Cluster] = append(byCluster[s.Cluster], s)
+		}
+	}
+
+	buckets := make([]string, 0, len(byCluster)+1)
+	for c := range byCluster {
+		buckets = append(buckets, c)
+	}
+	sort.Strings(buckets)
+	buckets = append(buckets, infraBucket)
+
+	rosters := make([]roster, 0)
+	for _, bucket := range buckets {
+		var candidates []*repository.ServiceDB
+		if bucket == infraBucket {
+			candidates = active // relevant providers everywhere
+		} else {
+			candidates = append(candidates, byCluster[bucket]...)
+			candidates = append(candidates, infra...)
+		}
+
+		for _, consumer := range AllServiceTypes {
+			rel := RelevantProviders(consumer)
+			if len(rel) == 0 {
+				continue
+			}
+			relSet := make(map[ServiceType]struct{}, len(rel))
+			for _, t := range rel {
+				relSet[t] = struct{}{}
+			}
+
+			providers := make([]ProviderInfo, 0)
+			for _, s := range candidates {
+				if _, ok := relSet[ServiceType(s.ServiceType)]; ok {
+					providers = append(providers, toProviderInfo(s))
+				}
+			}
+			sort.Slice(providers, func(i, j int) bool {
+				if providers[i].Type != providers[j].Type {
+					return providers[i].Type < providers[j].Type
+				}
+				return providers[i].Hostname < providers[j].Hostname
+			})
+
+			rosters = append(rosters, roster{
+				subject:   prefix + "." + bucket + "." + string(consumer),
+				bucket:    bucket,
+				consumer:  consumer,
+				providers: providers,
+			})
+		}
+	}
+	return rosters
+}
+
+func toProviderInfo(s *repository.ServiceDB) ProviderInfo {
+	meta, err := unmarshalMeta(s.MetaData)
+	if err != nil {
+		cclog.Warnf("fleet: ignoring unparseable meta_data for %s/%s: %v", s.Hostname, s.ServiceType, err)
+		meta = nil
+	}
+	return ProviderInfo{
+		Type:     ServiceType(s.ServiceType),
+		Hostname: s.Hostname,
+		State:    s.State,
+		Meta:     meta,
+	}
+}
+
+// encodeRoster serializes a roster as an InfluxDB line-protocol event, matching
+// the decode path in internal/api/nats.go: measurement "fleetdiscovery", tags
+// cluster/type, and a JSON array of providers in the "event" field.
+func encodeRoster(r roster) ([]byte, error) {
+	payload, err := json.Marshal(r.providers)
+	if err != nil {
+		return nil, err
+	}
+	tags := map[string]string{
+		"cluster": r.bucket,
+		"type":    string(r.consumer),
+	}
+	msg, err := lp.NewEvent("fleetdiscovery", tags, nil, string(payload), time.Now())
+	if err != nil {
+		return nil, err
+	}
+	return []byte(msg.ToLineProtocol(nil)), nil
+}

internal/fleet/discovery_test.go

@@ -0,0 +1,209 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package fleet
+
+import (
+	"database/sql"
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/ClusterCockpit/cc-backend/internal/repository"
+	"github.com/ClusterCockpit/cc-lib/v2/receivers"
+	influx "github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol"
+)
+
+func mkSvc(scope, cluster, stype, host, state, metaJSON string) *repository.ServiceDB {
+	meta := sql.NullString{}
+	if metaJSON != "" {
+		meta = sql.NullString{String: metaJSON, Valid: true}
+	}
+	return &repository.ServiceDB{
+		Scope: scope, Cluster: cluster, ServiceType: stype, Hostname: host,
+		State: state, InstanceID: "iid-" + host, MetaData: meta,
+	}
+}
+
+func findRoster(rosters []roster, subject string) *roster {
+	for i := range rosters {
+		if rosters[i].subject == subject {
+			return &rosters[i]
+		}
+	}
+	return nil
+}
+
+func TestBuildRosters(t *testing.T) {
+	active := []*repository.ServiceDB{
+		mkSvc(ScopeInfra, "", "ccb", "mgmt01", "active", `{"endpoint":"https://mgmt:8080"}`),
+		mkSvc(ScopeCluster, "fritz", "ccb", "f-ccb", "active", ""), // a cluster-local backend
+		mkSvc(ScopeInfra, "", "ccms", "store01", "active", ""),
+		mkSvc(ScopeCluster, "fritz", "ccmc", "f-node01", "active", ""),
+		mkSvc(ScopeCluster, "alex", "ccmc", "a-node01", "active", ""),
+	}
+
+	rosters := buildRosters(active, DefaultDiscoveryPrefix)
+
+	t.Run("cluster consumer sees own-cluster + infra providers", func(t *testing.T) {
+		r := findRoster(rosters, "cc.fleet.discovery.fritz.ccmc")
+		if r == nil {
+			t.Fatal("missing fritz.ccmc roster")
+		}
+		// ccmc -> {ccb}: the fritz-local ccb AND the infra ccb, sorted by host.
+		if len(r.providers) != 2 {
+			t.Fatalf("want 2 providers, got %+v", r.providers)
+		}
+		if r.providers[0].Hostname != "f-ccb" || r.providers[1].Hostname != "mgmt01" {
+			t.Fatalf("unexpected providers/order: %+v", r.providers)
+		}
+		for _, p := range r.providers {
+			if p.Type != ServiceTypeBackend {
+				t.Fatalf("only ccb is relevant to ccmc, got %q", p.Type)
+			}
+		}
+	})
+
+	t.Run("cluster filter excludes other clusters", func(t *testing.T) {
+		r := findRoster(rosters, "cc.fleet.discovery.alex.ccmc")
+		if r == nil {
+			t.Fatal("missing alex.ccmc roster")
+		}
+		// alex sees only the infra ccb, never fritz's f-ccb.
+		if len(r.providers) != 1 || r.providers[0].Hostname != "mgmt01" {
+			t.Fatalf("cluster filter leaked: %+v", r.providers)
+		}
+	})
+
+	t.Run("infra bucket exists and carries relevant providers", func(t *testing.T) {
+		r := findRoster(rosters, "cc.fleet.discovery.infra.ccms")
+		if r == nil {
+			t.Fatal("missing infra.ccms roster")
+		}
+		// ccms -> {ccb}: sees both ccb instances anywhere.
+		if len(r.providers) != 2 {
+			t.Fatalf("want 2 ccb providers in infra bucket, got %+v", r.providers)
+		}
+	})
+
+	t.Run("meta is carried through", func(t *testing.T) {
+		r := findRoster(rosters, "cc.fleet.discovery.fritz.ccmc")
+		var mgmt *ProviderInfo
+		for i := range r.providers {
+			if r.providers[i].Hostname == "mgmt01" {
+				mgmt = &r.providers[i]
+			}
+		}
+		if mgmt == nil || mgmt.Meta["endpoint"] != "https://mgmt:8080" {
+			t.Fatalf("endpoint meta not carried: %+v", mgmt)
+		}
+	})
+
+	t.Run("consumer with no relevant providers is not published", func(t *testing.T) {
+		if r := findRoster(rosters, "cc.fleet.discovery.fritz.ccb"); r != nil {
+			t.Fatalf("ccb has no relevant providers; should emit no roster, got %+v", r.providers)
+		}
+	})
+}
+
+// capturePub records published messages instead of sending them to a broker.
+type capturePub struct{ msgs map[string][]byte }
+
+func (c *capturePub) publish(subject string, data []byte) error {
+	c.msgs[subject] = data
+	return nil
+}
+
+func TestPublishRostersRoundTripNoLeak(t *testing.T) {
+	setupDB(t)
+
+	infraReg := NewInfraRegistry()
+	reg := NewRegistry(time.Hour)
+
+	// Register + activate a global backend, a metric store, and a cluster collector.
+	rb, err := infraReg.Register(InfraRegistrationRequest{
+		Hostname: "mgmt01", ServiceType: ServiceTypeBackend,
+		MetaData: map[string]string{"endpoint": "https://mgmt:8080"},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	rs, err := infraReg.Register(InfraRegistrationRequest{Hostname: "store01", ServiceType: ServiceTypeMetricStore})
+	if err != nil {
+		t.Fatal(err)
+	}
+	rc, err := reg.Register(RegistrationRequest{Cluster: "fritz", Hostname: "f-node01", ServiceType: ServiceTypeCollector})
+	if err != nil {
+		t.Fatal(err)
+	}
+	instanceIDs := []string{rb.InstanceID, rs.InstanceID, rc.InstanceID}
+
+	// Only active services are advertised.
+	for _, id := range instanceIDs {
+		if err := infraReg.Heartbeat(id, time.Unix(1000, 0)); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	cap := &capturePub{msgs: make(map[string][]byte)}
+	pub := NewFleetPublisher(cap.publish, "")
+	if err := pub.PublishRosters(); err != nil {
+		t.Fatal(err)
+	}
+	if len(cap.msgs) == 0 {
+		t.Fatal("no rosters published")
+	}
+
+	// SECURITY: the registration credential must never appear on the wire.
+	for subject, data := range cap.msgs {
+		for _, id := range instanceIDs {
+			if strings.Contains(string(data), id) {
+				t.Fatalf("instance_id %q leaked into %q", id, subject)
+			}
+		}
+	}
+
+	// Round-trip: the fritz collector's roster decodes via the same line-protocol
+	// path as internal/api/nats.go and contains the backend.
+	data, ok := cap.msgs["cc.fleet.discovery.fritz.ccmc"]
+	if !ok {
+		t.Fatal("missing fritz.ccmc roster")
+	}
+	providers := decodeRoster(t, data)
+	found := false
+	for _, p := range providers {
+		if p.Type == ServiceTypeBackend && p.Hostname == "mgmt01" {
+			found = true
+			if p.Meta["endpoint"] != "https://mgmt:8080" {
+				t.Errorf("endpoint meta lost in round-trip: %+v", p)
+			}
+		}
+	}
+	if !found {
+		t.Fatalf("ccb/mgmt01 not in decoded roster: %+v", providers)
+	}
+}
+
+func decodeRoster(t *testing.T, data []byte) []ProviderInfo {
+	t.Helper()
+	d := influx.NewDecoderWithBytes(data)
+	if !d.Next() {
+		t.Fatal("no line-protocol message decoded")
+	}
+	m, err := receivers.DecodeInfluxMessage(d)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ev, ok := m.GetEventValue()
+	if !ok {
+		t.Fatal("message has no event field")
+	}
+	var providers []ProviderInfo
+	if err := json.Unmarshal([]byte(ev), &providers); err != nil {
+		t.Fatal(err)
+	}
+	return providers
+}