Allow multiple nats subscriptions

README.md

@@ -77,7 +77,9 @@ All durations are specified as string that will be parsed [like this](https://pk
 - `nats`:
     - `address`: Url of NATS.io server, example: "nats://localhost:4222"
     - `username` and `password`: Optional, if provided use those for the connection
+    - `subscriptions`:
         - `subscribe-to`: Where to expect the measurements to be published
+        - `cluster-tag`: Default value for the cluster tag
 - `http-api`:
     - `address`: Address to bind to, for example `0.0.0.0:8080`
     - `https-cert-file` and `https-key-file`: Optional, if provided enable HTTPS using those files as certificate/key

lineprotocol.go

@@ -33,10 +33,13 @@ func ReceiveNats(conf *NatsConfig, handleLine func(*lineprotocol.Decoder, string
 	defer nc.Close()
 
 	var wg sync.WaitGroup
-	var sub *nats.Subscription
+	var subs []*nats.Subscription
 
 	msgs := make(chan *nats.Msg, workers*2)
 
+	for _, sc := range conf.Subscriptions {
+		clusterTag := sc.ClusterTag
+		var sub *nats.Subscription
 		if workers > 1 {
 			wg.Add(workers)
 
@@ -44,7 +47,7 @@ func ReceiveNats(conf *NatsConfig, handleLine func(*lineprotocol.Decoder, string
 				go func() {
 					for m := range msgs {
 						dec := lineprotocol.NewDecoderWithBytes(m.Data)
-					if err := handleLine(dec, conf.ClusterTag); err != nil {
+						if err := handleLine(dec, clusterTag); err != nil {
 							log.Printf("error: %s\n", err.Error())
 						}
 					}
@@ -53,13 +56,13 @@ func ReceiveNats(conf *NatsConfig, handleLine func(*lineprotocol.Decoder, string
 				}()
 			}
 
-		sub, err = nc.Subscribe(conf.SubscribeTo, func(m *nats.Msg) {
+			sub, err = nc.Subscribe(sc.SubscribeTo, func(m *nats.Msg) {
 				msgs <- m
 			})
 		} else {
-		sub, err = nc.Subscribe(conf.SubscribeTo, func(m *nats.Msg) {
+			sub, err = nc.Subscribe(sc.SubscribeTo, func(m *nats.Msg) {
 				dec := lineprotocol.NewDecoderWithBytes(m.Data)
-			if err := handleLine(dec, conf.ClusterTag); err != nil {
+				if err := handleLine(dec, clusterTag); err != nil {
 					log.Printf("error: %s\n", err.Error())
 				}
 			})
@@ -68,18 +71,20 @@ func ReceiveNats(conf *NatsConfig, handleLine func(*lineprotocol.Decoder, string
 		if err != nil {
 			return err
 		}
-
+		log.Printf("NATS subscription to '%s' on '%s' established\n", sc.SubscribeTo, conf.Address)
-	log.Printf("NATS subscription to '%s' on '%s' established\n", conf.SubscribeTo, conf.Address)
+		subs = append(subs, sub)
+	}
 
 	<-ctx.Done()
+	for _, sub := range subs {
 		err = sub.Unsubscribe()
+		if err != nil {
+			log.Printf("NATS unsubscribe failed: %s", err.Error())
+		}
+	}
 	close(msgs)
 	wg.Wait()
 
-	if err != nil {
-		return err
-	}
-
 	nc.Close()
 	log.Println("NATS connection closed")
 	return nil

metric-store.go

@@ -38,21 +38,23 @@ type NatsConfig struct {
 	// Address of the nats server
 	Address string `json:"address"`
 
+	// Username/Password, optional
+	Username string `json:"username"`
+	Password string `json:"password"`
+
+	Subscriptions []struct {
 		// Channel name
 		SubscribeTo string `json:"subscribe-to"`
 
 		// Allow lines without a cluster tag, use this as default, optional
 		ClusterTag string `json:"cluster-tag"`
-
+	} `json:"subscriptions"`
-	// Username/Password, optional
-	Username string `json:"username"`
-	Password string `json:"password"`
 }
 
 type Config struct {
 	Metrics           map[string]MetricConfig `json:"metrics"`
 	RetentionInMemory string                  `json:"retention-in-memory"`
-	Nats              *NatsConfig             `json:"nats"`
+	Nats              []*NatsConfig           `json:"nats"`
 	JwtPublicKey      string                  `json:"jwt-public-key"`
 	HttpConfig        *HttpConfig             `json:"http-api"`
 	Checkpoints       struct {
@@ -237,11 +239,13 @@ func main() {
 	}()
 
 	if conf.Nats != nil {
+		for _, natsConf := range conf.Nats {
+			// TODO: When multiple nats configs share a URL, do a single connect.
 			wg.Add(1)
-
+			nc := natsConf
 			go func() {
 				// err := ReceiveNats(conf.Nats, decodeLine, runtime.NumCPU()-1, ctx)
-			err := ReceiveNats(conf.Nats, decodeLine, 1, ctx)
+				err := ReceiveNats(nc, decodeLine, 1, ctx)
 
 				if err != nil {
 					log.Fatal(err)
@@ -249,6 +253,7 @@ func main() {
 				wg.Done()
 			}()
 		}
+	}
 
 	wg.Wait()