From 27faafef7800a31b9f0441878a1a2d8e70f2b3b3 Mon Sep 17 00:00:00 2001 From: Thomas Roehl Date: Fri, 20 Dec 2024 15:43:19 +0100 Subject: [PATCH 1/9] Use common function to add message to ILP encoder --- sinks/httpSink.go | 64 +------------------------------------------ sinks/metricSink.go | 56 +++++++++++++++++++++++++++++++++++++ sinks/natsSink.go | 67 +++++++-------------------------------------- 3 files changed, 67 insertions(+), 120 deletions(-) diff --git a/sinks/httpSink.go b/sinks/httpSink.go index 44d6dea..67dd85a 100644 --- a/sinks/httpSink.go +++ b/sinks/httpSink.go @@ -51,11 +51,6 @@ type HttpSinkConfig struct { Precision string `json:"precision,omitempty"` } -type key_value_pair struct { - key string - value string -} - type HttpSink struct { sink client *http.Client @@ -84,64 +79,7 @@ func (s *HttpSink) Write(msg lp.CCMessage) error { // Lock for encoder usage s.encoderLock.Lock() - // Encode measurement name - s.encoder.StartLine(m.Name()) - - // copy tags and meta data which should be used as tags - s.extended_tag_list = s.extended_tag_list[:0] - for key, value := range m.Tags() { - s.extended_tag_list = - append( - s.extended_tag_list, - key_value_pair{ - key: key, - value: value, - }, - ) - } - // for _, key := range s.config.MetaAsTags { - // if value, ok := m.GetMeta(key); ok { - // s.extended_tag_list = - // append( - // s.extended_tag_list, - // key_value_pair{ - // key: key, - // value: value, - // }, - // ) - // } - // } - - // Encode tags (they musts be in lexical order) - slices.SortFunc( - s.extended_tag_list, - func(a key_value_pair, b key_value_pair) int { - if a.key < b.key { - return -1 - } - if a.key > b.key { - return +1 - } - return 0 - }, - ) - for i := range s.extended_tag_list { - s.encoder.AddTag( - s.extended_tag_list[i].key, - s.extended_tag_list[i].value, - ) - } - - // Encode fields - for key, value := range m.Fields() { - s.encoder.AddField(key, influx.MustNewValue(value)) - } - - // Encode time stamp - s.encoder.EndLine(m.Time()) - - // Check for encoder errors - err := s.encoder.Err() + err = EncoderAdd(&s.encoder, m) // Unlock encoder usage s.encoderLock.Unlock() diff --git a/sinks/metricSink.go b/sinks/metricSink.go index 4cac04b..f3e95a3 100644 --- a/sinks/metricSink.go +++ b/sinks/metricSink.go @@ -5,6 +5,8 @@ import ( lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message" mp "github.com/ClusterCockpit/cc-metric-collector/pkg/messageProcessor" + influx "github.com/influxdata/line-protocol/v2/lineprotocol" + "golang.org/x/exp/slices" ) type defaultSinkConfig struct { @@ -30,3 +32,57 @@ type Sink interface { func (s *sink) Name() string { return s.name } + +type key_value_pair struct { + key string + value string +} + +func EncoderAdd(encoder *influx.Encoder, msg lp.CCMessage) error { + // Encode measurement name + encoder.StartLine(msg.Name()) + + tag_list := make([]key_value_pair, 0, 10) + + // copy tags and meta data which should be used as tags + for key, value := range msg.Tags() { + tag_list = + append( + tag_list, + key_value_pair{ + key: key, + value: value, + }, + ) + } + // Encode tags (they musts be in lexical order) + slices.SortFunc( + tag_list, + func(a key_value_pair, b key_value_pair) int { + if a.key < b.key { + return -1 + } + if a.key > b.key { + return +1 + } + return 0 + }, + ) + for i := range tag_list { + encoder.AddTag( + tag_list[i].key, + tag_list[i].value, + ) + } + + // Encode fields + for key, value := range msg.Fields() { + encoder.AddField(key, influx.MustNewValue(value)) + } + + // Encode time stamp + encoder.EndLine(msg.Time()) + + // Return encoder errors + return encoder.Err() +} diff --git a/sinks/natsSink.go b/sinks/natsSink.go index 1982bfe..b2f0b18 100644 --- a/sinks/natsSink.go +++ b/sinks/natsSink.go @@ -30,19 +30,16 @@ type NatsSinkConfig struct { Precision string `json:"precision,omitempty"` } - type NatsSink struct { sink - client *nats.Conn - encoder influx.Encoder - buffer *bytes.Buffer - config NatsSinkConfig + client *nats.Conn + encoder influx.Encoder + encoderLock sync.Mutex + config NatsSinkConfig - lock sync.Mutex flushDelay time.Duration flushTimer *time.Timer - - extended_tag_list []key_value_pair + //timerLock sync.Mutex } func (s *NatsSink) connect() error { @@ -78,54 +75,11 @@ func (s *NatsSink) connect() error { func (s *NatsSink) Write(m lp.CCMessage) error { msg, err := s.mp.ProcessMessage(m) if err == nil && msg != nil { - s.lock.Lock() - // Encode measurement name - s.encoder.StartLine(msg.Name()) + s.encoderLock.Lock() - // copy tags and meta data which should be used as tags - s.extended_tag_list = s.extended_tag_list[:0] - for key, value := range m.Tags() { - s.extended_tag_list = - append( - s.extended_tag_list, - key_value_pair{ - key: key, - value: value, - }, - ) - } - // Encode tags (they musts be in lexical order) - slices.SortFunc( - s.extended_tag_list, - func(a key_value_pair, b key_value_pair) int { - if a.key < b.key { - return -1 - } - if a.key > b.key { - return +1 - } - return 0 - }, - ) - for i := range s.extended_tag_list { - s.encoder.AddTag( - s.extended_tag_list[i].key, - s.extended_tag_list[i].value, - ) - } + err = EncoderAdd(&s.encoder, msg) - // Encode fields - for key, value := range msg.Fields() { - s.encoder.AddField(key, influx.MustNewValue(value)) - } - - // Encode time stamp - s.encoder.EndLine(msg.Time()) - - // Check for encoder errors - err := s.encoder.Err() - - s.lock.Unlock() + s.encoderLock.Unlock() if err != nil { cclog.ComponentError(s.name, "Write:", err.Error()) return err @@ -146,10 +100,10 @@ func (s *NatsSink) Write(m lp.CCMessage) error { } func (s *NatsSink) Flush() error { - s.lock.Lock() + s.encoderLock.Lock() buf := slices.Clone(s.encoder.Bytes()) s.encoder.Reset() - s.lock.Unlock() + s.encoderLock.Unlock() if len(buf) == 0 { return nil @@ -233,7 +187,6 @@ func NewNatsSink(name string, config json.RawMessage) (Sink, error) { return nil, err } } - s.extended_tag_list = make([]key_value_pair, 0) return s, nil } From 8915c2fd5d3c6d7e3c12c581f6c113713995c1b2 Mon Sep 17 00:00:00 2001 From: Thomas Roehl Date: Fri, 20 Dec 2024 17:06:50 +0100 Subject: [PATCH 2/9] httpSink: remove unused extended tag list --- sinks/httpSink.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sinks/httpSink.go b/sinks/httpSink.go index 67dd85a..8e1a8fd 100644 --- a/sinks/httpSink.go +++ b/sinks/httpSink.go @@ -56,8 +56,7 @@ type HttpSink struct { client *http.Client // influx line protocol encoder encoder influx.Encoder - // List of tags and meta data tags which should be used as tags - extended_tag_list []key_value_pair + // Flush() runs in another goroutine and accesses the influx line protocol encoder, // so this encoderLock has to protect the encoder encoderLock sync.Mutex @@ -302,7 +301,6 @@ func NewHttpSink(name string, config json.RawMessage) (Sink, error) { // Configure influx line protocol encoder s.encoder.SetPrecision(precision) - s.extended_tag_list = make([]key_value_pair, 0) return s, nil } From 87309fcd2b3f529505b8e22e82449e8e3740b8c0 Mon Sep 17 00:00:00 2001 From: Thomas Roehl Date: Fri, 20 Dec 2024 17:07:16 +0100 Subject: [PATCH 3/9] natsSink: Use flush timer handling from httpSink and some comments --- sinks/natsSink.go | 82 +++++++++++++++++++++++++++++++++-------------- sinks/natsSink.md | 2 +- 2 files changed, 59 insertions(+), 25 deletions(-) diff --git a/sinks/natsSink.go b/sinks/natsSink.go index b2f0b18..08cb4d1 100644 --- a/sinks/natsSink.go +++ b/sinks/natsSink.go @@ -25,6 +25,7 @@ type NatsSinkConfig struct { User string `json:"user,omitempty"` Password string `json:"password,omitempty"` FlushDelay string `json:"flush_delay,omitempty"` + flushDelay time.Duration NkeyFile string `json:"nkey_file,omitempty"` // Timestamp precision Precision string `json:"precision,omitempty"` @@ -37,9 +38,8 @@ type NatsSink struct { encoderLock sync.Mutex config NatsSinkConfig - flushDelay time.Duration flushTimer *time.Timer - //timerLock sync.Mutex + timerLock sync.Mutex } func (s *NatsSink) connect() error { @@ -75,39 +75,67 @@ func (s *NatsSink) connect() error { func (s *NatsSink) Write(m lp.CCMessage) error { msg, err := s.mp.ProcessMessage(m) if err == nil && msg != nil { + // Lock for encoder usage s.encoderLock.Lock() - err = EncoderAdd(&s.encoder, msg) + // Add message to encoder + err = EncoderAdd(&s.encoder, m) + // Unlock encoder usage s.encoderLock.Unlock() + + // Check that encoding worked if err != nil { cclog.ComponentError(s.name, "Write:", err.Error()) return err } } - if s.flushDelay == 0 { - s.Flush() - } else if s.flushTimer == nil { - s.flushTimer = time.AfterFunc(s.flushDelay, func() { - s.Flush() - }) - } else { - s.flushTimer.Reset(s.flushDelay) + if s.config.flushDelay == 0 { + // Directly flush if no flush delay is configured + return s.Flush() + } else if s.timerLock.TryLock() { + // Setup flush timer when flush delay is configured + // and no other timer is already running + if s.flushTimer != nil { + + // Restarting existing flush timer + cclog.ComponentDebug(s.name, "Write(): Restarting flush timer") + s.flushTimer.Reset(s.config.flushDelay) + } else { + + // Creating and starting flush timer + cclog.ComponentDebug(s.name, "Write(): Starting new flush timer") + s.flushTimer = time.AfterFunc( + s.config.flushDelay, + func() { + defer s.timerLock.Unlock() + cclog.ComponentDebug(s.name, "Starting flush triggered by flush timer") + if err := s.Flush(); err != nil { + cclog.ComponentError(s.name, "Flush triggered by flush timer: flush failed:", err) + } + }) + } } return nil } func (s *NatsSink) Flush() error { + // Lock for encoder usage + // Own lock for as short as possible: the time it takes to clone the buffer. s.encoderLock.Lock() + buf := slices.Clone(s.encoder.Bytes()) s.encoder.Reset() + + // Unlock encoder usage s.encoderLock.Unlock() if len(buf) == 0 { return nil } + if err := s.client.Publish(s.config.Subject, buf); err != nil { cclog.ComponentError(s.name, "Flush:", err.Error()) return err @@ -116,14 +144,21 @@ func (s *NatsSink) Flush() error { } func (s *NatsSink) Close() { - cclog.ComponentDebug(s.name, "Close") + // Stop existing timer and immediately flush + if s.flushTimer != nil { + if ok := s.flushTimer.Stop(); ok { + s.timerLock.Unlock() + } + } + cclog.ComponentDebug(s.name, "Close NATS connection") s.client.Close() } func NewNatsSink(name string, config json.RawMessage) (Sink, error) { s := new(NatsSink) s.name = fmt.Sprintf("NatsSink(%s)", name) - s.flushDelay = 10 * time.Second + s.config.flushDelay = 5 * time.Second + s.config.FlushDelay = "5s" s.config.Port = "4222" s.config.Precision = "s" if len(config) > 0 { @@ -139,21 +174,25 @@ func NewNatsSink(name string, config json.RawMessage) (Sink, error) { len(s.config.Subject) == 0 { return nil, errors.New("not all configuration variables set required by NatsSink") } + // Create a new message processor p, err := mp.NewMessageProcessor() if err != nil { return nil, fmt.Errorf("initialization of message processor failed: %v", err.Error()) } s.mp = p + // Read config related to message processor if len(s.config.MessageProcessor) > 0 { err = s.mp.FromConfigJSON(s.config.MessageProcessor) if err != nil { return nil, fmt.Errorf("failed parsing JSON for message processor: %v", err.Error()) } } - // Create lookup map to use meta infos as tags in the output metric + // Add meta_as_tags list to message processor for _, k := range s.config.MetaAsTags { s.mp.AddMoveMetaToTags("true", k, k) } + + // Setup Influx line protocol encoder precision := influx.Second if len(s.config.Precision) > 0 { switch s.config.Precision { @@ -168,11 +207,6 @@ func NewNatsSink(name string, config json.RawMessage) (Sink, error) { } } - // s.meta_as_tags = make(map[string]bool) - // for _, k := range s.config.MetaAsTags { - // s.meta_as_tags[k] = true - // } - // Setup Influx line protocol s.encoder.SetPrecision(precision) // Setup infos for connection if err := s.connect(); err != nil { @@ -180,11 +214,11 @@ func NewNatsSink(name string, config json.RawMessage) (Sink, error) { } s.flushTimer = nil - if len(s.config.FlushDelay) != 0 { - var err error - s.flushDelay, err = time.ParseDuration(s.config.FlushDelay) - if err != nil { - return nil, err + if len(s.config.FlushDelay) > 0 { + t, err := time.ParseDuration(s.config.FlushDelay) + if err == nil { + s.config.flushDelay = t + cclog.ComponentDebug(s.name, "Init(): flushDelay", t) } } diff --git a/sinks/natsSink.md b/sinks/natsSink.md index 8bafbb0..ef536d4 100644 --- a/sinks/natsSink.md +++ b/sinks/natsSink.md @@ -31,7 +31,7 @@ The `nats` sink publishes all metrics into a NATS network. The publishing key is - `user`: Username for basic authentication - `password`: Password for basic authentication - `nkey_file`: Path to credentials file with NKEY -- `flush_delay`: Maximum time until metrics are sent out +- `flush_delay`: Maximum time until metrics are sent out (default '5s') - `precision`: Precision of the timestamp. Valid values are 's', 'ms', 'us' and 'ns'. (default is 's') - `process_messages`: Process messages with given rules before progressing or dropping, see [here](../pkg/messageProcessor/README.md) (optional) - `meta_as_tags`: print all meta information as tags in the output (deprecated, optional) From 31994f44faa29ee99921428cdc6ddbbd9a3f7351 Mon Sep 17 00:00:00 2001 From: Thomas Roehl Date: Fri, 20 Dec 2024 17:32:01 +0100 Subject: [PATCH 4/9] Update Github Action with new OS versions and action versions --- .github/workflows/Release.yml | 225 ++++++++++++++++++++++++++-------- .github/workflows/runonce.yml | 36 +++++- 2 files changed, 207 insertions(+), 54 deletions(-) diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml index 36b812e..5bb73ca 100644 --- a/.github/workflows/Release.yml +++ b/.github/workflows/Release.yml @@ -15,7 +15,7 @@ jobs: # # Build on AlmaLinux 8 using go-toolset # - AlmaLinux-RPM-build: + AlmaLinux8-RPM-build: runs-on: ubuntu-latest # See: https://hub.docker.com/_/almalinux container: almalinux:8 @@ -41,19 +41,15 @@ jobs: submodules: recursive fetch-depth: 0 - # Use dnf to install build dependencies - - name: Install build dependencies - run: | - wget -q https://go.dev/dl/go1.22.4.linux-amd64.tar.gz --output-document=- | \ - tar --directory=/usr/local --extract --gzip - export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH - go version + - name: Setup Golang + uses: actions/setup-go@v5 + with: + go-version: 'stable' - name: RPM build MetricCollector id: rpmbuild run: | git config --global --add safe.directory /__w/cc-metric-collector/cc-metric-collector - export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH make RPM # AlmaLinux 8 is a derivate of RedHat Enterprise Linux 8 (UBI8), @@ -70,21 +66,90 @@ jobs: NEW_SRPM=${OLD_SRPM/el8/alma8} mv "${OLD_RPM}" "${NEW_RPM}" mv "${OLD_SRPM}" "${NEW_SRPM}" - echo "::set-output name=SRPM::${NEW_SRPM}" - echo "::set-output name=RPM::${NEW_RPM}" + echo "SRPM=${NEW_SRPM}" >> $GITHUB_OUTPUT + echo "RPM=${NEW_RPM}" >> $GITHUB_OUTPUT # See: https://github.com/actions/upload-artifact - name: Save RPM as artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: cc-metric-collector RPM for AlmaLinux 8 path: ${{ steps.rpmrename.outputs.RPM }} - name: Save SRPM as artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: cc-metric-collector SRPM for AlmaLinux 8 path: ${{ steps.rpmrename.outputs.SRPM }} + # + # Build on AlmaLinux 9 using go-toolset + # + AlmaLinux8-RPM-build: + runs-on: ubuntu-latest + # See: https://hub.docker.com/_/almalinux + container: almalinux:9 + # The job outputs link to the outputs of the 'rpmrename' step + # Only job outputs can be used in child jobs + outputs: + rpm : ${{steps.rpmrename.outputs.RPM}} + srpm : ${{steps.rpmrename.outputs.SRPM}} + steps: + + # Use dnf to install development packages + - name: Install development packages + run: | + dnf --assumeyes group install "Development Tools" "RPM Development Tools" + dnf --assumeyes install wget openssl-devel diffutils delve which + + # Checkout git repository and submodules + # fetch-depth must be 0 to use git describe + # See: https://github.com/marketplace/actions/checkout + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + - name: Setup Golang + uses: actions/setup-go@v5 + with: + go-version: 'stable' + + - name: RPM build MetricCollector + id: rpmbuild + run: | + git config --global --add safe.directory /__w/cc-metric-collector/cc-metric-collector + make RPM + + # AlmaLinux 9 is a derivate of RedHat Enterprise Linux 8 (UBI8), + # so the created RPM both contain the substring 'el9' in the RPM file names + # This step replaces the substring 'el8' to 'alma8'. It uses the move operation + # because it is unclear whether the default AlmaLinux 8 container contains the + # 'rename' command. This way we also get the new names for output. + - name: Rename RPMs (s/el9/alma9/) + id: rpmrename + run: | + OLD_RPM="${{steps.rpmbuild.outputs.RPM}}" + OLD_SRPM="${{steps.rpmbuild.outputs.SRPM}}" + NEW_RPM="${OLD_RPM/el9/alma9}" + NEW_SRPM=${OLD_SRPM/el9/alma9} + mv "${OLD_RPM}" "${NEW_RPM}" + mv "${OLD_SRPM}" "${NEW_SRPM}" + echo "SRPM=${NEW_SRPM}" >> $GITHUB_OUTPUT + echo "RPM=${NEW_RPM}" >> $GITHUB_OUTPUT + + # See: https://github.com/actions/upload-artifact + - name: Save RPM as artifact + uses: actions/upload-artifact@v4 + with: + name: cc-metric-collector RPM for AlmaLinux 9 + path: ${{ steps.rpmrename.outputs.RPM }} + - name: Save SRPM as artifact + uses: actions/upload-artifact@v4 + with: + name: cc-metric-collector SRPM for AlmaLinux 9 + path: ${{ steps.rpmrename.outputs.SRPM }} + # # Build on UBI 8 using go-toolset # @@ -111,29 +176,25 @@ jobs: submodules: recursive fetch-depth: 0 - # Use dnf to install build dependencies - - name: Install build dependencies - run: | - wget -q https://go.dev/dl/go1.22.4.linux-amd64.tar.gz --output-document=- | \ - tar --directory=/usr/local --extract --gzip - export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH - go version + - name: Setup Golang + uses: actions/setup-go@v5 + with: + go-version: 'stable' - name: RPM build MetricCollector id: rpmbuild run: | git config --global --add safe.directory /__w/cc-metric-collector/cc-metric-collector - export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH make RPM # See: https://github.com/actions/upload-artifact - name: Save RPM as artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: cc-metric-collector RPM for UBI 8 path: ${{ steps.rpmbuild.outputs.RPM }} - name: Save SRPM as artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: cc-metric-collector SRPM for UBI 8 path: ${{ steps.rpmbuild.outputs.SRPM }} @@ -162,17 +223,15 @@ jobs: with: submodules: recursive fetch-depth: 0 - # Use official golang package - - name: Install Golang - run: | - wget -q https://go.dev/dl/go1.22.4.linux-amd64.tar.gz --output-document=- | \ - tar --directory=/usr/local --extract --gzip - export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH - go version + + - name: Setup Golang + uses: actions/setup-go@v5 + with: + go-version: 'stable' + - name: DEB build MetricCollector id: dpkg-build run: | - export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH git config --global --add safe.directory /__w/cc-metric-collector/cc-metric-collector make DEB - name: Rename DEB (add '_ubuntu22.04') @@ -181,47 +240,111 @@ jobs: OLD_DEB_NAME=$(echo "${{steps.dpkg-build.outputs.DEB}}" | rev | cut -d '.' -f 2- | rev) NEW_DEB_FILE="${OLD_DEB_NAME}_ubuntu22.04.deb" mv "${{steps.dpkg-build.outputs.DEB}}" "${NEW_DEB_FILE}" - echo "::set-output name=DEB::${NEW_DEB_FILE}" + echo "DEB=${NEW_DEB_FILE}" >> $GITHUB_OUTPUT # See: https://github.com/actions/upload-artifact - name: Save DEB as artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: cc-metric-collector DEB for Ubuntu 22.04 path: ${{ steps.debrename.outputs.DEB }} + # + # Build on Ubuntu 24.04 using official go package + # + Ubuntu-noblenumbat-build: + runs-on: ubuntu-latest + container: ubuntu:24.04 + # The job outputs link to the outputs of the 'debrename' step + # Only job outputs can be used in child jobs + outputs: + deb : ${{steps.debrename.outputs.DEB}} + steps: + # Use apt to install development packages + - name: Install development packages + run: | + apt update && apt --assume-yes upgrade + apt --assume-yes install build-essential sed git wget bash + # Checkout git repository and submodules + # fetch-depth must be 0 to use git describe + # See: https://github.com/marketplace/actions/checkout + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + - name: Setup Golang + uses: actions/setup-go@v5 + with: + go-version: 'stable' + + - name: DEB build MetricCollector + id: dpkg-build + run: | + git config --global --add safe.directory /__w/cc-metric-collector/cc-metric-collector + make DEB + - name: Rename DEB (add '_ubuntu24.04') + id: debrename + run: | + OLD_DEB_NAME=$(echo "${{steps.dpkg-build.outputs.DEB}}" | rev | cut -d '.' -f 2- | rev) + NEW_DEB_FILE="${OLD_DEB_NAME}_ubuntu24.04.deb" + mv "${{steps.dpkg-build.outputs.DEB}}" "${NEW_DEB_FILE}" + echo "DEB=${NEW_DEB_FILE}" >> $GITHUB_OUTPUT + # See: https://github.com/actions/upload-artifact + - name: Save DEB as artifact + uses: actions/upload-artifact@v4 + with: + name: cc-metric-collector DEB for Ubuntu 24.04 + path: ${{ steps.debrename.outputs.DEB }} + + # # Create release with fresh RPMs # Release: runs-on: ubuntu-latest # We need the RPMs, so add dependency - needs: [AlmaLinux-RPM-build, UBI-8-RPM-build, Ubuntu-jammy-build] + needs: [AlmaLinux8-RPM-build, AlmaLinux9-RPM-build, UBI-8-RPM-build, Ubuntu-jammy-build, Ubuntu-noblenumbat-build] steps: # See: https://github.com/actions/download-artifact - name: Download AlmaLinux 8 RPM - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: cc-metric-collector RPM for AlmaLinux 8 - name: Download AlmaLinux 8 SRPM - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: cc-metric-collector SRPM for AlmaLinux 8 + - name: Download AlmaLinux 9 RPM + uses: actions/download-artifact@v4 + with: + name: cc-metric-collector RPM for AlmaLinux 9 + - name: Download AlmaLinux 9 SRPM + uses: actions/download-artifact@v4 + with: + name: cc-metric-collector SRPM for AlmaLinux 9 + - name: Download UBI 8 RPM - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: cc-metric-collector RPM for UBI 8 - name: Download UBI 8 SRPM - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: cc-metric-collector SRPM for UBI 8 - name: Download Ubuntu 22.04 DEB - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: cc-metric-collector DEB for Ubuntu 22.04 + - name: Download Ubuntu 24.04 DEB + uses: actions/download-artifact@v4 + with: + name: cc-metric-collector DEB for Ubuntu 24.04 + # The download actions do not publish the name of the downloaded file, # so we re-use the job outputs of the parent jobs. The files are all # downloaded to the current folder. @@ -231,21 +354,24 @@ jobs: - name: Set RPM variables id: files run: | - ALMA_8_RPM=$(basename "${{ needs.AlmaLinux-RPM-build.outputs.rpm}}") - ALMA_8_SRPM=$(basename "${{ needs.AlmaLinux-RPM-build.outputs.srpm}}") + ALMA_8_RPM=$(basename "${{ needs.AlmaLinux8-RPM-build.outputs.rpm}}") + ALMA_8_SRPM=$(basename "${{ needs.AlmaLinux8-RPM-build.outputs.srpm}}") UBI_8_RPM=$(basename "${{ needs.UBI-8-RPM-build.outputs.rpm}}") UBI_8_SRPM=$(basename "${{ needs.UBI-8-RPM-build.outputs.srpm}}") - U_2004_DEB=$(basename "${{ needs.Ubuntu-focal-build.outputs.deb}}") + U_2204_DEB=$(basename "${{ needs.Ubuntu-jammy-build.outputs.deb}}") + U_2404_DEB=$(basename "${{ needs.Ubuntu-jammy-build.outputs.deb}}") echo "ALMA_8_RPM::${ALMA_8_RPM}" echo "ALMA_8_SRPM::${ALMA_8_SRPM}" echo "UBI_8_RPM::${UBI_8_RPM}" echo "UBI_8_SRPM::${UBI_8_SRPM}" - echo "U_2004_DEB::${U_2004_DEB}" - echo "::set-output name=ALMA_8_RPM::${ALMA_8_RPM}" - echo "::set-output name=ALMA_8_SRPM::${ALMA_8_SRPM}" - echo "::set-output name=UBI_8_RPM::${UBI_8_RPM}" - echo "::set-output name=UBI_8_SRPM::${UBI_8_SRPM}" - echo "::set-output name=U_2004_DEB::${U_2004_DEB}" + echo "U_2204_DEB::${U_2204_DEB}" + echo "U_2404_DEB::${U_2404_DEB}" + echo "ALMA_8_RPM=${ALMA_8_RPM}" >> $GITHUB_OUTPUT + echo "ALMA_8_SRPM=${ALMA_8_SRPM}" >> $GITHUB_OUTPUT + echo "UBI_8_RPM=${UBI_8_RPM}" >> $GITHUB_OUTPUT + echo "UBI_8_SRPM=${UBI_8_SRPM}" >> $GITHUB_OUTPUT + echo "U_2204_DEB=${U_2204_DEB}" >> $GITHUB_OUTPUT + echo "U_2404_DEB=${U_2404_DEB}" >> $GITHUB_OUTPUT # See: https://github.com/softprops/action-gh-release - name: Release @@ -258,4 +384,5 @@ jobs: ${{ steps.files.outputs.ALMA_8_SRPM }} ${{ steps.files.outputs.UBI_8_RPM }} ${{ steps.files.outputs.UBI_8_SRPM }} - ${{ steps.files.outputs.U_2004_DEB }} + ${{ steps.files.outputs.U_2204_DEB }} + ${{ steps.files.outputs.U_2404_DEB }} diff --git a/.github/workflows/runonce.yml b/.github/workflows/runonce.yml index 182e043..d50eb9f 100644 --- a/.github/workflows/runonce.yml +++ b/.github/workflows/runonce.yml @@ -26,7 +26,7 @@ jobs: # See: https://github.com/marketplace/actions/setup-go-environment - name: Setup Golang - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: '1.21' @@ -52,7 +52,7 @@ jobs: # See: https://github.com/marketplace/actions/setup-go-environment - name: Setup Golang - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: '1.22' @@ -62,6 +62,32 @@ jobs: - name: Run MetricCollector once run: ./cc-metric-collector --once --config .github/ci-config.json + # + # Job build-1-23 + # Build on latest Ubuntu using golang version 1.23 + # + build-1-23: + runs-on: ubuntu-latest + steps: + # See: https://github.com/marketplace/actions/checkout + # Checkout git repository and submodules + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + # See: https://github.com/marketplace/actions/setup-go-environment + - name: Setup Golang + uses: actions/setup-go@v5 + with: + go-version: '1.23' + + - name: Build MetricCollector + run: make + + - name: Run MetricCollector once + run: ./cc-metric-collector --once --config .github/ci-config.json + # # Build on AlmaLinux 8 using go-toolset # @@ -90,7 +116,7 @@ jobs: # See: https://github.com/marketplace/actions/setup-go-environment - name: Setup Golang - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: '1.21' @@ -126,7 +152,7 @@ jobs: # See: https://github.com/marketplace/actions/setup-go-environment - name: Setup Golang - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: '1.21' @@ -161,7 +187,7 @@ jobs: # Use official golang package # See: https://github.com/marketplace/actions/setup-go-environment - name: Setup Golang - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: '1.21' From ea04b7ed280876faf62860a4bfca2618e47b2f76 Mon Sep 17 00:00:00 2001 From: Thomas Roehl Date: Fri, 20 Dec 2024 17:32:52 +0100 Subject: [PATCH 5/9] Fix job name --- .github/workflows/Release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml index 5bb73ca..2ada7d1 100644 --- a/.github/workflows/Release.yml +++ b/.github/workflows/Release.yml @@ -84,7 +84,7 @@ jobs: # # Build on AlmaLinux 9 using go-toolset # - AlmaLinux8-RPM-build: + AlmaLinux9-RPM-build: runs-on: ubuntu-latest # See: https://hub.docker.com/_/almalinux container: almalinux:9 From 100d306473c5c0d3c4614ea3da9b8c615a08d72e Mon Sep 17 00:00:00 2001 From: Thomas Roehl Date: Fri, 20 Dec 2024 17:36:54 +0100 Subject: [PATCH 6/9] Add more test builds to runonce workflow --- .github/workflows/runonce.yml | 123 +++++++++++++++++++++++++++++++--- 1 file changed, 115 insertions(+), 8 deletions(-) diff --git a/.github/workflows/runonce.yml b/.github/workflows/runonce.yml index d50eb9f..b613d08 100644 --- a/.github/workflows/runonce.yml +++ b/.github/workflows/runonce.yml @@ -89,9 +89,9 @@ jobs: run: ./cc-metric-collector --once --config .github/ci-config.json # - # Build on AlmaLinux 8 using go-toolset + # Build on AlmaLinux 8 # - AlmaLinux-RPM-build: + AlmaLinux8-RPM-build: runs-on: ubuntu-latest # See: https://hub.docker.com/_/almalinux container: almalinux:8 @@ -118,13 +118,88 @@ jobs: - name: Setup Golang uses: actions/setup-go@v5 with: - go-version: '1.21' + go-version: 'stable' + + - name: RPM build MetricCollector + id: rpmbuild + run: | + git config --global --add safe.directory /__w/cc-metric-collector/cc-metric-collector + make RPM + + # + # Build on AlmaLinux 9 + # + AlmaLinux9-RPM-build: + runs-on: ubuntu-latest + # See: https://hub.docker.com/_/almalinux + container: almalinux:9 + # The job outputs link to the outputs of the 'rpmrename' step + # Only job outputs can be used in child jobs + steps: + + # Use dnf to install development packages + - name: Install development packages + run: | + dnf --assumeyes group install "Development Tools" "RPM Development Tools" + dnf --assumeyes install wget openssl-devel diffutils delve which + + # Checkout git repository and submodules + # fetch-depth must be 0 to use git describe + # See: https://github.com/marketplace/actions/checkout + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + # See: https://github.com/marketplace/actions/setup-go-environment + - name: Setup Golang + uses: actions/setup-go@v5 + with: + go-version: 'stable' + + - name: RPM build MetricCollector + id: rpmbuild + run: | + git config --global --add safe.directory /__w/cc-metric-collector/cc-metric-collector + make RPM + +# + # Build on AlmaLinux 9 + # + AlmaLinux10-RPM-build: + runs-on: ubuntu-latest + # See: https://hub.docker.com/_/almalinux + container: almalinux:10 + # The job outputs link to the outputs of the 'rpmrename' step + # Only job outputs can be used in child jobs + steps: + + # Use dnf to install development packages + - name: Install development packages + run: | + dnf --assumeyes group install "Development Tools" "RPM Development Tools" + dnf --assumeyes install wget openssl-devel diffutils delve which + + # Checkout git repository and submodules + # fetch-depth must be 0 to use git describe + # See: https://github.com/marketplace/actions/checkout + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + # See: https://github.com/marketplace/actions/setup-go-environment + - name: Setup Golang + uses: actions/setup-go@v5 + with: + go-version: 'stable' - name: RPM build MetricCollector id: rpmbuild run: | git config --global --add safe.directory /__w/cc-metric-collector/cc-metric-collector - export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH make RPM # @@ -154,13 +229,12 @@ jobs: - name: Setup Golang uses: actions/setup-go@v5 with: - go-version: '1.21' + go-version: 'stable' - name: RPM build MetricCollector id: rpmbuild run: | git config --global --add safe.directory /__w/cc-metric-collector/cc-metric-collector - export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH make RPM # @@ -189,11 +263,44 @@ jobs: - name: Setup Golang uses: actions/setup-go@v5 with: - go-version: '1.21' + go-version: 'stable' - name: DEB build MetricCollector id: dpkg-build run: | export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH - git config --global --add safe.directory /__w/cc-metric-collector/cc-metric-collector make DEB + + # + # Build on Ubuntu 24.04 using official go package + # + Ubuntu-noblenumbat-build: + runs-on: ubuntu-latest + container: ubuntu:24.04 + + steps: + # Use apt to install development packages + - name: Install development packages + run: | + apt update && apt --assume-yes upgrade + apt --assume-yes install build-essential sed git wget bash + # Checkout git repository and submodules + # fetch-depth must be 0 to use git describe + # See: https://github.com/marketplace/actions/checkout + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + # Use official golang package + # See: https://github.com/marketplace/actions/setup-go-environment + - name: Setup Golang + uses: actions/setup-go@v5 + with: + go-version: 'stable' + + - name: DEB build MetricCollector + id: dpkg-build + run: | + export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH + make DEB \ No newline at end of file From b3f1b6361756edd5b1486e7ecf5a3a77771a42c2 Mon Sep 17 00:00:00 2001 From: Thomas Roehl Date: Fri, 20 Dec 2024 17:43:14 +0100 Subject: [PATCH 7/9] Add UBI9 build and different container sources for UBI images --- .github/workflows/runonce.yml | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/.github/workflows/runonce.yml b/.github/workflows/runonce.yml index b613d08..5cc01f2 100644 --- a/.github/workflows/runonce.yml +++ b/.github/workflows/runonce.yml @@ -164,22 +164,20 @@ jobs: git config --global --add safe.directory /__w/cc-metric-collector/cc-metric-collector make RPM -# - # Build on AlmaLinux 9 + # - AlmaLinux10-RPM-build: + # Build on UBI 8 using go-toolset + # + UBI-8-RPM-build: runs-on: ubuntu-latest - # See: https://hub.docker.com/_/almalinux - container: almalinux:10 - # The job outputs link to the outputs of the 'rpmrename' step - # Only job outputs can be used in child jobs + # See: https://catalog.redhat.com/software/containers/ubi8/ubi/5c359854d70cc534b3a3784e?container-tabs=gti + container: redhat/ubi8 + # The job outputs link to the outputs of the 'rpmbuild' step steps: # Use dnf to install development packages - name: Install development packages - run: | - dnf --assumeyes group install "Development Tools" "RPM Development Tools" - dnf --assumeyes install wget openssl-devel diffutils delve which + run: dnf --assumeyes --disableplugin=subscription-manager install rpm-build go-srpm-macros rpm-build-libs rpm-libs gcc make python38 git wget openssl-devel diffutils delve which # Checkout git repository and submodules # fetch-depth must be 0 to use git describe @@ -203,12 +201,12 @@ jobs: make RPM # - # Build on UBI 8 using go-toolset + # Build on UBI 9 using go-toolset # - UBI-8-RPM-build: + UBI-9-RPM-build: runs-on: ubuntu-latest # See: https://catalog.redhat.com/software/containers/ubi8/ubi/5c359854d70cc534b3a3784e?container-tabs=gti - container: registry.access.redhat.com/ubi8/ubi:8.8-1032.1692772289 + container: redhat/ubi9 # The job outputs link to the outputs of the 'rpmbuild' step steps: From c2c8f3c73ea2f3767a264e31806167e781ddff9c Mon Sep 17 00:00:00 2001 From: Thomas Roehl Date: Fri, 20 Dec 2024 17:46:46 +0100 Subject: [PATCH 8/9] Fix dependency installation for UBI9 --- .github/workflows/runonce.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runonce.yml b/.github/workflows/runonce.yml index 5cc01f2..4d7b596 100644 --- a/.github/workflows/runonce.yml +++ b/.github/workflows/runonce.yml @@ -212,7 +212,7 @@ jobs: # Use dnf to install development packages - name: Install development packages - run: dnf --assumeyes --disableplugin=subscription-manager install rpm-build go-srpm-macros rpm-build-libs rpm-libs gcc make python38 git wget openssl-devel diffutils delve which + run: dnf --assumeyes --disableplugin=subscription-manager install rpm-build go-srpm-macros gcc make python39 git wget openssl-devel diffutils delve # Checkout git repository and submodules # fetch-depth must be 0 to use git describe From 02344f30a4572614c5eaca4c246f0413d396044d Mon Sep 17 00:00:00 2001 From: Thomas Roehl Date: Fri, 20 Dec 2024 17:52:09 +0100 Subject: [PATCH 9/9] Add Alma9, UBI9 and Ubuntu 24.04 to release workflow --- .github/workflows/Release.yml | 72 ++++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml index 2ada7d1..219ea0d 100644 --- a/.github/workflows/Release.yml +++ b/.github/workflows/Release.yml @@ -199,6 +199,53 @@ jobs: name: cc-metric-collector SRPM for UBI 8 path: ${{ steps.rpmbuild.outputs.SRPM }} + # + # Build on UBI 9 using go-toolset + # + UBI-9-RPM-build: + runs-on: ubuntu-latest + # See: https://catalog.redhat.com/software/containers/ubi8/ubi/5c359854d70cc534b3a3784e?container-tabs=gti + container: redhat/ubi9 + # The job outputs link to the outputs of the 'rpmbuild' step + steps: + + # Use dnf to install development packages + - name: Install development packages + run: dnf --assumeyes --disableplugin=subscription-manager install rpm-build go-srpm-macros gcc make python39 git wget openssl-devel diffutils delve + + # Checkout git repository and submodules + # fetch-depth must be 0 to use git describe + # See: https://github.com/marketplace/actions/checkout + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + # See: https://github.com/marketplace/actions/setup-go-environment + - name: Setup Golang + uses: actions/setup-go@v5 + with: + go-version: 'stable' + + - name: RPM build MetricCollector + id: rpmbuild + run: | + git config --global --add safe.directory /__w/cc-metric-collector/cc-metric-collector + make RPM + + # See: https://github.com/actions/upload-artifact + - name: Save RPM as artifact + uses: actions/upload-artifact@v4 + with: + name: cc-metric-collector RPM for UBI 8 + path: ${{ steps.rpmbuild.outputs.RPM }} + - name: Save SRPM as artifact + uses: actions/upload-artifact@v4 + with: + name: cc-metric-collector SRPM for UBI 8 + path: ${{ steps.rpmbuild.outputs.SRPM }} + # # Build on Ubuntu 22.04 using official go package # @@ -304,7 +351,7 @@ jobs: Release: runs-on: ubuntu-latest # We need the RPMs, so add dependency - needs: [AlmaLinux8-RPM-build, AlmaLinux9-RPM-build, UBI-8-RPM-build, Ubuntu-jammy-build, Ubuntu-noblenumbat-build] + needs: [AlmaLinux8-RPM-build, AlmaLinux9-RPM-build, UBI-8-RPM-build, UBI-9-RPM-build, Ubuntu-jammy-build, Ubuntu-noblenumbat-build] steps: # See: https://github.com/actions/download-artifact @@ -335,6 +382,15 @@ jobs: with: name: cc-metric-collector SRPM for UBI 8 + - name: Download UBI 9 RPM + uses: actions/download-artifact@v4 + with: + name: cc-metric-collector RPM for UBI 9 + - name: Download UBI 9 SRPM + uses: actions/download-artifact@v4 + with: + name: cc-metric-collector SRPM for UBI 9 + - name: Download Ubuntu 22.04 DEB uses: actions/download-artifact@v4 with: @@ -356,33 +412,45 @@ jobs: run: | ALMA_8_RPM=$(basename "${{ needs.AlmaLinux8-RPM-build.outputs.rpm}}") ALMA_8_SRPM=$(basename "${{ needs.AlmaLinux8-RPM-build.outputs.srpm}}") + ALMA_9_RPM=$(basename "${{ needs.AlmaLinux9-RPM-build.outputs.rpm}}") + ALMA_9_SRPM=$(basename "${{ needs.AlmaLinux9-RPM-build.outputs.srpm}}") UBI_8_RPM=$(basename "${{ needs.UBI-8-RPM-build.outputs.rpm}}") UBI_8_SRPM=$(basename "${{ needs.UBI-8-RPM-build.outputs.srpm}}") U_2204_DEB=$(basename "${{ needs.Ubuntu-jammy-build.outputs.deb}}") U_2404_DEB=$(basename "${{ needs.Ubuntu-jammy-build.outputs.deb}}") echo "ALMA_8_RPM::${ALMA_8_RPM}" echo "ALMA_8_SRPM::${ALMA_8_SRPM}" + echo "ALMA_9_RPM::${ALMA_9_RPM}" + echo "ALMA_9_SRPM::${ALMA_9_SRPM}" echo "UBI_8_RPM::${UBI_8_RPM}" echo "UBI_8_SRPM::${UBI_8_SRPM}" + echo "UBI_9_RPM::${UBI_9_RPM}" + echo "UBI_9_SRPM::${UBI_9_SRPM}" echo "U_2204_DEB::${U_2204_DEB}" echo "U_2404_DEB::${U_2404_DEB}" echo "ALMA_8_RPM=${ALMA_8_RPM}" >> $GITHUB_OUTPUT echo "ALMA_8_SRPM=${ALMA_8_SRPM}" >> $GITHUB_OUTPUT echo "UBI_8_RPM=${UBI_8_RPM}" >> $GITHUB_OUTPUT echo "UBI_8_SRPM=${UBI_8_SRPM}" >> $GITHUB_OUTPUT + echo "UBI_9_RPM=${UBI_9_RPM}" >> $GITHUB_OUTPUT + echo "UBI_9_SRPM=${UBI_9_SRPM}" >> $GITHUB_OUTPUT echo "U_2204_DEB=${U_2204_DEB}" >> $GITHUB_OUTPUT echo "U_2404_DEB=${U_2404_DEB}" >> $GITHUB_OUTPUT # See: https://github.com/softprops/action-gh-release - name: Release - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 if: startsWith(github.ref, 'refs/tags/') with: name: cc-metric-collector-${{github.ref_name}} files: | ${{ steps.files.outputs.ALMA_8_RPM }} ${{ steps.files.outputs.ALMA_8_SRPM }} + ${{ steps.files.outputs.ALMA_9_RPM }} + ${{ steps.files.outputs.ALMA_9_SRPM }} ${{ steps.files.outputs.UBI_8_RPM }} ${{ steps.files.outputs.UBI_8_SRPM }} + ${{ steps.files.outputs.UBI_9_RPM }} + ${{ steps.files.outputs.UBI_9_SRPM }} ${{ steps.files.outputs.U_2204_DEB }} ${{ steps.files.outputs.U_2404_DEB }}