Merge branch 'develop' into main

This commit is contained in:
Thomas Roehl 2022-03-02 10:38:17 +01:00
commit 02baef8c71
16 changed files with 287 additions and 122 deletions

61
.github/workflows/AlmaLinux.yml vendored Normal file
View File

@ -0,0 +1,61 @@
# See: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
# Workflow name
name: AlmaLinux 8.5 RPM build
# Run on event push
on: push
jobs:
#
# Build on AlmaLinux 8.5 using go-toolset
#
AlmaLinux-RPM-build:
runs-on: ubuntu-latest
# See: https://hub.docker.com/_/almalinux
container: almalinux:8.5
steps:
# Use dnf to install development packages
- name: Install development packages
run: dnf --assumeyes group install "Development Tools" "RPM Development Tools"
# Checkout git repository and submodules
# fetch-depth must be 0 to use git describe
# See: https://github.com/marketplace/actions/checkout
- name: Checkout
uses: actions/checkout@v2
with:
submodules: recursive
fetch-depth: 0
# Use dnf to install build dependencies
- name: Install build dependencies
run: dnf --assumeyes builddep scripts/cc-metric-collector.spec
- name: RPM build MetricCollector
id: rpmbuild
run: make RPM
# See: https://github.com/actions/upload-artifact
- name: Save RPM as artifact
uses: actions/upload-artifact@v2
with:
name: cc-metric-collector RPM for AlmaLinux 8.5
path: ${{ steps.rpmbuild.outputs.RPM }}
- name: Save SRPM as artifact
uses: actions/upload-artifact@v2
with:
name: cc-metric-collector SRPM for AlmaLinux 8.5
path: ${{ steps.rpmbuild.outputs.SRPM }}
# See: https://github.com/softprops/action-gh-release
- name: Release
uses: softprops/action-gh-release@v1
if: startsWith(github.ref, 'refs/tags/')
with:
name: cc-metric-collector-${{github.ref_name}}
files: |
${{ steps.rpmbuild.outputs.RPM }}
${{ steps.rpmbuild.outputs.SRPM }}

View File

@ -0,0 +1,61 @@
# See: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
# Workflow name
name: Red Hat Universal Base Image 8 RPM build
# Run on event push
on: push
jobs:
#
# Build on UBI 8 using go-toolset
#
UBI-8-RPM-build:
runs-on: ubuntu-latest
# See: https://catalog.redhat.com/software/containers/ubi8/ubi/5c359854d70cc534b3a3784e?container-tabs=gti
container: registry.access.redhat.com/ubi8/ubi:8.5-226.1645809065
steps:
# Use dnf to install development packages
- name: Install development packages
run: dnf --assumeyes --disableplugin=subscription-manager install rpm-build go-srpm-macros rpm-build-libs rpm-libs gcc make python38 git
# Checkout git repository and submodules
# fetch-depth must be 0 to use git describe
# See: https://github.com/marketplace/actions/checkout
- name: Checkout
uses: actions/checkout@v2
with:
submodules: recursive
fetch-depth: 0
# Use dnf to install build dependencies
- name: Install build dependencies
run: dnf --assumeyes --disableplugin=subscription-manager builddep scripts/cc-metric-collector.spec
- name: RPM build MetricCollector
id: rpmbuild
run: make RPM
# See: https://github.com/actions/upload-artifact
- name: Save RPM as artifact
uses: actions/upload-artifact@v2
with:
name: cc-metric-collector RPM for UBI 8
path: ${{ steps.rpmbuild.outputs.RPM }}
- name: Save SRPM as artifact
uses: actions/upload-artifact@v2
with:
name: cc-metric-collector SRPM for UBI 8
path: ${{ steps.rpmbuild.outputs.SRPM }}
# See: https://github.com/softprops/action-gh-release
- name: Release
uses: softprops/action-gh-release@v1
if: startsWith(github.ref, 'refs/tags/')
with:
name: cc-metric-collector-${{github.ref_name}}
files: |
${{ steps.rpmbuild.outputs.RPM }}
${{ steps.rpmbuild.outputs.SRPM }}

View File

@ -1,58 +0,0 @@
name: Run RPM Build
on:
push:
tags:
- '**'
jobs:
build-alma-8_5:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: TomTheBear/rpmbuild@alma8.5
id: rpm
name: Build RPM package on AlmaLinux 8.5
with:
spec_file: "./scripts/cc-metric-collector.spec"
- name: Save RPM as artifact
uses: actions/upload-artifact@v1.0.0
with:
name: cc-metric-collector RPM AlmaLinux 8.5
path: ${{ steps.rpm.outputs.rpm_path }}
- name: Save SRPM as artifact
uses: actions/upload-artifact@v1.0.0
with:
name: cc-metric-collector SRPM AlmaLinux 8.5
path: ${{ steps.rpm.outputs.source_rpm_path }}
- name: Release
uses: softprops/action-gh-release@v1
with:
name: cc-metric-collector-${{github.ref_name}}
files: |
${{ steps.rpm.outputs.source_rpm_path }}
${{ steps.rpm.outputs.rpm_path }}
build-rhel-ubi8:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: TomTheBear/rpmbuild@rh-ubi8
id: rpm
name: Build RPM package on Red Hat Universal Base Image 8
with:
spec_file: "./scripts/cc-metric-collector.spec"
- name: Save RPM as artifact
uses: actions/upload-artifact@v1.0.0
with:
name: cc-metric-collector RPM Red Hat Universal Base Image 8
path: ${{ steps.rpm.outputs.rpm_path }}
- name: Save SRPM as artifact
uses: actions/upload-artifact@v1.0.0
with:
name: cc-metric-collector SRPM Red Hat Universal Base Image 8
path: ${{ steps.rpm.outputs.source_rpm_path }}
- name: Release
uses: softprops/action-gh-release@v1
with:
files: |
${{ steps.rpm.outputs.source_rpm_path }}
${{ steps.rpm.outputs.rpm_path }}

View File

@ -1,46 +1,68 @@
# See: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
# Workflow name
name: Run Test
# Run on event push
on: push
jobs:
#
# Job build-1-17
# Build on latest Ubuntu using golang version 1.17
#
build-1-17:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
# See: https://github.com/marketplace/actions/checkout
# Checkout git repository and submodules
- name: Checkout
uses: actions/checkout@v2
with:
submodules: recursive
# See: https://github.com/marketplace/actions/setup-go-environment
- name: Setup Golang
uses: actions/setup-go@v2.1.5
uses: actions/setup-go@v2
with:
go-version: '^1.17.6'
go-version: '^1.17.7'
# Install libganglia
- name: Setup Ganglia
run: sudo apt install ganglia-monitor libganglia1
- name: Build MetricCollector
run: make
- name: Run MetricCollector
- name: Run MetricCollector once
run: ./cc-metric-collector --once --config .github/ci-config.json
#
# Job build-1-16
# Build on latest Ubuntu using golang version 1.16
#
build-1-16:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
# See: https://github.com/marketplace/actions/checkout
# Checkout git repository and submodules
- name: Checkout
uses: actions/checkout@v2
with:
submodules: recursive
# See: https://github.com/marketplace/actions/setup-go-environment
- name: Setup Golang
uses: actions/setup-go@v2.1.5
uses: actions/setup-go@v2
with:
go-version: '^1.16.7' # The version AlmaLinux 8.5 uses
# Install libganglia
- name: Setup Ganglia
run: sudo apt install ganglia-monitor libganglia1
- name: Build MetricCollector
run: make
- name: Run MetricCollectorlibganglia1
- name: Run MetricCollector once
run: ./cc-metric-collector --once --config .github/ci-config.json

View File

@ -56,3 +56,33 @@ vet:
staticcheck:
go install honnef.co/go/tools/cmd/staticcheck@latest
$$(go env GOPATH)/bin/staticcheck ./...
.ONESHELL:
.PHONY: RPM
RPM: scripts/cc-metric-collector.spec
@WORKSPACE="$${PWD}"
@SPECFILE="$${WORKSPACE}/scripts/cc-metric-collector.spec"
# Setup RPM build tree
@eval $$(rpm --eval "ARCH='%{_arch}' RPMDIR='%{_rpmdir}' SOURCEDIR='%{_sourcedir}' SPECDIR='%{_specdir}' SRPMDIR='%{_srcrpmdir}' BUILDDIR='%{_builddir}'")
@mkdir --parents --verbose "$${RPMDIR}" "$${SOURCEDIR}" "$${SPECDIR}" "$${SRPMDIR}" "$${BUILDDIR}"
# Create source tarball
@COMMITISH="HEAD"
@VERS=$$(git describe --tags $${COMMITISH})
@VERS=$${VERS#v}
@VERS=$${VERS//-/_}
@eval $$(rpmspec --query --queryformat "NAME='%{name}' VERSION='%{version}' RELEASE='%{release}' NVR='%{NVR}' NVRA='%{NVRA}'" --define="VERS $${VERS}" "$${SPECFILE}")
@PREFIX="$${NAME}-$${VERSION}"
@FORMAT="tar.gz"
@SRCFILE="$${SOURCEDIR}/$${PREFIX}.$${FORMAT}"
@git archive --verbose --format "$${FORMAT}" --prefix="$${PREFIX}/" --output="$${SRCFILE}" $${COMMITISH}
# Build RPM and SRPM
@rpmbuild -ba --define="VERS $${VERS}" --rmsource --clean "$${SPECFILE}"
# Report RPMs and SRPMs when in GitHub Workflow
@if [[ "$${GITHUB_ACTIONS}" == true ]]; then
@ RPMFILE="$${RPMDIR}/$${ARCH}/$${NVRA}.rpm"
@ SRPMFILE="$${SRPMDIR}/$${NVR}.src.rpm"
@ echo "RPM: $${RPMFILE}"
@ echo "SRPM: $${SRPMFILE}"
@ echo "::set-output name=SRPM::$${SRPMFILE}"
@ echo "::set-output name=RPM::$${RPMFILE}"
@fi

View File

@ -55,16 +55,13 @@ $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION): $(BUILD_FOLDER)/likwid-$(LIKWID_VERSIO
tar -C $(BUILD_FOLDER) -xf $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION).tar.gz
$(INSTALL_FOLDER)/liblikwid.a: $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION) $(INSTALL_FOLDER)
sed -i -e s+"PREFIX ?= .*"+"PREFIX = $(LIKWID_BASE)"+g \
-e s+"SHARED_LIBRARY = .*"+"SHARED_LIBRARY = false"+g \
-e s+"ACCESSMODE = .*"+"ACCESSMODE = $(ACCESSMODE)"+g \
-e s+"INSTALLED_ACCESSDAEMON = .*"+"INSTALLED_ACCESSDAEMON = $(DAEMON_INSTALLDIR)/likwid-accessD"+g \
$(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/config.mk
cd $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION) && make
cp $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/liblikwid.a $(INSTALL_FOLDER)
cp $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/ext/hwloc/liblikwid-hwloc.a $(INSTALL_FOLDER)
cp $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/src/includes/likwid*.h $(INSTALL_FOLDER)
cp $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/src/includes/bstrlib.h $(INSTALL_FOLDER)
cd "$(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)" && make "PREFIX=$(LIKWID_BASE)" "SHARED_LIBRARY=false" "ACCESSMODE=$(ACCESSMODE)" "INSTALLED_ACCESSDAEMON=$(DAEMON_INSTALLDIR)/likwid-accessD"
cp \
$(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/liblikwid.a \
$(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/ext/hwloc/liblikwid-hwloc.a \
$(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/src/includes/likwid*.h \
$(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/src/includes/bstrlib.h \
$(INSTALL_FOLDER)
$(DAEMON_INSTALLDIR)/likwid-accessD: $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/likwid-accessD
sudo -u $(DAEMON_USER) -g $(DAEMON_GROUP) install -m 4775 $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/likwid-accessD $(DAEMON_INSTALLDIR)/likwid-accessD

View File

@ -13,29 +13,30 @@ import (
)
type MetricCollector interface {
Name() string
Init(config json.RawMessage) error
Initialized() bool
Read(duration time.Duration, output chan lp.CCMetric)
Close()
Name() string // Name of the metric collector
Init(config json.RawMessage) error // Initialize metric collector
Initialized() bool // Is metric collector initialized?
Read(duration time.Duration, output chan lp.CCMetric) // Read metrics from metric collector
Close() // Close / finish metric collector
}
type metricCollector struct {
name string
init bool
meta map[string]string
name string // name of the metric
init bool // is metric collector initialized?
meta map[string]string // static meta data tags
}
// Name() returns the name of the metric collector
// Name returns the name of the metric collector
func (c *metricCollector) Name() string {
return c.name
}
// Setup is for future use
func (c *metricCollector) setup() error {
return nil
}
// Initialized() indicates whether the metric collector has been initialized.
// Initialized indicates whether the metric collector has been initialized
func (c *metricCollector) Initialized() bool {
return c.init
}
@ -64,6 +65,7 @@ func stringArrayContains(array []string, str string) (int, bool) {
return -1, false
}
// SocketList returns the list of physical sockets as read from /proc/cpuinfo
func SocketList() []int {
buffer, err := ioutil.ReadFile("/proc/cpuinfo")
if err != nil {
@ -89,6 +91,7 @@ func SocketList() []int {
return packs
}
// CpuList returns the list of physical CPUs (in contrast to logical CPUs) as read from /proc/cpuinfo
func CpuList() []int {
buffer, err := ioutil.ReadFile("/proc/cpuinfo")
if err != nil {
@ -117,8 +120,8 @@ func CpuList() []int {
// RemoveFromStringList removes the string r from the array of strings s
// If r is not contained in the array an error is returned
func RemoveFromStringList(s []string, r string) ([]string, error) {
for i, item := range s {
if r == item {
for i := range s {
if r == s[i] {
return append(s[:i], s[i+1:]...), nil
}
}

View File

@ -22,6 +22,13 @@ type SampleCollector struct {
tags map[string]string // default tags
}
// Functions to implement MetricCollector interface
// Init(...), Read(...), Close()
// See: metricCollector.go
// Init initializes the sample collector
// Called once by the collector manager
// All tags, meta data tags and metrics that do not change over the runtime should be set here
func (m *SampleCollector) Init(config json.RawMessage) error {
var err error = nil
// Always set the name early in Init() to use it in cclog.Component* functions
@ -56,12 +63,14 @@ func (m *SampleCollector) Init(config json.RawMessage) error {
return err
}
// Read collects all metrics belonging to the sample collector
// and sends them through the output channel to the collector manager
func (m *SampleCollector) Read(interval time.Duration, output chan lp.CCMetric) {
// Create a sample metric
timestamp := time.Now()
value := 1.0
// If you want to measure something for a specific amout of time, use interval
// If you want to measure something for a specific amount of time, use interval
// start := readState()
// time.Sleep(interval)
// stop := readState()
@ -75,6 +84,8 @@ func (m *SampleCollector) Read(interval time.Duration, output chan lp.CCMetric)
}
// Close metric collector: close network connection, close files, close libraries, ...
// Called once by the collector manager
func (m *SampleCollector) Close() {
// Unset flag
m.init = false

View File

@ -9,32 +9,20 @@ The configuration file for the receivers is a list of configurations. The `type`
```json
{
"myreceivername" : {
"type": "receiver-type",
<receiver-specific configuration>
}
}
```
## Type `nats`
This allows to specify
```json
{
"type": "nats",
"address": "<nats-URI or hostname>",
"port" : "<portnumber>",
"subject": "<subscribe topic>"
}
```
## Available receivers
The `nats` receiver subscribes to the topic `database` and listens on `address` and `port` for metrics in the InfluxDB line protocol.
- [`nats`](./natsReceiver.md): Receive metrics from the NATS network
- [`prometheus`](./prometheusReceiver.md): Scrape data from a Prometheus client
# Contributing own receivers
A receiver contains a few functions and is derived from the type `Receiver` (in `metricReceiver.go`):
* `Start() error`
* `Close()`
* `Name() string`
* `SetSink(sink chan lp.CCMetric)`
* `New<Typename>(name string, config json.RawMessage)`
The data structures should be set up in `Init()` like opening a file or server connection. The `Start()` function should either start a go routine or issue some other asynchronous mechanism for receiving metrics. The `Close()` function should tear down anything created in `Init()`.
Finally, the receiver needs to be registered in the `receiveManager.go`. There is a list of receivers called `AvailableReceivers` which is a map (`receiver_type_string` -> `pointer to NewReceiver function`). Add a new entry with a descriptive name and the new receiver.
For an example, check the [sample receiver](./sampleReceiver.go)

View File

@ -8,6 +8,7 @@ type defaultReceiverConfig struct {
Type string `json:"type"`
}
// Receiver configuration: Listen address, port
type ReceiverConfig struct {
Addr string `json:"address"`
Port string `json:"port"`
@ -23,15 +24,17 @@ type receiver struct {
type Receiver interface {
Start()
Close()
Name() string
SetSink(sink chan lp.CCMetric)
Close() // Close / finish metric receiver
Name() string // Name of the metric receiver
SetSink(sink chan lp.CCMetric) // Set sink channel
}
// Name returns the name of the metric receiver
func (r *receiver) Name() string {
return r.name
}
// SetSink set the sink channel
func (r *receiver) SetSink(sink chan lp.CCMetric) {
r.sink = sink
}

21
receivers/natsReceiver.md Normal file
View File

@ -0,0 +1,21 @@
## `nats` receiver
The `nats` receiver can be used receive metrics from the NATS network. The `nats` receiver subscribes to the topic `database` and listens on `address` and `port` for metrics in the InfluxDB line protocol.
### Configuration structure
```json
{
"<name>": {
"type": "nats",
"address" : "nats-server.example.org",
"port" : "4222",
"subject" : "subject"
}
}
```
- `type`: makes the receiver a `nats` receiver
- `address`: Address of the NATS control server
- `port`: Port of the NATS control server
- `subject`: Subscribes to this subject and receive metrics

View File

@ -10,7 +10,7 @@ The `prometheus` receiver can be used to scrape the metrics of a single `prometh
"type": "prometheus",
"address" : "testpromhost",
"port" : "12345",
"port" : "/prometheus",
"path" : "/prometheus",
"interval": "5s",
"ssl" : true,
}

View File

@ -7,6 +7,7 @@ import (
cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
)
// SampleReceiver configuration: receiver type, listen address, port
type SampleReceiverConfig struct {
Type string `json:"type"`
Addr string `json:"address"`
@ -24,6 +25,10 @@ type SampleReceiver struct {
// wg sync.WaitGroup
}
// Implement functions required for Receiver interface
// Start(), Close()
// See: metricReceiver.go
func (r *SampleReceiver) Start() {
cclog.ComponentDebug(r.name, "START")
@ -44,6 +49,7 @@ func (r *SampleReceiver) Start() {
// }()
}
// Close receiver: close network connection, close files, close libraries, ...
func (r *SampleReceiver) Close() {
cclog.ComponentDebug(r.name, "CLOSE")
@ -54,13 +60,21 @@ func (r *SampleReceiver) Close() {
// r.wg.Wait()
}
// New function to create a new instance of the receiver
// Initialize the receiver by giving it a name and reading in the config JSON
func NewSampleReceiver(name string, config json.RawMessage) (Receiver, error) {
r := new(SampleReceiver)
r.name = fmt.Sprintf("HttpReceiver(%s)", name)
// Set name of SampleReceiver
// The name should be chosen in such a way that different instances of SampleReceiver can be distinguished
r.name = fmt.Sprintf("SampleReceiver(%s)", name)
// Set static information
r.meta = map[string]string{"source": r.name}
// Set defaults in r.config
// Allow overwriting these defaults by reading config JSON
// Read the sample receiver specific JSON config
if len(config) > 0 {
err := json.Unmarshal(config, &r.config)

View File

@ -1,5 +1,5 @@
Name: cc-metric-collector
Version: 0.2
Version: %{VERS}
Release: 1%{?dist}
Summary: Metric collection daemon from the ClusterCockpit suite

View File

@ -10,17 +10,18 @@ type defaultSinkConfig struct {
}
type sink struct {
meta_as_tags bool
name string
meta_as_tags bool // Use meta data tags as tags
name string // Name of the sink
}
type Sink interface {
Write(point lp.CCMetric) error
Flush() error
Close()
Name() string
Write(point lp.CCMetric) error // Write metric to the sink
Flush() error // Flush buffered metrics
Close() // Close / finish metric sink
Name() string // Name of the metric sink
}
// Name returns the name of the metric sink
func (s *sink) Name() string {
return s.name
}

View File

@ -10,15 +10,22 @@ import (
)
type SampleSinkConfig struct {
defaultSinkConfig // defines JSON tags for 'type' and 'meta_as_tags'
// Add additional options
// defines JSON tags for 'type' and 'meta_as_tags'
// See: metricSink.go
defaultSinkConfig
// Additional config options, for SampleSink
}
type SampleSink struct {
sink // declarate 'name' and 'meta_as_tags'
// declares elements 'name' and 'meta_as_tags'
sink
config SampleSinkConfig // entry point to the SampleSinkConfig
}
// Implement functions required for Sink interface
// Write(...), Flush(), Close()
// See: metricSink.go
// Code to submit a single CCMetric to the sink
func (s *SampleSink) Write(point lp.CCMetric) error {
log.Print(point)
@ -39,9 +46,13 @@ func (s *SampleSink) Close() {
// Initialize the sink by giving it a name and reading in the config JSON
func NewSampleSink(name string, config json.RawMessage) (Sink, error) {
s := new(SampleSink)
// Set name of sampleSink
// The name should be chosen in such a way that different instances of SampleSink can be distinguished
s.name = fmt.Sprintf("SampleSink(%s)", name) // Always specify a name here
// Set defaults in s.config
// Allow overwriting these defaults by reading config JSON
// Read in the config JSON
if len(config) > 0 {
@ -52,7 +63,7 @@ func NewSampleSink(name string, config json.RawMessage) (Sink, error) {
}
// Check if all required fields in the config are set
// Use 'len(s.config.Option) > 0' for string settings
// E.g. use 'len(s.config.Option) > 0' for string settings
// Establish connection to the server, library, ...
// Check required files exist and lookup path(s) of executable(s)