diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml index f7143cb..1d1906b 100644 --- a/.github/workflows/Release.yml +++ b/.github/workflows/Release.yml @@ -133,13 +133,63 @@ jobs: name: cc-metric-collector SRPM for UBI 8 path: ${{ steps.rpmbuild.outputs.SRPM }} + # + # Build on Ubuntu 20.04 using official go package + # + Ubuntu-focal-build: + runs-on: ubuntu-latest + container: ubuntu:20.04 + # The job outputs link to the outputs of the 'debrename' step + # Only job outputs can be used in child jobs + outputs: + deb : ${{steps.debrename.outputs.DEB}} + steps: + # Use apt to install development packages + - name: Install development packages + run: | + apt update && apt --assume-yes upgrade + apt --assume-yes install build-essential sed git wget bash + # Checkout git repository and submodules + # fetch-depth must be 0 to use git describe + # See: https://github.com/marketplace/actions/checkout + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: recursive + fetch-depth: 0 + # Use official golang package + - name: Install Golang + run: | + wget -q https://go.dev/dl/go1.19.1.linux-amd64.tar.gz + tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz + export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH + go version + - name: DEB build MetricCollector + id: dpkg-build + run: | + export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH + make DEB + - name: Rename DEB (add '_ubuntu20.04') + id: debrename + run: | + OLD_DEB_NAME=$(echo "${{steps.dpkg-build.outputs.DEB}}" | rev | cut -d '.' -f 2- | rev) + NEW_DEB_FILE="${OLD_DEB_NAME}_ubuntu20.04.deb" + mv "${{steps.dpkg-build.outputs.DEB}}" "${NEW_DEB_FILE}" + echo "::set-output name=DEB::${NEW_DEB_FILE}" + # See: https://github.com/actions/upload-artifact + - name: Save DEB as artifact + uses: actions/upload-artifact@v2 + with: + name: cc-metric-collector DEB for Ubuntu 20.04 + path: ${{ steps.debrename.outputs.DEB }} + # # Create release with fresh RPMs # Release: runs-on: ubuntu-latest # We need the RPMs, so add dependency - needs: [AlmaLinux-RPM-build, UBI-8-RPM-build] + needs: [AlmaLinux-RPM-build, UBI-8-RPM-build, Ubuntu-focal-build] steps: # See: https://github.com/actions/download-artifact @@ -161,6 +211,11 @@ jobs: with: name: cc-metric-collector SRPM for UBI 8 + - name: Download Ubuntu 20.04 DEB + uses: actions/download-artifact@v2 + with: + name: cc-metric-collector DEB for Ubuntu 20.04 + # The download actions do not publish the name of the downloaded file, # so we re-use the job outputs of the parent jobs. The files are all # downloaded to the current folder. @@ -174,14 +229,17 @@ jobs: ALMA_85_SRPM=$(basename "${{ needs.AlmaLinux-RPM-build.outputs.srpm}}") UBI_8_RPM=$(basename "${{ needs.UBI-8-RPM-build.outputs.rpm}}") UBI_8_SRPM=$(basename "${{ needs.UBI-8-RPM-build.outputs.srpm}}") + U_2004_DEB=$(basename "${{ needs.Ubuntu-focal-build.outputs.deb}}") echo "ALMA_85_RPM::${ALMA_85_RPM}" echo "ALMA_85_SRPM::${ALMA_85_SRPM}" echo "UBI_8_RPM::${UBI_8_RPM}" echo "UBI_8_SRPM::${UBI_8_SRPM}" + echo "U_2004_DEB::${U_2004_DEB}" echo "::set-output name=ALMA_85_RPM::${ALMA_85_RPM}" echo "::set-output name=ALMA_85_SRPM::${ALMA_85_SRPM}" echo "::set-output name=UBI_8_RPM::${UBI_8_RPM}" echo "::set-output name=UBI_8_SRPM::${UBI_8_SRPM}" + echo "::set-output name=U_2004_DEB::${U_2004_DEB}" # See: https://github.com/softprops/action-gh-release - name: Release @@ -194,3 +252,4 @@ jobs: ${{ steps.files.outputs.ALMA_85_SRPM }} ${{ steps.files.outputs.UBI_8_RPM }} ${{ steps.files.outputs.UBI_8_SRPM }} + ${{ steps.files.outputs.U_2004_DEB }} diff --git a/cc-metric-collector.go b/cc-metric-collector.go index 42f7843..5544af8 100644 --- a/cc-metric-collector.go +++ b/cc-metric-collector.go @@ -15,10 +15,10 @@ import ( "sync" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" mr "github.com/ClusterCockpit/cc-metric-collector/internal/metricRouter" - mct "github.com/ClusterCockpit/cc-metric-collector/internal/multiChanTicker" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" + mct "github.com/ClusterCockpit/cc-metric-collector/pkg/multiChanTicker" ) type CentralConfigFile struct { diff --git a/collectors/beegfsmetaMetric.go b/collectors/beegfsmetaMetric.go index a27faf2..d202773 100644 --- a/collectors/beegfsmetaMetric.go +++ b/collectors/beegfsmetaMetric.go @@ -5,7 +5,7 @@ import ( "bytes" "encoding/json" "fmt" - "io/ioutil" + "io" "os" "os/exec" "os/user" @@ -14,8 +14,8 @@ import ( "strings" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) const DEFAULT_BEEGFS_CMD = "beegfs-ctl" @@ -115,7 +115,7 @@ func (m *BeegfsMetaCollector) Read(interval time.Duration, output chan lp.CCMetr return } //get mounpoint - buffer, _ := ioutil.ReadFile(string("/proc/mounts")) + buffer, _ := os.ReadFile(string("/proc/mounts")) mounts := strings.Split(string(buffer), "\n") var mountpoints []string for _, line := range mounts { @@ -157,9 +157,9 @@ func (m *BeegfsMetaCollector) Read(interval time.Duration, output chan lp.CCMetr if err != nil { fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): Failed to execute command \"%s\": %s\n", cmd.String(), err.Error()) fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): command exit code: \"%d\"\n", cmd.ProcessState.ExitCode()) - data, _ := ioutil.ReadAll(cmdStderr) + data, _ := io.ReadAll(cmdStderr) fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): command stderr: \"%s\"\n", string(data)) - data, _ = ioutil.ReadAll(cmdStdout) + data, _ = io.ReadAll(cmdStdout) fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): command stdout: \"%s\"\n", string(data)) return } diff --git a/collectors/beegfsstorageMetric.go b/collectors/beegfsstorageMetric.go index 1160664..be57e0f 100644 --- a/collectors/beegfsstorageMetric.go +++ b/collectors/beegfsstorageMetric.go @@ -5,7 +5,7 @@ import ( "bytes" "encoding/json" "fmt" - "io/ioutil" + "io" "os" "os/exec" "os/user" @@ -14,8 +14,8 @@ import ( "strings" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) // Struct for the collector-specific JSON config @@ -108,7 +108,7 @@ func (m *BeegfsStorageCollector) Read(interval time.Duration, output chan lp.CCM return } //get mounpoint - buffer, _ := ioutil.ReadFile(string("/proc/mounts")) + buffer, _ := os.ReadFile(string("/proc/mounts")) mounts := strings.Split(string(buffer), "\n") var mountpoints []string for _, line := range mounts { @@ -149,9 +149,9 @@ func (m *BeegfsStorageCollector) Read(interval time.Duration, output chan lp.CCM if err != nil { fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): Failed to execute command \"%s\": %s\n", cmd.String(), err.Error()) fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): command exit code: \"%d\"\n", cmd.ProcessState.ExitCode()) - data, _ := ioutil.ReadAll(cmdStderr) + data, _ := io.ReadAll(cmdStderr) fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): command stderr: \"%s\"\n", string(data)) - data, _ = ioutil.ReadAll(cmdStdout) + data, _ = io.ReadAll(cmdStdout) fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): command stdout: \"%s\"\n", string(data)) return } diff --git a/collectors/collectorManager.go b/collectors/collectorManager.go index 63d0cb4..ea648ef 100644 --- a/collectors/collectorManager.go +++ b/collectors/collectorManager.go @@ -6,9 +6,9 @@ import ( "sync" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" - mct "github.com/ClusterCockpit/cc-metric-collector/internal/multiChanTicker" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" + mct "github.com/ClusterCockpit/cc-metric-collector/pkg/multiChanTicker" ) // Map of all available metric collectors @@ -37,7 +37,8 @@ var AvailableCollectors = map[string]MetricCollector{ "beegfs_meta": new(BeegfsMetaCollector), "beegfs_storage": new(BeegfsStorageCollector), "rocm_smi": new(RocmSmiCollector), - "schedstat": new(SchedstatCollector), + "self": new(SelfCollector), + "schedstat": new(SchedstatCollector), } // Metric collector manager data structure diff --git a/collectors/cpufreqCpuinfoMetric.go b/collectors/cpufreqCpuinfoMetric.go index 80732ff..a6716d3 100644 --- a/collectors/cpufreqCpuinfoMetric.go +++ b/collectors/cpufreqCpuinfoMetric.go @@ -10,8 +10,8 @@ import ( "strings" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) // diff --git a/collectors/cpufreqMetric.go b/collectors/cpufreqMetric.go index e6a0081..cf67457 100644 --- a/collectors/cpufreqMetric.go +++ b/collectors/cpufreqMetric.go @@ -3,14 +3,14 @@ package collectors import ( "encoding/json" "fmt" - "io/ioutil" + "os" "path/filepath" "strconv" "strings" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" "golang.org/x/sys/unix" ) @@ -88,7 +88,7 @@ func (m *CPUFreqCollector) Init(config json.RawMessage) error { // Read package ID physicalPackageIDFile := filepath.Join(cpuDir, "topology", "physical_package_id") - line, err := ioutil.ReadFile(physicalPackageIDFile) + line, err := os.ReadFile(physicalPackageIDFile) if err != nil { return fmt.Errorf("unable to read physical package ID from file '%s': %v", physicalPackageIDFile, err) } @@ -100,7 +100,7 @@ func (m *CPUFreqCollector) Init(config json.RawMessage) error { // Read core ID coreIDFile := filepath.Join(cpuDir, "topology", "core_id") - line, err = ioutil.ReadFile(coreIDFile) + line, err = os.ReadFile(coreIDFile) if err != nil { return fmt.Errorf("unable to read core ID from file '%s': %v", coreIDFile, err) } @@ -188,7 +188,7 @@ func (m *CPUFreqCollector) Read(interval time.Duration, output chan lp.CCMetric) } // Read current frequency - line, err := ioutil.ReadFile(t.scalingCurFreqFile) + line, err := os.ReadFile(t.scalingCurFreqFile) if err != nil { cclog.ComponentError( m.name, diff --git a/collectors/cpustatMetric.go b/collectors/cpustatMetric.go index 3c09b83..bd6ec2f 100644 --- a/collectors/cpustatMetric.go +++ b/collectors/cpustatMetric.go @@ -9,8 +9,8 @@ import ( "strings" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" sysconf "github.com/tklauser/go-sysconf" ) @@ -22,12 +22,12 @@ type CpustatCollectorConfig struct { type CpustatCollector struct { metricCollector - config CpustatCollectorConfig - lastTimestamp time.Time // Store time stamp of last tick to derive values - matches map[string]int - cputags map[string]map[string]string - nodetags map[string]string - olddata map[string]map[string]int64 + config CpustatCollectorConfig + lastTimestamp time.Time // Store time stamp of last tick to derive values + matches map[string]int + cputags map[string]map[string]string + nodetags map[string]string + olddata map[string]map[string]int64 } func (m *CpustatCollector) Init(config json.RawMessage) error { @@ -118,7 +118,7 @@ func (m *CpustatCollector) parseStatLine(linefields []string, tags map[string]st } } } - + for name, value := range values { y, err := lp.New(name, tags, m.meta, map[string]interface{}{"value": value * 100}, now) if err == nil { diff --git a/collectors/customCmdMetric.go b/collectors/customCmdMetric.go index 492dd48..e150014 100644 --- a/collectors/customCmdMetric.go +++ b/collectors/customCmdMetric.go @@ -3,13 +3,13 @@ package collectors import ( "encoding/json" "errors" - "io/ioutil" "log" + "os" "os/exec" "strings" "time" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" influx "github.com/influxdata/line-protocol" ) @@ -53,7 +53,7 @@ func (m *CustomCmdCollector) Init(config json.RawMessage) error { } } for _, f := range m.config.Files { - _, err = ioutil.ReadFile(f) + _, err = os.ReadFile(f) if err == nil { m.files = append(m.files, f) } else { @@ -106,7 +106,7 @@ func (m *CustomCmdCollector) Read(interval time.Duration, output chan lp.CCMetri } } for _, file := range m.files { - buffer, err := ioutil.ReadFile(file) + buffer, err := os.ReadFile(file) if err != nil { log.Print(err) return diff --git a/collectors/diskstatMetric.go b/collectors/diskstatMetric.go index 69ffe07..d1ec4fc 100644 --- a/collectors/diskstatMetric.go +++ b/collectors/diskstatMetric.go @@ -8,8 +8,8 @@ import ( "syscall" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) // "log" diff --git a/collectors/gpfsMetric.go b/collectors/gpfsMetric.go index ca9affe..b3f9fc4 100644 --- a/collectors/gpfsMetric.go +++ b/collectors/gpfsMetric.go @@ -5,7 +5,7 @@ import ( "bytes" "encoding/json" "fmt" - "io/ioutil" + "io" "log" "os/exec" "os/user" @@ -13,8 +13,8 @@ import ( "strings" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) const DEFAULT_GPFS_CMD = "mmpmon" @@ -118,8 +118,8 @@ func (m *GpfsCollector) Read(interval time.Duration, output chan lp.CCMetric) { cmd.Stderr = cmdStderr err := cmd.Run() if err != nil { - dataStdErr, _ := ioutil.ReadAll(cmdStderr) - dataStdOut, _ := ioutil.ReadAll(cmdStdout) + dataStdErr, _ := io.ReadAll(cmdStderr) + dataStdOut, _ := io.ReadAll(cmdStdout) cclog.ComponentError( m.name, fmt.Sprintf("Read(): Failed to execute command \"%s\": %v\n", cmd.String(), err), diff --git a/collectors/infinibandMetric.go b/collectors/infinibandMetric.go index d6613c5..7560af9 100644 --- a/collectors/infinibandMetric.go +++ b/collectors/infinibandMetric.go @@ -2,11 +2,10 @@ package collectors import ( "fmt" - "io/ioutil" "os" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" "golang.org/x/sys/unix" "encoding/json" @@ -85,7 +84,7 @@ func (m *InfinibandCollector) Init(config json.RawMessage) error { for _, path := range ibDirs { // Skip, when no LID is assigned - line, err := ioutil.ReadFile(filepath.Join(path, "lid")) + line, err := os.ReadFile(filepath.Join(path, "lid")) if err != nil { continue } @@ -175,7 +174,7 @@ func (m *InfinibandCollector) Read(interval time.Duration, output chan lp.CCMetr for counterName, counterDef := range info.portCounterFiles { // Read counter file - line, err := ioutil.ReadFile(counterDef.path) + line, err := os.ReadFile(counterDef.path) if err != nil { cclog.ComponentError( m.name, diff --git a/collectors/iostatMetric.go b/collectors/iostatMetric.go index 19b4157..4d1dbd1 100644 --- a/collectors/iostatMetric.go +++ b/collectors/iostatMetric.go @@ -4,8 +4,8 @@ import ( "bufio" "os" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" // "log" "encoding/json" diff --git a/collectors/ipmiMetric.go b/collectors/ipmiMetric.go index 50605ac..32c4c45 100644 --- a/collectors/ipmiMetric.go +++ b/collectors/ipmiMetric.go @@ -10,7 +10,7 @@ import ( "strings" "time" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) const IPMITOOL_PATH = `ipmitool` diff --git a/collectors/likwidMetric.go b/collectors/likwidMetric.go index f22d486..265d84c 100644 --- a/collectors/likwidMetric.go +++ b/collectors/likwidMetric.go @@ -12,7 +12,6 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" "math" "os" "os/signal" @@ -24,10 +23,10 @@ import ( "time" "unsafe" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" - topo "github.com/ClusterCockpit/cc-metric-collector/internal/ccTopology" agg "github.com/ClusterCockpit/cc-metric-collector/internal/metricAggregator" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" + topo "github.com/ClusterCockpit/cc-metric-collector/pkg/ccTopology" "github.com/NVIDIA/go-nvml/pkg/dl" ) @@ -154,7 +153,7 @@ func getBaseFreq() float64 { } var freq float64 = math.NaN() for _, f := range files { - buffer, err := ioutil.ReadFile(f) + buffer, err := os.ReadFile(f) if err == nil { data := strings.Replace(string(buffer), "\n", "", -1) x, err := strconv.ParseInt(data, 0, 64) diff --git a/collectors/loadavgMetric.go b/collectors/loadavgMetric.go index 58fb102..887e63e 100644 --- a/collectors/loadavgMetric.go +++ b/collectors/loadavgMetric.go @@ -3,13 +3,13 @@ package collectors import ( "encoding/json" "fmt" - "io/ioutil" + "os" "strconv" "strings" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) // @@ -72,7 +72,7 @@ func (m *LoadavgCollector) Read(interval time.Duration, output chan lp.CCMetric) if !m.init { return } - buffer, err := ioutil.ReadFile(LOADAVGFILE) + buffer, err := os.ReadFile(LOADAVGFILE) if err != nil { if err != nil { cclog.ComponentError( diff --git a/collectors/lustreMetric.go b/collectors/lustreMetric.go index eade2ca..bcb9ca6 100644 --- a/collectors/lustreMetric.go +++ b/collectors/lustreMetric.go @@ -10,8 +10,8 @@ import ( "strings" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) const LUSTRE_SYSFS = `/sys/fs/lustre` diff --git a/collectors/memstatMetric.go b/collectors/memstatMetric.go index 9841a01..4aec4c8 100644 --- a/collectors/memstatMetric.go +++ b/collectors/memstatMetric.go @@ -12,8 +12,8 @@ import ( "strings" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) const MEMSTATFILE = "/proc/meminfo" @@ -68,7 +68,8 @@ func getStats(filename string) map[string]MemstatStats { } else if len(linefields) == 5 { v, err := strconv.ParseFloat(linefields[3], 64) if err == nil { - stats[strings.Trim(linefields[0], ":")] = MemstatStats{ + cclog.ComponentDebug("getStats", strings.Trim(linefields[2], ":"), v, linefields[4]) + stats[strings.Trim(linefields[2], ":")] = MemstatStats{ value: v, unit: linefields[4], } @@ -160,7 +161,6 @@ func (m *MemstatCollector) Init(config json.RawMessage) error { func (m *MemstatCollector) Read(interval time.Duration, output chan lp.CCMetric) { if !m.init { - cclog.ComponentPrint(m.name, "Here") return } @@ -188,16 +188,20 @@ func (m *MemstatCollector) Read(interval time.Duration, output chan lp.CCMetric) unit := "" if totalVal, total := stats["MemTotal"]; total { if freeVal, free := stats["MemFree"]; free { + memUsed = totalVal.value - freeVal.value + if len(totalVal.unit) > 0 { + unit = totalVal.unit + } else if len(freeVal.unit) > 0 { + unit = freeVal.unit + } if bufVal, buffers := stats["Buffers"]; buffers { + memUsed -= bufVal.value + if len(bufVal.unit) > 0 && len(unit) == 0 { + unit = bufVal.unit + } if cacheVal, cached := stats["Cached"]; cached { - memUsed = totalVal.value - (freeVal.value + bufVal.value + cacheVal.value) - if len(totalVal.unit) > 0 { - unit = totalVal.unit - } else if len(freeVal.unit) > 0 { - unit = freeVal.unit - } else if len(bufVal.unit) > 0 { - unit = bufVal.unit - } else if len(cacheVal.unit) > 0 { + memUsed -= cacheVal.value + if len(cacheVal.unit) > 0 && len(unit) == 0 { unit = cacheVal.unit } } diff --git a/collectors/metricCollector.go b/collectors/metricCollector.go index 4d52571..f09fa61 100644 --- a/collectors/metricCollector.go +++ b/collectors/metricCollector.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) type MetricCollector interface { diff --git a/collectors/netstatMetric.go b/collectors/netstatMetric.go index 8cfb34e..8428df1 100644 --- a/collectors/netstatMetric.go +++ b/collectors/netstatMetric.go @@ -9,8 +9,8 @@ import ( "strings" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) const NETSTATFILE = "/proc/net/dev" diff --git a/collectors/nfsMetric.go b/collectors/nfsMetric.go index 6b15784..7dca096 100644 --- a/collectors/nfsMetric.go +++ b/collectors/nfsMetric.go @@ -11,7 +11,7 @@ import ( "strings" "time" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) // First part contains the code for the general NfsCollector. diff --git a/collectors/numastatsMetric.go b/collectors/numastatsMetric.go index f65a019..8eaac67 100644 --- a/collectors/numastatsMetric.go +++ b/collectors/numastatsMetric.go @@ -10,8 +10,8 @@ import ( "strings" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) // diff --git a/collectors/nvidiaMetric.go b/collectors/nvidiaMetric.go index 0eb7c8a..a1fed03 100644 --- a/collectors/nvidiaMetric.go +++ b/collectors/nvidiaMetric.go @@ -8,8 +8,8 @@ import ( "strings" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" "github.com/NVIDIA/go-nvml/pkg/nvml" ) diff --git a/collectors/rocmsmiMetric.go b/collectors/rocmsmiMetric.go index c717a5d..9d8625d 100644 --- a/collectors/rocmsmiMetric.go +++ b/collectors/rocmsmiMetric.go @@ -6,8 +6,8 @@ import ( "fmt" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" "github.com/ClusterCockpit/go-rocm-smi/pkg/rocm_smi" ) @@ -66,14 +66,14 @@ func (m *RocmSmiCollector) Init(config json.RawMessage) error { ret := rocm_smi.Init() if ret != rocm_smi.STATUS_SUCCESS { - err = errors.New("Failed to initialize ROCm SMI library") + err = errors.New("failed to initialize ROCm SMI library") cclog.ComponentError(m.name, err.Error()) return err } numDevs, ret := rocm_smi.NumMonitorDevices() if ret != rocm_smi.STATUS_SUCCESS { - err = errors.New("Failed to get number of GPUs from ROCm SMI library") + err = errors.New("failed to get number of GPUs from ROCm SMI library") cclog.ComponentError(m.name, err.Error()) return err } @@ -98,14 +98,14 @@ func (m *RocmSmiCollector) Init(config json.RawMessage) error { } device, ret := rocm_smi.DeviceGetHandleByIndex(i) if ret != rocm_smi.STATUS_SUCCESS { - err = fmt.Errorf("Failed to get handle for GPU %d", i) + err = fmt.Errorf("failed to get handle for GPU %d", i) cclog.ComponentError(m.name, err.Error()) return err } pciInfo, ret := rocm_smi.DeviceGetPciInfo(device) if ret != rocm_smi.STATUS_SUCCESS { - err = fmt.Errorf("Failed to get PCI information for GPU %d", i) + err = fmt.Errorf("failed to get PCI information for GPU %d", i) cclog.ComponentError(m.name, err.Error()) return err } diff --git a/collectors/sampleMetric.go b/collectors/sampleMetric.go index 47ec296..056ed85 100644 --- a/collectors/sampleMetric.go +++ b/collectors/sampleMetric.go @@ -4,8 +4,8 @@ import ( "encoding/json" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) // These are the fields we read from the JSON configuration diff --git a/collectors/sampleTimerMetric.go b/collectors/sampleTimerMetric.go index aa6807e..8b09bc1 100644 --- a/collectors/sampleTimerMetric.go +++ b/collectors/sampleTimerMetric.go @@ -5,8 +5,8 @@ import ( "sync" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) // These are the fields we read from the JSON configuration diff --git a/collectors/schedstatMetric.go b/collectors/schedstatMetric.go index e3041ae..8c010ed 100644 --- a/collectors/schedstatMetric.go +++ b/collectors/schedstatMetric.go @@ -1,17 +1,17 @@ package collectors import ( + "bufio" "encoding/json" "fmt" - "bufio" - "time" - "os" - "strings" - "strconv" "math" + "os" + "strconv" + "strings" + "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) const SCHEDSTATFILE = `/proc/schedstat` @@ -25,11 +25,11 @@ type SchedstatCollectorConfig struct { // defined by metricCollector (name, init, ...) type SchedstatCollector struct { metricCollector - config SchedstatCollectorConfig // the configuration structure - lastTimestamp time.Time // Store time stamp of last tick to derive values - meta map[string]string // default meta information - cputags map[string]map[string]string // default tags - olddata map[string]map[string]int64 // default tags + config SchedstatCollectorConfig // the configuration structure + lastTimestamp time.Time // Store time stamp of last tick to derive values + meta map[string]string // default meta information + cputags map[string]map[string]string // default tags + olddata map[string]map[string]int64 // default tags } // Functions to implement MetricCollector interface @@ -52,7 +52,7 @@ func (m *SchedstatCollector) Init(config json.RawMessage) error { // Define meta information sent with each metric // (Can also be dynamic or this is the basic set with extension through AddMeta()) m.meta = map[string]string{"source": m.name, "group": "SCHEDSTAT"} - + // Read in the JSON configuration if len(config) > 0 { err = json.Unmarshal(config, &m.config) @@ -83,12 +83,11 @@ func (m *SchedstatCollector) Init(config json.RawMessage) error { running, _ := strconv.ParseInt(linefields[7], 10, 64) waiting, _ := strconv.ParseInt(linefields[8], 10, 64) m.cputags[linefields[0]] = map[string]string{"type": "hwthread", "type-id": fmt.Sprintf("%d", cpu)} - m.olddata[linefields[0]] = map[string]int64{"running" : running, "waiting" : waiting} + m.olddata[linefields[0]] = map[string]int64{"running": running, "waiting": waiting} num_cpus++ } } - // Save current timestamp m.lastTimestamp = time.Now() @@ -102,7 +101,7 @@ func (m *SchedstatCollector) ParseProcLine(linefields []string, tags map[string] waiting, _ := strconv.ParseInt(linefields[8], 10, 64) diff_running := running - m.olddata[linefields[0]]["running"] diff_waiting := waiting - m.olddata[linefields[0]]["waiting"] - + var l_running float64 = float64(diff_running) / tsdelta.Seconds() / (math.Pow(1000, 3)) var l_waiting float64 = float64(diff_waiting) / tsdelta.Seconds() / (math.Pow(1000, 3)) @@ -110,11 +109,11 @@ func (m *SchedstatCollector) ParseProcLine(linefields []string, tags map[string] m.olddata[linefields[0]]["waiting"] = waiting value := l_running + l_waiting - y, err := lp.New("cpu_load_core", tags, m.meta, map[string]interface{}{"value": value}, now) + y, err := lp.New("cpu_load_core", tags, m.meta, map[string]interface{}{"value": value}, now) if err == nil { // Send it to output channel output <- y - } + } } // Read collects all metrics belonging to the sample collector diff --git a/collectors/selfMetric.go b/collectors/selfMetric.go new file mode 100644 index 0000000..4fc95c0 --- /dev/null +++ b/collectors/selfMetric.go @@ -0,0 +1,144 @@ +package collectors + +import ( + "encoding/json" + "runtime" + "syscall" + "time" + + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" +) + +type SelfCollectorConfig struct { + MemStats bool `json:"read_mem_stats"` + GoRoutines bool `json:"read_goroutines"` + CgoCalls bool `json:"read_cgo_calls"` + Rusage bool `json:"read_rusage"` +} + +type SelfCollector struct { + metricCollector + config SelfCollectorConfig // the configuration structure + meta map[string]string // default meta information + tags map[string]string // default tags +} + +func (m *SelfCollector) Init(config json.RawMessage) error { + var err error = nil + m.name = "SelfCollector" + m.setup() + m.parallel = true + m.meta = map[string]string{"source": m.name, "group": "Self"} + m.tags = map[string]string{"type": "node"} + if len(config) > 0 { + err = json.Unmarshal(config, &m.config) + if err != nil { + cclog.ComponentError(m.name, "Error reading config:", err.Error()) + return err + } + } + m.init = true + return err +} + +func (m *SelfCollector) Read(interval time.Duration, output chan lp.CCMetric) { + timestamp := time.Now() + + if m.config.MemStats { + var memstats runtime.MemStats + runtime.ReadMemStats(&memstats) + + y, err := lp.New("total_alloc", m.tags, m.meta, map[string]interface{}{"value": memstats.TotalAlloc}, timestamp) + if err == nil { + y.AddMeta("unit", "Bytes") + output <- y + } + y, err = lp.New("heap_alloc", m.tags, m.meta, map[string]interface{}{"value": memstats.HeapAlloc}, timestamp) + if err == nil { + y.AddMeta("unit", "Bytes") + output <- y + } + y, err = lp.New("heap_sys", m.tags, m.meta, map[string]interface{}{"value": memstats.HeapSys}, timestamp) + if err == nil { + y.AddMeta("unit", "Bytes") + output <- y + } + y, err = lp.New("heap_idle", m.tags, m.meta, map[string]interface{}{"value": memstats.HeapIdle}, timestamp) + if err == nil { + y.AddMeta("unit", "Bytes") + output <- y + } + y, err = lp.New("heap_inuse", m.tags, m.meta, map[string]interface{}{"value": memstats.HeapInuse}, timestamp) + if err == nil { + y.AddMeta("unit", "Bytes") + output <- y + } + y, err = lp.New("heap_released", m.tags, m.meta, map[string]interface{}{"value": memstats.HeapReleased}, timestamp) + if err == nil { + y.AddMeta("unit", "Bytes") + output <- y + } + y, err = lp.New("heap_objects", m.tags, m.meta, map[string]interface{}{"value": memstats.HeapObjects}, timestamp) + if err == nil { + output <- y + } + } + if m.config.GoRoutines { + y, err := lp.New("num_goroutines", m.tags, m.meta, map[string]interface{}{"value": runtime.NumGoroutine()}, timestamp) + if err == nil { + output <- y + } + } + if m.config.CgoCalls { + y, err := lp.New("num_cgo_calls", m.tags, m.meta, map[string]interface{}{"value": runtime.NumCgoCall()}, timestamp) + if err == nil { + output <- y + } + } + if m.config.Rusage { + var rusage syscall.Rusage + err := syscall.Getrusage(syscall.RUSAGE_SELF, &rusage) + if err == nil { + sec, nsec := rusage.Utime.Unix() + t := float64(sec) + (float64(nsec) * 1e-9) + y, err := lp.New("rusage_user_time", m.tags, m.meta, map[string]interface{}{"value": t}, timestamp) + if err == nil { + y.AddMeta("unit", "seconds") + output <- y + } + sec, nsec = rusage.Stime.Unix() + t = float64(sec) + (float64(nsec) * 1e-9) + y, err = lp.New("rusage_system_time", m.tags, m.meta, map[string]interface{}{"value": t}, timestamp) + if err == nil { + y.AddMeta("unit", "seconds") + output <- y + } + y, err = lp.New("rusage_vol_ctx_switch", m.tags, m.meta, map[string]interface{}{"value": rusage.Nvcsw}, timestamp) + if err == nil { + output <- y + } + y, err = lp.New("rusage_invol_ctx_switch", m.tags, m.meta, map[string]interface{}{"value": rusage.Nivcsw}, timestamp) + if err == nil { + output <- y + } + y, err = lp.New("rusage_signals", m.tags, m.meta, map[string]interface{}{"value": rusage.Nsignals}, timestamp) + if err == nil { + output <- y + } + y, err = lp.New("rusage_major_pgfaults", m.tags, m.meta, map[string]interface{}{"value": rusage.Majflt}, timestamp) + if err == nil { + output <- y + } + y, err = lp.New("rusage_minor_pgfaults", m.tags, m.meta, map[string]interface{}{"value": rusage.Minflt}, timestamp) + if err == nil { + output <- y + } + } + + } +} + +func (m *SelfCollector) Close() { + m.init = false +} diff --git a/collectors/selfMetric.md b/collectors/selfMetric.md new file mode 100644 index 0000000..ab8e50b --- /dev/null +++ b/collectors/selfMetric.md @@ -0,0 +1,34 @@ +## `self` collector + +```json + "self": { + "read_mem_stats" : true, + "read_goroutines" : true, + "read_cgo_calls" : true, + "read_rusage" : true + } +``` + +The `self` collector reads the data from the `runtime` and `syscall` packages, so monitors the execution of the cc-metric-collector itself. + +Metrics: +* If `read_mem_stats == true`: + * `total_alloc`: The metric reports cumulative bytes allocated for heap objects. + * `heap_alloc`: The metric reports bytes of allocated heap objects. + * `heap_sys`: The metric reports bytes of heap memory obtained from the OS. + * `heap_idle`: The metric reports bytes in idle (unused) spans. + * `heap_inuse`: The metric reports bytes in in-use spans. + * `heap_released`: The metric reports bytes of physical memory returned to the OS. + * `heap_objects`: The metric reports the number of allocated heap objects. +* If `read_goroutines == true`: + * `num_goroutines`: The metric reports the number of goroutines that currently exist. +* If `read_cgo_calls == true`: + * `num_cgo_calls`: The metric reports the number of cgo calls made by the current process. +* If `read_rusage == true`: + * `rusage_user_time`: The metric reports the amount of time that this process has been scheduled in user mode. + * `rusage_system_time`: The metric reports the amount of time that this process has been scheduled in kernel mode. + * `rusage_vol_ctx_switch`: The metric reports the amount of voluntary context switches. + * `rusage_invol_ctx_switch`: The metric reports the amount of involuntary context switches. + * `rusage_signals`: The metric reports the number of signals received. + * `rusage_major_pgfaults`: The metric reports the number of major faults the process has made which have required loading a memory page from disk. + * `rusage_minor_pgfaults`: The metric reports the number of minor faults the process has made which have not required loading a memory page from disk. diff --git a/collectors/tempMetric.go b/collectors/tempMetric.go index af9d7fd..303be4a 100644 --- a/collectors/tempMetric.go +++ b/collectors/tempMetric.go @@ -3,14 +3,14 @@ package collectors import ( "encoding/json" "fmt" - "io/ioutil" + "os" "path/filepath" "strconv" "strings" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) // See: https://www.kernel.org/doc/html/latest/hwmon/sysfs-interface.html @@ -83,14 +83,14 @@ func (m *TempCollector) Init(config json.RawMessage) error { // sensor name nameFile := filepath.Join(filepath.Dir(file), "name") - name, err := ioutil.ReadFile(nameFile) + name, err := os.ReadFile(nameFile) if err == nil { sensor.name = strings.TrimSpace(string(name)) } // sensor label labelFile := strings.TrimSuffix(file, "_input") + "_label" - label, err := ioutil.ReadFile(labelFile) + label, err := os.ReadFile(labelFile) if err == nil { sensor.label = strings.TrimSpace(string(label)) } @@ -117,7 +117,7 @@ func (m *TempCollector) Init(config json.RawMessage) error { } // Sensor file - _, err = ioutil.ReadFile(file) + _, err = os.ReadFile(file) if err != nil { continue } @@ -139,7 +139,7 @@ func (m *TempCollector) Init(config json.RawMessage) error { // max temperature if m.config.ReportMaxTemp { maxTempFile := strings.TrimSuffix(file, "_input") + "_max" - if buffer, err := ioutil.ReadFile(maxTempFile); err == nil { + if buffer, err := os.ReadFile(maxTempFile); err == nil { if x, err := strconv.ParseInt(strings.TrimSpace(string(buffer)), 10, 64); err == nil { sensor.maxTempName = strings.Replace(sensor.metricName, "temp", "max_temp", 1) sensor.maxTemp = x / 1000 @@ -150,7 +150,7 @@ func (m *TempCollector) Init(config json.RawMessage) error { // critical temperature if m.config.ReportCriticalTemp { criticalTempFile := strings.TrimSuffix(file, "_input") + "_crit" - if buffer, err := ioutil.ReadFile(criticalTempFile); err == nil { + if buffer, err := os.ReadFile(criticalTempFile); err == nil { if x, err := strconv.ParseInt(strings.TrimSpace(string(buffer)), 10, 64); err == nil { sensor.critTempName = strings.Replace(sensor.metricName, "temp", "crit_temp", 1) sensor.critTemp = x / 1000 @@ -175,7 +175,7 @@ func (m *TempCollector) Read(interval time.Duration, output chan lp.CCMetric) { for _, sensor := range m.sensors { // Read sensor file - buffer, err := ioutil.ReadFile(sensor.file) + buffer, err := os.ReadFile(sensor.file) if err != nil { cclog.ComponentError( m.name, diff --git a/collectors/topprocsMetric.go b/collectors/topprocsMetric.go index 1f4aaca..08dbae0 100644 --- a/collectors/topprocsMetric.go +++ b/collectors/topprocsMetric.go @@ -9,7 +9,7 @@ import ( "strings" "time" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) const MAX_NUM_PROCS = 10 diff --git a/go.mod b/go.mod index a42ae8a..30a961c 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/nats-io/nats.go v1.16.0 github.com/prometheus/client_golang v1.12.2 github.com/stmcginnis/gofish v0.13.0 + github.com/tklauser/go-sysconf v0.3.10 golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e ) @@ -33,6 +34,7 @@ require ( github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.7.3 // indirect github.com/shopspring/decimal v1.3.1 // indirect + github.com/tklauser/numcpus v0.4.0 // indirect golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect golang.org/x/net v0.0.0-20220708220712-1185a9018129 // indirect google.golang.org/protobuf v1.28.0 // indirect diff --git a/go.sum b/go.sum index 9f65f7e..03d7b08 100644 --- a/go.sum +++ b/go.sum @@ -287,6 +287,10 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw= +github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= +github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o= +github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= @@ -445,6 +449,7 @@ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e h1:NHvCuwuS43lGnYhten69ZWqi2QOj/CiDNcKbVqwVoew= golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/metricAggregator/metricAggregator.go b/internal/metricAggregator/metricAggregator.go index f5c7ada..50bc963 100644 --- a/internal/metricAggregator/metricAggregator.go +++ b/internal/metricAggregator/metricAggregator.go @@ -9,10 +9,10 @@ import ( "sync" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" - topo "github.com/ClusterCockpit/cc-metric-collector/internal/ccTopology" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" + topo "github.com/ClusterCockpit/cc-metric-collector/pkg/ccTopology" "github.com/PaesslerAG/gval" ) diff --git a/internal/metricAggregator/metricAggregatorFunctions.go b/internal/metricAggregator/metricAggregatorFunctions.go index 945dc6d..68d205b 100644 --- a/internal/metricAggregator/metricAggregatorFunctions.go +++ b/internal/metricAggregator/metricAggregatorFunctions.go @@ -8,8 +8,8 @@ import ( "sort" "strings" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - topo "github.com/ClusterCockpit/cc-metric-collector/internal/ccTopology" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + topo "github.com/ClusterCockpit/cc-metric-collector/pkg/ccTopology" ) /* diff --git a/internal/metricRouter/metricCache.go b/internal/metricRouter/metricCache.go index 8886f47..81e69a9 100644 --- a/internal/metricRouter/metricCache.go +++ b/internal/metricRouter/metricCache.go @@ -4,11 +4,11 @@ import ( "sync" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" agg "github.com/ClusterCockpit/cc-metric-collector/internal/metricAggregator" - mct "github.com/ClusterCockpit/cc-metric-collector/internal/multiChanTicker" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" + mct "github.com/ClusterCockpit/cc-metric-collector/pkg/multiChanTicker" ) type metricCachePeriod struct { diff --git a/internal/metricRouter/metricRouter.go b/internal/metricRouter/metricRouter.go index 2614ced..32ac0f3 100644 --- a/internal/metricRouter/metricRouter.go +++ b/internal/metricRouter/metricRouter.go @@ -7,11 +7,11 @@ import ( "sync" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" agg "github.com/ClusterCockpit/cc-metric-collector/internal/metricAggregator" - mct "github.com/ClusterCockpit/cc-metric-collector/internal/multiChanTicker" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" + mct "github.com/ClusterCockpit/cc-metric-collector/pkg/multiChanTicker" units "github.com/ClusterCockpit/cc-units" ) diff --git a/internal/ccLogger/cclogger.go b/pkg/ccLogger/cclogger.go similarity index 100% rename from internal/ccLogger/cclogger.go rename to pkg/ccLogger/cclogger.go diff --git a/internal/ccMetric/README.md b/pkg/ccMetric/README.md similarity index 100% rename from internal/ccMetric/README.md rename to pkg/ccMetric/README.md diff --git a/internal/ccMetric/ccMetric.go b/pkg/ccMetric/ccMetric.go similarity index 100% rename from internal/ccMetric/ccMetric.go rename to pkg/ccMetric/ccMetric.go diff --git a/internal/ccTopology/ccTopology.go b/pkg/ccTopology/ccTopology.go similarity index 99% rename from internal/ccTopology/ccTopology.go rename to pkg/ccTopology/ccTopology.go index 0ed8883..f7d7092 100644 --- a/internal/ccTopology/ccTopology.go +++ b/pkg/ccTopology/ccTopology.go @@ -10,7 +10,7 @@ import ( "strconv" "strings" - cclogger "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" + cclogger "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" ) const SYSFS_NUMABASE = `/sys/devices/system/node` diff --git a/internal/multiChanTicker/README.md b/pkg/multiChanTicker/README.md similarity index 100% rename from internal/multiChanTicker/README.md rename to pkg/multiChanTicker/README.md diff --git a/internal/multiChanTicker/multiChanTicker.go b/pkg/multiChanTicker/multiChanTicker.go similarity index 93% rename from internal/multiChanTicker/multiChanTicker.go rename to pkg/multiChanTicker/multiChanTicker.go index e0eca43..1791474 100644 --- a/internal/multiChanTicker/multiChanTicker.go +++ b/pkg/multiChanTicker/multiChanTicker.go @@ -3,7 +3,7 @@ package multiChanTicker import ( "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" ) type multiChanTicker struct { diff --git a/receivers/httpReceiver.go b/receivers/httpReceiver.go index 974bcd6..efd31ac 100644 --- a/receivers/httpReceiver.go +++ b/receivers/httpReceiver.go @@ -10,8 +10,8 @@ import ( "strings" "sync" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" "github.com/gorilla/mux" influx "github.com/influxdata/line-protocol" ) diff --git a/receivers/metricReceiver.go b/receivers/metricReceiver.go index 6b85fd4..1edef4e 100644 --- a/receivers/metricReceiver.go +++ b/receivers/metricReceiver.go @@ -1,7 +1,7 @@ package receivers import ( - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) type defaultReceiverConfig struct { diff --git a/receivers/natsReceiver.go b/receivers/natsReceiver.go index 1a5f47b..095a7ee 100644 --- a/receivers/natsReceiver.go +++ b/receivers/natsReceiver.go @@ -6,8 +6,8 @@ import ( "fmt" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" influx "github.com/influxdata/line-protocol" nats "github.com/nats-io/nats.go" ) diff --git a/receivers/prometheusReceiver.go b/receivers/prometheusReceiver.go index c22969d..7846c1d 100644 --- a/receivers/prometheusReceiver.go +++ b/receivers/prometheusReceiver.go @@ -12,8 +12,8 @@ import ( "sync" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) type PrometheusReceiverConfig struct { diff --git a/receivers/receiveManager.go b/receivers/receiveManager.go index 7a20fac..c47c3cc 100644 --- a/receivers/receiveManager.go +++ b/receivers/receiveManager.go @@ -5,8 +5,8 @@ import ( "os" "sync" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) var AvailableReceivers = map[string]func(name string, config json.RawMessage) (Receiver, error){ diff --git a/receivers/redfishReceiver.go b/receivers/redfishReceiver.go index 18c3ee8..de9744d 100644 --- a/receivers/redfishReceiver.go +++ b/receivers/redfishReceiver.go @@ -10,8 +10,8 @@ import ( "sync" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" // See: https://pkg.go.dev/github.com/stmcginnis/gofish "github.com/stmcginnis/gofish" diff --git a/receivers/sampleReceiver.go b/receivers/sampleReceiver.go index 19d6f25..86e68cd 100644 --- a/receivers/sampleReceiver.go +++ b/receivers/sampleReceiver.go @@ -4,7 +4,7 @@ import ( "encoding/json" "fmt" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" ) // SampleReceiver configuration: receiver type, listen address, port diff --git a/sinks/gangliaCommon.go b/sinks/gangliaCommon.go index f92550b..1c846f0 100644 --- a/sinks/gangliaCommon.go +++ b/sinks/gangliaCommon.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) func GangliaMetricName(point lp.CCMetric) string { diff --git a/sinks/gangliaSink.go b/sinks/gangliaSink.go index 5324123..b6a0646 100644 --- a/sinks/gangliaSink.go +++ b/sinks/gangliaSink.go @@ -9,8 +9,8 @@ import ( // "time" "os/exec" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) const GMETRIC_EXEC = `gmetric` diff --git a/sinks/httpSink.go b/sinks/httpSink.go index 466915d..49d98b0 100644 --- a/sinks/httpSink.go +++ b/sinks/httpSink.go @@ -9,8 +9,8 @@ import ( "sync" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" influx "github.com/influxdata/line-protocol" ) diff --git a/sinks/influxAsyncSink.go b/sinks/influxAsyncSink.go index bf88079..ccf5302 100644 --- a/sinks/influxAsyncSink.go +++ b/sinks/influxAsyncSink.go @@ -9,8 +9,8 @@ import ( "strings" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" influxdb2 "github.com/influxdata/influxdb-client-go/v2" influxdb2Api "github.com/influxdata/influxdb-client-go/v2/api" influxdb2ApiHttp "github.com/influxdata/influxdb-client-go/v2/api/http" diff --git a/sinks/influxSink.go b/sinks/influxSink.go index 90f8da8..94704cd 100644 --- a/sinks/influxSink.go +++ b/sinks/influxSink.go @@ -9,8 +9,8 @@ import ( "sync" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" influxdb2 "github.com/influxdata/influxdb-client-go/v2" influxdb2Api "github.com/influxdata/influxdb-client-go/v2/api" "github.com/influxdata/influxdb-client-go/v2/api/write" diff --git a/sinks/libgangliaSink.go b/sinks/libgangliaSink.go index 3651584..bb5488a 100644 --- a/sinks/libgangliaSink.go +++ b/sinks/libgangliaSink.go @@ -71,8 +71,8 @@ import ( "fmt" "unsafe" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" "github.com/NVIDIA/go-nvml/pkg/dl" ) diff --git a/sinks/metricSink.go b/sinks/metricSink.go index c6c6860..2fd429c 100644 --- a/sinks/metricSink.go +++ b/sinks/metricSink.go @@ -1,7 +1,7 @@ package sinks import ( - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) type defaultSinkConfig struct { diff --git a/sinks/natsSink.go b/sinks/natsSink.go index 4d43454..a9a980d 100644 --- a/sinks/natsSink.go +++ b/sinks/natsSink.go @@ -8,8 +8,8 @@ import ( "sync" "time" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" influx "github.com/influxdata/line-protocol" nats "github.com/nats-io/nats.go" ) diff --git a/sinks/prometheusSink.go b/sinks/prometheusSink.go index 5011ac0..7a1163d 100644 --- a/sinks/prometheusSink.go +++ b/sinks/prometheusSink.go @@ -9,8 +9,8 @@ import ( "strings" "sync" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" "github.com/gorilla/mux" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" diff --git a/sinks/sampleSink.go b/sinks/sampleSink.go index 2a823e6..9d9e991 100644 --- a/sinks/sampleSink.go +++ b/sinks/sampleSink.go @@ -5,8 +5,8 @@ import ( "fmt" "log" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) type SampleSinkConfig struct { diff --git a/sinks/sinkManager.go b/sinks/sinkManager.go index 6af8614..392346d 100644 --- a/sinks/sinkManager.go +++ b/sinks/sinkManager.go @@ -6,8 +6,8 @@ import ( "os" "sync" - cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) const SINK_MAX_FORWARD = 50 diff --git a/sinks/stdoutSink.go b/sinks/stdoutSink.go index e091af3..eadef4e 100644 --- a/sinks/stdoutSink.go +++ b/sinks/stdoutSink.go @@ -7,7 +7,7 @@ import ( "strings" // "time" - lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" ) type StdoutSink struct {