ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2024-12-26 13:29:05 +01:00
Code Issues Packages Releases Activity

Sync state. Add job tagging scripts and siurce code.

Browse Source
This commit is contained in:
Jan Eitzinger 2019-11-11 09:37:56 +01:00
parent cb0fae0216
commit 95f616f45e
19 changed files with 1539 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
anonDB.pl
View File

@ -110,15 +110,17 @@ while ($sth_select_all->fetch) {
); );
} }
# convert job meta file open(my $fh, '<:encoding(UTF-8)', './jobIds.txt')
opendir my $dh, $basedir or die "can't open directory: $!"; or die "Could not open file $!";
while ( readdir $dh ) {
chomp;
next if $_ eq '.' or $_ eq '..';
my $jobID = $_; # convert job meta file
my $jobmeta_json = read_file("$basedir/$jobID/meta.json"); while ( <$fh> ) {
my $job = decode_json $jobmeta_json;
my $line = $_;
my ($jobID, $path1, $path2) = split ' ', $line;
my $json = read_file("$basedir/$path1/$path2/meta.json");
my $job = decode_json $json;
my $user = $job->{'user_id'}; my $user = $job->{'user_id'};
@ -143,10 +145,9 @@ while ( readdir $dh ) {
$job->{user_id} = $user; $job->{user_id} = $user;
$job->{project_id} = $project; $job->{project_id} = $project;
$jobmeta_json = encode_json $job; $json = encode_json $job;
# print "$jobmeta_json\n"; write_file("$basedir/$path1/$path2/meta.json", $json);
write_file("$basedir/$jobID/meta.json", $jobmeta_json);
} }
closedir $dh or die "can't close directory: $!"; close $fh;
$dbh->disconnect; $dbh->disconnect;

12
extractJobIDs.pl
View File

@ -33,6 +33,7 @@ open(my $fh, '>:encoding(UTF-8)', 'jobIds.txt')
or die "Could not open file $!"; or die "Could not open file $!";
opendir my $odh, $basedir or die "can't open directory: $!"; opendir my $odh, $basedir or die "can't open directory: $!";
my $count = 0;
while ( readdir $odh ) { while ( readdir $odh ) {
chomp; chomp;
@ -48,10 +49,19 @@ while ( readdir $odh ) {
next if $_ eq '.' or $_ eq '..'; next if $_ eq '.' or $_ eq '..';
my $jobID2 = $_; my $jobID2 = $_;
print $fh "$jobID1$jobID2.eadm $jobID1 $jobID2\n"; unless (-e "$basedir/$jobID1/$jobID2/data.json") {
print "$basedir/$jobID1/$jobID2/ File Doesn't Exist!\n";
# rmdir "$basedir/$jobID1/$jobID2";
$count++;
} else {
print $fh "$jobID1$jobID2.eadm $jobID1 $jobID2\n";
}
} }
closedir $idh or die "can't close directory: $!"; closedir $idh or die "can't close directory: $!";
} }
closedir $odh or die "can't close directory: $!"; closedir $odh or die "can't close directory: $!";
close $fh; close $fh;
print "$count empty jobs!\n";

88
generatePlots.pl Executable file
View File

@ -0,0 +1,88 @@
#!/usr/bin/env perl
use strict;
use warnings;
use utf8;
use File::Slurp;
use Data::Dumper;
use JSON::MaybeXS qw(encode_json decode_json);
my $jobDirectory = '../data';
sub gnuplotControl {
my $jobID = shift;
my $metricName = shift;
my $numNodes = shift;
my $unit = shift;
my $gpMacros = <<"END";
set terminal png size 1400,768 enhanced font ,12
set output '$jobID-$metricName.png'
set xlabel 'runtime [s]'
set ylabel '[$unit]'
END
$gpMacros .= "plot '$metricName.dat' u 2 w lines notitle";
foreach my $col ( 3 ... $numNodes ){
$gpMacros .= ", '$metricName.dat' u $col w lines notitle";
}
open(my $fh, '>:encoding(UTF-8)', './metric.plot')
or die "Could not open file $!";
print $fh $gpMacros;
close $fh;
system('gnuplot','metric.plot');
}
sub createPlot {
my $jobID = shift;
my $metricName = shift;
my $metric = shift;
my $unit = shift;
my @lines;
foreach my $node ( @$metric ) {
my $i = 0;
foreach my $val ( @{$node->{data}} ){
$lines[$i++] .= " $val";
}
}
open(my $fh, '>:encoding(UTF-8)', './'.$metricName.'.dat')
or die "Could not open file $!";
my $timestamp = 0;
foreach my $line ( @lines ) {
print $fh $timestamp.$line."\n";
$timestamp += 60;
}
close $fh;
gnuplotControl($jobID, $metricName, $#$metric + 2, $unit);
}
mkdir('./plots');
chdir('./plots');
while ( <> ) {
my $jobID = $_;
$jobID =~ s/\.eadm//;
chomp $jobID;
my $level1 = $jobID/1000;
my $level2 = $jobID%1000;
my $jobpath = sprintf("%s/%d/%03d", $jobDirectory, $level1, $level2);
my $json = read_file($jobpath.'/data.json');
my $data = decode_json $json;
$json = read_file($jobpath.'/meta.json');
my $meta = decode_json $json;
createPlot($jobID, 'flops_any', $data->{flops_any}->{series}, $data->{flops_any}->{unit});
createPlot($jobID, 'mem_bw', $data->{mem_bw}->{series}, $data->{mem_bw}->{unit});
}

68
jobTag.pl Normal file
View File

@ -0,0 +1,68 @@
#!/usr/bin/env perl
# =======================================================================================
#
# Author: Jan Eitzinger (je), jan.eitzinger@fau.de
# Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# =======================================================================================
use strict;
use warnings;
use utf8;
use DBI;
my $database = 'jobDB';
my %attr = (
PrintError => 1,
RaiseError => 1
);
my $dbh = DBI->connect(
"DBI:SQLite:dbname=$database", "", "", \%attr)
or die "Could not connect to database: $DBI::errstr";
my $sth_select_job = $dbh->prepare(qq{
SELECT id, user_id, job_id, cluster_id,
start_time, stop_time, duration, num_nodes,
has_profile
FROM job
WHERE job_id=?
});
my $sth_update_job = $dbh->prepare(qq{
UPDATE job
SET has_profile = ?,
mem_used_max = ?,
flops_any_avg = ?,
mem_bw_avg = ?
WHERE id=?;
});
my $sth_select_tag = $dbh->prepare(qq{
SELECT id
FROM tag
WHERE tag_name=?
});
my $CMD = $ARGV[0];

78
jobTagger/Makefile Normal file
View File

@ -0,0 +1,78 @@
#=======================================================================================
#
# Author: Jan Eitzinger (je), jan.eitzinger@fau.de
# Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
#=======================================================================================
TAG = CLANG
#CONFIGURE BUILD SYSTEM
TARGET = jobTagger-$(TAG)
BUILD_DIR = ./$(TAG)
SRC_DIR = ./src
MAKE_DIR = ./
Q ?= @
#DO NOT EDIT BELOW
include $(MAKE_DIR)/include_$(TAG).mk
VPATH = $(SRC_DIR)
ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s,$(wildcard $(SRC_DIR)/*.c))
OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o,$(wildcard $(SRC_DIR)/*.c))
CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES)
${TARGET}: $(BUILD_DIR) $(OBJ)
@echo "===> LINKING $(TARGET)"
$(Q)${LINKER} ${LFLAGS} -o $(TARGET) $(OBJ) $(LIBS)
asm: $(BUILD_DIR) $(ASM)
$(BUILD_DIR)/%.o: %.c
@echo "===> COMPILE $@"
$(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
$(Q)$(GCC) $(CPPFLAGS) -MT $(@:.d=.o) -MM $< > $(BUILD_DIR)/$*.d
$(BUILD_DIR)/%.s: %.c
@echo "===> GENERATE ASM $@"
$(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@
tags:
@echo "===> GENERATE TAGS"
$(Q)ctags -R
$(BUILD_DIR):
@mkdir $(BUILD_DIR)
ifeq ($(findstring $(MAKECMDGOALS),clean),)
-include $(OBJ:.o=.d)
endif
.PHONY: clean distclean
clean:
@echo "===> CLEAN"
@rm -rf $(BUILD_DIR)
@rm -f tags
distclean: clean
@echo "===> DIST CLEAN"
@rm -f $(TARGET)
@rm -f tags

10
jobTagger/include_CLANG.mk Normal file
View File

@ -0,0 +1,10 @@
CC = clang
GCC = gcc
LINKER = $(CC)
OPENMP = -fopenmp
CFLAGS = -Ofast -std=c99 $(OPENMP)
LFLAGS = $(OPENMP)
DEFINES = -D_GNU_SOURCE -DJSMN_PARENT_LINKS
INCLUDES =
LIBS =

13
jobTagger/include_GCC.mk Normal file
View File

@ -0,0 +1,13 @@
CC = gcc
GCC = gcc
LINKER = $(CC)
ifeq ($(ENABLE_OPENMP),true)
OPENMP = -fopenmp
endif
CFLAGS = -Ofast -ffreestanding -std=c99 $(OPENMP)
LFLAGS = $(OPENMP)
DEFINES = -D_GNU_SOURCE
INCLUDES =
LIBS =

13
jobTagger/include_ICC.mk Normal file
View File

@ -0,0 +1,13 @@
CC = icc
GCC = gcc
LINKER = $(CC)
ifeq ($(ENABLE_OPENMP),true)
OPENMP = -qopenmp
endif
CFLAGS = -qopt-report -Ofast -xHost -std=c99 -ffreestanding $(OPENMP)
LFLAGS = $(OPENMP)
DEFINES = -D_GNU_SOURCE
INCLUDES =
LIBS =

88
jobTagger/src/affinity.c Normal file
View File

@ -0,0 +1,88 @@
/*
* =======================================================================================
*
* Author: Jan Eitzinger (je), jan.eitzinger@fau.de
* Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
* =======================================================================================
*/
#ifdef __linux__
#ifdef _OPENMP
#include <stdlib.h>
#include <stdio.h>
#include <sched.h>
#include <unistd.h>
#include <sys/types.h>
#include <pthread.h>
#include <sys/syscall.h>
#define MAX_NUM_THREADS 128
#define gettid() syscall(SYS_gettid)
static int
getProcessorID(cpu_set_t* cpu_set)
{
int processorId;
for ( processorId = 0; processorId < MAX_NUM_THREADS; processorId++ )
{
if ( CPU_ISSET(processorId,cpu_set) )
{
break;
}
}
return processorId;
}
int
affinity_getProcessorId()
{
cpu_set_t cpu_set;
CPU_ZERO(&cpu_set);
sched_getaffinity(gettid(),sizeof(cpu_set_t), &cpu_set);
return getProcessorID(&cpu_set);
}
void
affinity_pinThread(int processorId)
{
cpu_set_t cpuset;
pthread_t thread;
thread = pthread_self();
CPU_ZERO(&cpuset);
CPU_SET(processorId, &cpuset);
pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset);
}
void
affinity_pinProcess(int processorId)
{
cpu_set_t cpuset;
CPU_ZERO(&cpuset);
CPU_SET(processorId, &cpuset);
sched_setaffinity(0, sizeof(cpu_set_t), &cpuset);
}
#endif /*_OPENMP*/
#endif /*__linux__*/

35
jobTagger/src/affinity.h Normal file
View File

@ -0,0 +1,35 @@
/*
* =======================================================================================
*
* Author: Jan Eitzinger (je), jan.eitzinger@fau.de
* Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
* =======================================================================================
*/
#ifndef AFFINITY_H
#define AFFINITY_H
extern int affinity_getProcessorId();
extern void affinity_pinProcess(int);
extern void affinity_pinThread(int);
#endif /*AFFINITY_H*/

58
jobTagger/src/allocate.c Normal file
View File

@ -0,0 +1,58 @@
/*
* =======================================================================================
*
* Author: Jan Eitzinger (je), jan.eitzinger@fau.de
* Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
* =======================================================================================
*/
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
void* allocate (int alignment, size_t bytesize)
{
int errorCode;
void* ptr;
errorCode = posix_memalign(&ptr, alignment, bytesize);
if (errorCode) {
if (errorCode == EINVAL) {
fprintf(stderr,
"Error: Alignment parameter is not a power of two\n");
exit(EXIT_FAILURE);
}
if (errorCode == ENOMEM) {
fprintf(stderr,
"Error: Insufficient memory to fulfill the request\n");
exit(EXIT_FAILURE);
}
}
if (ptr == NULL) {
fprintf(stderr, "Error: posix_memalign failed!\n");
exit(EXIT_FAILURE);
}
return ptr;
}

33
jobTagger/src/allocate.h Normal file
View File

@ -0,0 +1,33 @@
/*
* =======================================================================================
*
* Author: Jan Eitzinger (je), jan.eitzinger@fau.de
* Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
* =======================================================================================
*/
#ifndef __ALLOCATE_H_
#define __ALLOCATE_H_
extern void* allocate (int alignment, size_t bytesize);
#endif

468
jobTagger/src/jsmn.h Normal file
View File

@ -0,0 +1,468 @@
/*
* MIT License
*
* Copyright (c) 2010 Serge Zaitsev
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef JSMN_H
#define JSMN_H
#include <stddef.h>
#ifdef __cplusplus
extern "C" {
#endif
#ifdef JSMN_STATIC
#define JSMN_API static
#else
#define JSMN_API extern
#endif
/**
* JSON type identifier. Basic types are:
* o Object
* o Array
* o String
* o Other primitive: number, boolean (true/false) or null
*/
typedef enum {
JSMN_UNDEFINED = 0,
JSMN_OBJECT = 1,
JSMN_ARRAY = 2,
JSMN_STRING = 3,
JSMN_PRIMITIVE = 4
} jsmntype_t;
enum jsmnerr {
/* Not enough tokens were provided */
JSMN_ERROR_NOMEM = -1,
/* Invalid character inside JSON string */
JSMN_ERROR_INVAL = -2,
/* The string is not a full JSON packet, more bytes expected */
JSMN_ERROR_PART = -3
};
/**
* JSON token description.
* type type (object, array, string etc.)
* start start position in JSON data string
* end end position in JSON data string
*/
typedef struct {
jsmntype_t type;
int start;
int end;
int size;
#ifdef JSMN_PARENT_LINKS
int parent;
#endif
} jsmntok_t;
/**
* JSON parser. Contains an array of token blocks available. Also stores
* the string being parsed now and current position in that string.
*/
typedef struct {
unsigned int pos; /* offset in the JSON string */
unsigned int toknext; /* next token to allocate */
int toksuper; /* superior token node, e.g. parent object or array */
} jsmn_parser;
/**
* Create JSON parser over an array of tokens
*/
JSMN_API void jsmn_init(jsmn_parser *parser);
/**
* Run JSON parser. It parses a JSON data string into and array of tokens, each
* describing
* a single JSON object.
*/
JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, const size_t len,
jsmntok_t *tokens, const unsigned int num_tokens);
#ifndef JSMN_HEADER
/**
* Allocates a fresh unused token from the token pool.
*/
static jsmntok_t *jsmn_alloc_token(jsmn_parser *parser, jsmntok_t *tokens,
const size_t num_tokens) {
jsmntok_t *tok;
if (parser->toknext >= num_tokens) {
return NULL;
}
tok = &tokens[parser->toknext++];
tok->start = tok->end = -1;
tok->size = 0;
#ifdef JSMN_PARENT_LINKS
tok->parent = -1;
#endif
return tok;
}
/**
* Fills token type and boundaries.
*/
static void jsmn_fill_token(jsmntok_t *token, const jsmntype_t type,
const int start, const int end) {
token->type = type;
token->start = start;
token->end = end;
token->size = 0;
}
/**
* Fills next available token with JSON primitive.
*/
static int jsmn_parse_primitive(jsmn_parser *parser, const char *js,
const size_t len, jsmntok_t *tokens,
const size_t num_tokens) {
jsmntok_t *token;
int start;
start = parser->pos;
for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) {
switch (js[parser->pos]) {
#ifndef JSMN_STRICT
/* In strict mode primitive must be followed by "," or "}" or "]" */
case ':':
#endif
case '\t':
case '\r':
case '\n':
case ' ':
case ',':
case ']':
case '}':
goto found;
}
if (js[parser->pos] < 32 || js[parser->pos] >= 127) {
parser->pos = start;
return JSMN_ERROR_INVAL;
}
}
#ifdef JSMN_STRICT
/* In strict mode primitive must be followed by a comma/object/array */
parser->pos = start;
return JSMN_ERROR_PART;
#endif
found:
if (tokens == NULL) {
parser->pos--;
return 0;
}
token = jsmn_alloc_token(parser, tokens, num_tokens);
if (token == NULL) {
parser->pos = start;
return JSMN_ERROR_NOMEM;
}
jsmn_fill_token(token, JSMN_PRIMITIVE, start, parser->pos);
#ifdef JSMN_PARENT_LINKS
token->parent = parser->toksuper;
#endif
parser->pos--;
return 0;
}
/**
* Fills next token with JSON string.
*/
static int jsmn_parse_string(jsmn_parser *parser, const char *js,
const size_t len, jsmntok_t *tokens,
const size_t num_tokens) {
jsmntok_t *token;
int start = parser->pos;
parser->pos++;
/* Skip starting quote */
for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) {
char c = js[parser->pos];
/* Quote: end of string */
if (c == '\"') {
if (tokens == NULL) {
return 0;
}
token = jsmn_alloc_token(parser, tokens, num_tokens);
if (token == NULL) {
parser->pos = start;
return JSMN_ERROR_NOMEM;
}
jsmn_fill_token(token, JSMN_STRING, start + 1, parser->pos);
#ifdef JSMN_PARENT_LINKS
token->parent = parser->toksuper;
#endif
return 0;
}
/* Backslash: Quoted symbol expected */
if (c == '\\' && parser->pos + 1 < len) {
int i;
parser->pos++;
switch (js[parser->pos]) {
/* Allowed escaped symbols */
case '\"':
case '/':
case '\\':
case 'b':
case 'f':
case 'r':
case 'n':
case 't':
break;
/* Allows escaped symbol \uXXXX */
case 'u':
parser->pos++;
for (i = 0; i < 4 && parser->pos < len && js[parser->pos] != '\0';
i++) {
/* If it isn't a hex character we have an error */
if (!((js[parser->pos] >= 48 && js[parser->pos] <= 57) || /* 0-9 */
(js[parser->pos] >= 65 && js[parser->pos] <= 70) || /* A-F */
(js[parser->pos] >= 97 && js[parser->pos] <= 102))) { /* a-f */
parser->pos = start;
return JSMN_ERROR_INVAL;
}
parser->pos++;
}
parser->pos--;
break;
/* Unexpected symbol */
default:
parser->pos = start;
return JSMN_ERROR_INVAL;
}
}
}
parser->pos = start;
return JSMN_ERROR_PART;
}
/**
* Parse JSON string and fill tokens.
*/
JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, const size_t len,
jsmntok_t *tokens, const unsigned int num_tokens) {
int r;
int i;
jsmntok_t *token;
int count = parser->toknext;
for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) {
char c;
jsmntype_t type;
c = js[parser->pos];
switch (c) {
case '{':
case '[':
count++;
if (tokens == NULL) {
break;
}
token = jsmn_alloc_token(parser, tokens, num_tokens);
if (token == NULL) {
return JSMN_ERROR_NOMEM;
}
if (parser->toksuper != -1) {
jsmntok_t *t = &tokens[parser->toksuper];
#ifdef JSMN_STRICT
/* In strict mode an object or array can't become a key */
if (t->type == JSMN_OBJECT) {
return JSMN_ERROR_INVAL;
}
#endif
t->size++;
#ifdef JSMN_PARENT_LINKS
token->parent = parser->toksuper;
#endif
}
token->type = (c == '{' ? JSMN_OBJECT : JSMN_ARRAY);
token->start = parser->pos;
parser->toksuper = parser->toknext - 1;
break;
case '}':
case ']':
if (tokens == NULL) {
break;
}
type = (c == '}' ? JSMN_OBJECT : JSMN_ARRAY);
#ifdef JSMN_PARENT_LINKS
if (parser->toknext < 1) {
return JSMN_ERROR_INVAL;
}
token = &tokens[parser->toknext - 1];
for (;;) {
if (token->start != -1 && token->end == -1) {
if (token->type != type) {
return JSMN_ERROR_INVAL;
}
token->end = parser->pos + 1;
parser->toksuper = token->parent;
break;
}
if (token->parent == -1) {
if (token->type != type || parser->toksuper == -1) {
return JSMN_ERROR_INVAL;
}
break;
}
token = &tokens[token->parent];
}
#else
for (i = parser->toknext - 1; i >= 0; i--) {
token = &tokens[i];
if (token->start != -1 && token->end == -1) {
if (token->type != type) {
return JSMN_ERROR_INVAL;
}
parser->toksuper = -1;
token->end = parser->pos + 1;
break;
}
}
/* Error if unmatched closing bracket */
if (i == -1) {
return JSMN_ERROR_INVAL;
}
for (; i >= 0; i--) {
token = &tokens[i];
if (token->start != -1 && token->end == -1) {
parser->toksuper = i;
break;
}
}
#endif
break;
case '\"':
r = jsmn_parse_string(parser, js, len, tokens, num_tokens);
if (r < 0) {
return r;
}
count++;
if (parser->toksuper != -1 && tokens != NULL) {
tokens[parser->toksuper].size++;
}
break;
case '\t':
case '\r':
case '\n':
case ' ':
break;
case ':':
parser->toksuper = parser->toknext - 1;
break;
case ',':
if (tokens != NULL && parser->toksuper != -1 &&
tokens[parser->toksuper].type != JSMN_ARRAY &&
tokens[parser->toksuper].type != JSMN_OBJECT) {
#ifdef JSMN_PARENT_LINKS
parser->toksuper = tokens[parser->toksuper].parent;
#else
for (i = parser->toknext - 1; i >= 0; i--) {
if (tokens[i].type == JSMN_ARRAY || tokens[i].type == JSMN_OBJECT) {
if (tokens[i].start != -1 && tokens[i].end == -1) {
parser->toksuper = i;
break;
}
}
}
#endif
}
break;
#ifdef JSMN_STRICT
/* In strict mode primitives are: numbers and booleans */
case '-':
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
case 't':
case 'f':
case 'n':
/* And they must not be keys of the object */
if (tokens != NULL && parser->toksuper != -1) {
const jsmntok_t *t = &tokens[parser->toksuper];
if (t->type == JSMN_OBJECT ||
(t->type == JSMN_STRING && t->size != 0)) {
return JSMN_ERROR_INVAL;
}
}
#else
/* In non-strict mode every unquoted value is a primitive */
default:
#endif
r = jsmn_parse_primitive(parser, js, len, tokens, num_tokens);
if (r < 0) {
return r;
}
count++;
if (parser->toksuper != -1 && tokens != NULL) {
tokens[parser->toksuper].size++;
}
break;
#ifdef JSMN_STRICT
/* Unexpected char in strict mode */
default:
return JSMN_ERROR_INVAL;
#endif
}
}
if (tokens != NULL) {
for (i = parser->toknext - 1; i >= 0; i--) {
/* Unmatched opened object or array */
if (tokens[i].start != -1 && tokens[i].end == -1) {
return JSMN_ERROR_PART;
}
}
}
return count;
}
/**
* Creates a new parser based over a given buffer with an array of tokens
* available.
*/
JSMN_API void jsmn_init(jsmn_parser *parser) {
parser->pos = 0;
parser->toknext = 0;
parser->toksuper = -1;
}
#endif /* JSMN_HEADER */
#ifdef __cplusplus
}
#endif
#endif /* JSMN_H */

304
jobTagger/src/main.c Normal file
View File

@ -0,0 +1,304 @@
/*
* =======================================================================================
*
* Author: Jan Eitzinger (je), jan.eitzinger@fau.de
* Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
* =======================================================================================
*/
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include <limits.h>
#include <float.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <fcntl.h>
#ifdef _OPENMP
#include <omp.h>
#endif
#include "jsmn.h"
#include "timing.h"
#include "allocate.h"
#include "affinity.h"
#define HLINE "----------------------------------------------------------------------------\n"
#ifndef MIN
#define MIN(x,y) ((x)<(y)?(x):(y))
#endif
#ifndef MAX
#define MAX(x,y) ((x)>(y)?(x):(y))
#endif
#ifndef ABS
#define ABS(a) ((a) >= 0 ? (a) : -(a))
#endif
char* json_fetch(char* filepath)
{
int fd = open(filepath, O_RDONLY);
if ( fd == -1) {
perror("Cannot open output file\n"); exit(1);
}
int len = lseek(fd, 0, SEEK_END);
void *data = mmap(0, len, PROT_READ, MAP_PRIVATE, fd, 0);
return (char*) data;
}
jsmntok_t * json_tokenise(char *js)
{
jsmn_parser parser;
jsmn_init(&parser);
unsigned int n = 4096;
jsmntok_t *tokens = malloc(sizeof(jsmntok_t) * n);
int ret = jsmn_parse(&parser, js, strlen(js), tokens, n);
while (ret == JSMN_ERROR_NOMEM)
{
n = n * 2 + 1;
tokens = realloc(tokens, sizeof(jsmntok_t) * n);
ret = jsmn_parse(&parser, js, strlen(js), tokens, n);
}
if (ret == JSMN_ERROR_INVAL) {
printf("jsmn_parse: invalid JSON string");
exit(EXIT_SUCCESS);
}
if (ret == JSMN_ERROR_PART) {
printf("jsmn_parse: truncated JSON string");
exit(EXIT_SUCCESS);
}
return tokens;
}
int json_token_streq(char* js, jsmntok_t* t, char* s)
{
return (strncmp(js + t->start, s, t->end - t->start) == 0
&& strlen(s) == (size_t) (t->end - t->start));
}
char* json_token_tostr(char* js, jsmntok_t* t)
{
js[t->end] = '\0';
return js + t->start;
}
void print_token(jsmntok_t* t)
{
char* type;
switch ( t->type ){
case JSMN_STRING:
type = "STRING";
break;
case JSMN_OBJECT:
type = "OBJECT";
break;
case JSMN_ARRAY:
type = "ARRAY";
break;
case JSMN_PRIMITIVE:
type = "PRIMITIVE";
break;
}
printf("%s: S%d E%d C%d\n", type, t->start, t->end, t->size);
}
int main (int argc, char** argv)
{
char* filepath;
if ( argc > 1 ) {
filepath = argv[1];
} else {
printf("Usage: %s <filepath>\n",argv[0]);
exit(EXIT_SUCCESS);
}
char* js = json_fetch(filepath);
jsmntok_t* tokens = json_tokenise(js);
typedef enum {
START,
METRIC, METRIC_OBJECT,
SERIES, NODE_ARRAY,
NODE_OBJECT,
DATA,
SKIP,
STOP
} parse_state;
parse_state state = START;
size_t node_tokens = 0;
size_t skip_tokens = 0;
size_t metrics = 0;
size_t nodes = 0;
size_t elements = 0;
for (size_t i = 0, j = 1; j > 0; i++, j--)
{
jsmntok_t* t = &tokens[i];
if (t->type == JSMN_ARRAY || t->type == JSMN_OBJECT){
j += t->size;
}
print_token(t);
switch (state)
{
case START:
if (t->type != JSMN_OBJECT){
printf("Invalid response: root element must be object.");
exit(EXIT_SUCCESS);
}
state = METRIC;
break;
case METRIC:
if (t->type != JSMN_STRING){
printf("Invalid response: metric key must be a string.");
exit(EXIT_SUCCESS);
}
printf("METRIC\n");
state = METRIC_OBJECT;
object_tokens = t->size;
break;
case METRIC_OBJECT:
printf("METRIC OBJECT %lu\n", object_tokens);
object_tokens--;
if (t->type == JSMN_STRING && json_token_streq(js, t, "series")) {
state = SERIES;
} else {
state = SKIP;
if (t->type == JSMN_ARRAY || t->type == JSMN_OBJECT) {
skip_tokens = t->size;
}
}
// Last object value
if (object_tokens == 0) {
state = METRIC;
}
break;
case SKIP:
skip_tokens--;
printf("SKIP\n");
if (t->type == JSMN_ARRAY || t->type == JSMN_OBJECT) {
skip_tokens += t->size;
}
break;
case SERIES:
if (t->type != JSMN_ARRAY) {
printf("Unknown series value: expected array.");
}
printf("SERIES\n");
nodes = t->size;
state = NODE_ARRAY;
if (nodes == 0) {
state = METRIC_OBJECT;
}
break;
case NODE_ARRAY:
nodes--;
printf("NODE_ARRAY\n");
node_tokens = t->size;
state = NODE_OBJECT;
// Last node object
if (nodes == 0) {
state = STOP;
}
break;
case NODE_OBJECT:
node_tokens--;
printf("NODE_OBJECT\n");
// Keys are odd-numbered tokens within the object
if (node_tokens % 2 == 1)
{
if (t->type == JSMN_STRING && json_token_streq(js, t, "data")) {
state = DATA;
} else {
state = SKIP;
if (t->type == JSMN_ARRAY || t->type == JSMN_OBJECT) {
skip_tokens = t->size;
}
}
}
// Last object value
if (node_tokens == 0) {
state = NODE_ARRAY;
}
break;
case DATA:
if (t->type != JSMN_ARRAY || t->type != JSMN_STRING) {
printf("Unknown data value: expected string or array.");
}
if (t->type == JSMN_ARRAY) {
elements = t->size;
printf("%lu elements\n", elements );
state = SKIP;
skip_tokens = elements;
}
break;
case STOP:
// Just consume the tokens
break;
default:
printf("Invalid state %u", state);
}
}
free(tokens);
return (EXIT_SUCCESS);
}

48
jobTagger/src/timing.c Normal file
View File

@ -0,0 +1,48 @@
/*
* =======================================================================================
*
* Author: Jan Eitzinger (je), jan.eitzinger@fau.de
* Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
* =======================================================================================
*/
#include <stdlib.h>
#include <time.h>
double getTimeStamp()
{
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9;
}
double getTimeResolution()
{
struct timespec ts;
clock_getres(CLOCK_MONOTONIC, &ts);
return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9;
}
double getTimeStamp_()
{
return getTimeStamp();
}

35
jobTagger/src/timing.h Normal file
View File

@ -0,0 +1,35 @@
/*
* =======================================================================================
*
* Author: Jan Eitzinger (je), jan.eitzinger@fau.de
* Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
* =======================================================================================
*/
#ifndef __TIMING_H_
#define __TIMING_H_
extern double getTimeStamp();
extern double getTimeResolution();
extern double getTimeStamp_();
#endif

36
restruct.pl Executable file
View File

@ -0,0 +1,36 @@
#!/usr/bin/env perl
use strict;
use warnings;
use utf8;
use File::Copy;
my $trunk = '/home/jan/prg/HPCJobDatabase';
my $basedir = $ARGV[0];
my $destdir = $ARGV[1];
opendir my $dh, $basedir or die "can't open directory: $!";
while ( readdir $dh ) {
use integer;
chomp;
next if $_ eq '.' or $_ eq '..';
my $jobID = $_;
my $srcPath = "$trunk/$basedir/$jobID";
$jobID =~ s/\.eadm//;
my $level1 = $jobID/1000;
my $level2 = $jobID%1000;
my $dstPath = sprintf("%s/%s/%d/%03d", $trunk, $destdir, $level1, $level2);
# print "COPY from $srcPath to $dstPath\n";
# print "$trunk/$destdir/$level1\n";
if (not -d "$trunk/$destdir/$level1") {
mkdir "$trunk/$destdir/$level1";
}
move($srcPath, $dstPath);
}

68
syncDB.pl
View File

@ -48,8 +48,7 @@ my $dbh = DBI->connect(
my $sth_select_job = $dbh->prepare(qq{ my $sth_select_job = $dbh->prepare(qq{
SELECT id, user_id, job_id, cluster_id, SELECT id, user_id, job_id, cluster_id,
start_time, stop_time, duration, num_nodes, start_time, stop_time, duration, num_nodes,
has_profile, mem_used_max, flops_any_avg, mem_bw_avg, has_profile
ib_bw_avg, file_bw_avg
FROM job FROM job
WHERE job_id=? WHERE job_id=?
}); });
@ -63,8 +62,6 @@ my $sth_update_job = $dbh->prepare(qq{
WHERE id=?; WHERE id=?;
}); });
my $jobcount = 0;
my $wrongjobcount = 0;
my ($TS, $TE); my ($TS, $TE);
my $counter = 0; my $counter = 0;
@ -80,7 +77,7 @@ while ( <$fh> ) {
my $jobmeta_json = read_file("$basedir/$path1/$path2/meta.json"); my $jobmeta_json = read_file("$basedir/$path1/$path2/meta.json");
my $job = decode_json $jobmeta_json; my $job = decode_json $jobmeta_json;
my @row = $dbh->selectrow_array($sth_select_job, undef, $jobID); my @row = $dbh->selectrow_array($sth_select_job, undef, $jobID);
my ($db_id, $db_user_id, $db_job_id, $db_cluster_id, $db_start_time, $db_stop_time, $db_duration, $db_num_nodes); my ($db_id, $db_user_id, $db_job_id, $db_cluster_id, $db_start_time, $db_stop_time, $db_duration, $db_num_nodes, $db_has_profile);
# print Dumper($job); # print Dumper($job);
@ -92,42 +89,48 @@ while ( <$fh> ) {
$db_start_time, $db_start_time,
$db_stop_time, $db_stop_time,
$db_duration, $db_duration,
$db_num_nodes) = @row; $db_num_nodes,
$db_has_profile) = @row;
my $stats = $job->{statistics}; if ($db_has_profile == 0) {
if ( $job->{user_id} ne $db_user_id ) { my $stats = $job->{statistics};
print "jobID $jobID $job->{user_id} $db_user_id\n";
$job->{user_id} = $db_user_id;
}
# if ( $job->{start_time} != $db_start_time ) { if ( $job->{user_id} ne $db_user_id ) {
# print "start $jobID $job->{start_time} $db_start_time\n"; print "jobID $jobID $job->{user_id} $db_user_id\n";
# } $job->{user_id} = $db_user_id;
# if ( $job->{stop_time} != $db_stop_time ) {
# print "stop $jobID $job->{stop_time} $db_stop_time\n";
# }
if ( $job->{duration} != $db_duration ) {
my $difference = $job->{duration} - $db_duration;
if ( abs($difference) > 120 ) {
print "duration $jobID $job->{duration} $db_duration $difference\n";
} }
# if ( $job->{start_time} != $db_start_time ) {
# print "start $jobID $job->{start_time} $db_start_time\n";
# }
# if ( $job->{stop_time} != $db_stop_time ) {
# print "stop $jobID $job->{stop_time} $db_stop_time\n";
# }
if ( $job->{duration} != $db_duration ) {
my $difference = $job->{duration} - $db_duration;
if ( abs($difference) > 120 ) {
print "####duration $jobID $job->{duration} $db_duration $difference\n";
}
}
if ( $job->{num_nodes} != $db_num_nodes ) {
print "####num nodes $jobID $job->{num_nodes} $db_num_nodes\n";
}
$sth_update_job->execute(
1,
$stats->{mem_used}->{max},
$stats->{flops_any}->{avg},
$stats->{mem_bw}->{avg},
$db_id
);
} }
$sth_update_job->execute(
1,
$stats->{mem_used}->{max},
$stats->{flops_any}->{avg},
$stats->{mem_bw}->{avg},
$db_id
);
$jobcount++;
} else { } else {
print "$jobID NOT in DB!\n"; print "$jobID NOT in DB!\n";
} }
if ( $counter == 50 ) { if ( $counter == 100 ) {
$TE = time(); $TE = time();
my $rate = $counter/($TE-$TS); my $rate = $counter/($TE-$TS);
$counter = 0; $counter = 0;
@ -138,4 +141,3 @@ while ( <$fh> ) {
$dbh->disconnect; $dbh->disconnect;
close $fh; close $fh;
print "$wrongjobcount of $jobcount need update\n";

105
tagController.pl Executable file
View File

@ -0,0 +1,105 @@
#!/usr/bin/env perl
# =======================================================================================
#
# Author: Jan Eitzinger (je), jan.eitzinger@fau.de
# Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# =======================================================================================
use strict;
use warnings;
use utf8;
use File::Slurp;
use Data::Dumper;
use JSON::MaybeXS qw(encode_json decode_json);
use DBI;
my $database = $ARGV[0];
my $basedir = $ARGV[1];
my %attr = (
PrintError => 1,
RaiseError => 1
);
my $dbh = DBI->connect(
"DBI:SQLite:dbname=$database", "", "", \%attr)
or die "Could not connect to database: $DBI::errstr";
my $sth_select_job = $dbh->prepare(qq{
SELECT id, user_id, job_id, cluster_id,
start_time, stop_time, duration, num_nodes,
has_profile
FROM job
WHERE job_id=?
});
my $sth_update_job = $dbh->prepare(qq{
UPDATE job
SET has_profile = ?,
mem_used_max = ?,
flops_any_avg = ?,
mem_bw_avg = ?
WHERE id=?;
});
my $sth_select_tag = $dbh->prepare(qq{
SELECT id
FROM tag
WHERE tag_name=?
});
my ($TS, $TE);
my $counter = 0;
open(my $fhn, '>:encoding(UTF-8)', './jobIds-tagged.txt')
or die "Could not open file $!";
open(my $fh, '<:encoding(UTF-8)', './jobIds.txt')
or die "Could not open file $!";
$TS = time();
while ( <$fh> ) {
my $line = $_;
my ($jobID, $path1, $path2) = split ' ', $line;
$counter++;
my $json = read_file($jobDirectory.'/data.json');
my $data = decode_json $json;
$json = read_file($jobDirectory.'/meta.json');
my $meta = decode_json $json;
my @flopsAny = $data->{flops_any}->{series};
my @memBw = $data->{mem_bw}->{series};
if ( $counter == 20 ) {
$TE = time();
my $rate = $counter/($TE-$TS);
$counter = 0;
print "Processing $rate jobs per second\n";
$TS = $TE;
}
}
$dbh->disconnect;
close $fh;
close $fhn;