From 95f616f45e3c8661a4f47bf569f56888b55a712a Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Mon, 11 Nov 2019 09:37:56 +0100 Subject: [PATCH] Sync state. Add job tagging scripts and siurce code. --- anonDB.pl | 25 +- extractJobIDs.pl | 12 +- generatePlots.pl | 88 +++++++ jobTag.pl | 68 ++++++ jobTagger/Makefile | 78 +++++++ jobTagger/include_CLANG.mk | 10 + jobTagger/include_GCC.mk | 13 ++ jobTagger/include_ICC.mk | 13 ++ jobTagger/src/affinity.c | 88 +++++++ jobTagger/src/affinity.h | 35 +++ jobTagger/src/allocate.c | 58 +++++ jobTagger/src/allocate.h | 33 +++ jobTagger/src/jsmn.h | 468 +++++++++++++++++++++++++++++++++++++ jobTagger/src/main.c | 304 ++++++++++++++++++++++++ jobTagger/src/timing.c | 48 ++++ jobTagger/src/timing.h | 35 +++ restruct.pl | 36 +++ syncDB.pl | 68 +++--- tagController.pl | 105 +++++++++ 19 files changed, 1539 insertions(+), 46 deletions(-) create mode 100755 generatePlots.pl create mode 100644 jobTag.pl create mode 100644 jobTagger/Makefile create mode 100644 jobTagger/include_CLANG.mk create mode 100644 jobTagger/include_GCC.mk create mode 100644 jobTagger/include_ICC.mk create mode 100644 jobTagger/src/affinity.c create mode 100644 jobTagger/src/affinity.h create mode 100644 jobTagger/src/allocate.c create mode 100644 jobTagger/src/allocate.h create mode 100644 jobTagger/src/jsmn.h create mode 100644 jobTagger/src/main.c create mode 100644 jobTagger/src/timing.c create mode 100644 jobTagger/src/timing.h create mode 100755 restruct.pl create mode 100755 tagController.pl diff --git a/anonDB.pl b/anonDB.pl index 317913b..38d19d1 100755 --- a/anonDB.pl +++ b/anonDB.pl @@ -110,15 +110,17 @@ while ($sth_select_all->fetch) { ); } -# convert job meta file -opendir my $dh, $basedir or die "can't open directory: $!"; -while ( readdir $dh ) { - chomp; - next if $_ eq '.' or $_ eq '..'; +open(my $fh, '<:encoding(UTF-8)', './jobIds.txt') + or die "Could not open file $!"; - my $jobID = $_; - my $jobmeta_json = read_file("$basedir/$jobID/meta.json"); - my $job = decode_json $jobmeta_json; +# convert job meta file +while ( <$fh> ) { + + my $line = $_; + my ($jobID, $path1, $path2) = split ' ', $line; + + my $json = read_file("$basedir/$path1/$path2/meta.json"); + my $job = decode_json $json; my $user = $job->{'user_id'}; @@ -143,10 +145,9 @@ while ( readdir $dh ) { $job->{user_id} = $user; $job->{project_id} = $project; - $jobmeta_json = encode_json $job; - # print "$jobmeta_json\n"; - write_file("$basedir/$jobID/meta.json", $jobmeta_json); + $json = encode_json $job; + write_file("$basedir/$path1/$path2/meta.json", $json); } -closedir $dh or die "can't close directory: $!"; +close $fh; $dbh->disconnect; diff --git a/extractJobIDs.pl b/extractJobIDs.pl index f5586c9..3d50155 100755 --- a/extractJobIDs.pl +++ b/extractJobIDs.pl @@ -33,6 +33,7 @@ open(my $fh, '>:encoding(UTF-8)', 'jobIds.txt') or die "Could not open file $!"; opendir my $odh, $basedir or die "can't open directory: $!"; +my $count = 0; while ( readdir $odh ) { chomp; @@ -48,10 +49,19 @@ while ( readdir $odh ) { next if $_ eq '.' or $_ eq '..'; my $jobID2 = $_; - print $fh "$jobID1$jobID2.eadm $jobID1 $jobID2\n"; + unless (-e "$basedir/$jobID1/$jobID2/data.json") { + print "$basedir/$jobID1/$jobID2/ File Doesn't Exist!\n"; + # rmdir "$basedir/$jobID1/$jobID2"; + $count++; + } else { + print $fh "$jobID1$jobID2.eadm $jobID1 $jobID2\n"; + } + } closedir $idh or die "can't close directory: $!"; } closedir $odh or die "can't close directory: $!"; close $fh; + +print "$count empty jobs!\n"; diff --git a/generatePlots.pl b/generatePlots.pl new file mode 100755 index 0000000..030fb7c --- /dev/null +++ b/generatePlots.pl @@ -0,0 +1,88 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use utf8; + +use File::Slurp; +use Data::Dumper; +use JSON::MaybeXS qw(encode_json decode_json); + +my $jobDirectory = '../data'; + +sub gnuplotControl { + my $jobID = shift; + my $metricName = shift; + my $numNodes = shift; + my $unit = shift; + + my $gpMacros = <<"END"; +set terminal png size 1400,768 enhanced font ,12 +set output '$jobID-$metricName.png' +set xlabel 'runtime [s]' +set ylabel '[$unit]' +END + + $gpMacros .= "plot '$metricName.dat' u 2 w lines notitle"; + foreach my $col ( 3 ... $numNodes ){ + $gpMacros .= ", '$metricName.dat' u $col w lines notitle"; + } + + open(my $fh, '>:encoding(UTF-8)', './metric.plot') + or die "Could not open file $!"; + print $fh $gpMacros; + close $fh; + + system('gnuplot','metric.plot'); +} + +sub createPlot { + my $jobID = shift; + my $metricName = shift; + my $metric = shift; + my $unit = shift; + + my @lines; + + foreach my $node ( @$metric ) { + my $i = 0; + + foreach my $val ( @{$node->{data}} ){ + $lines[$i++] .= " $val"; + } + } + + open(my $fh, '>:encoding(UTF-8)', './'.$metricName.'.dat') + or die "Could not open file $!"; + + my $timestamp = 0; + + foreach my $line ( @lines ) { + print $fh $timestamp.$line."\n"; + $timestamp += 60; + } + + close $fh; + gnuplotControl($jobID, $metricName, $#$metric + 2, $unit); +} + +mkdir('./plots'); +chdir('./plots'); + +while ( <> ) { + my $jobID = $_; + $jobID =~ s/\.eadm//; + chomp $jobID; + + my $level1 = $jobID/1000; + my $level2 = $jobID%1000; + my $jobpath = sprintf("%s/%d/%03d", $jobDirectory, $level1, $level2); + + my $json = read_file($jobpath.'/data.json'); + my $data = decode_json $json; + $json = read_file($jobpath.'/meta.json'); + my $meta = decode_json $json; + + createPlot($jobID, 'flops_any', $data->{flops_any}->{series}, $data->{flops_any}->{unit}); + createPlot($jobID, 'mem_bw', $data->{mem_bw}->{series}, $data->{mem_bw}->{unit}); +} diff --git a/jobTag.pl b/jobTag.pl new file mode 100644 index 0000000..a92d474 --- /dev/null +++ b/jobTag.pl @@ -0,0 +1,68 @@ +#!/usr/bin/env perl +# ======================================================================================= +# +# Author: Jan Eitzinger (je), jan.eitzinger@fau.de +# Copyright (c) 2019 RRZE, University Erlangen-Nuremberg +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# ======================================================================================= + +use strict; +use warnings; +use utf8; + +use DBI; + +my $database = 'jobDB'; + +my %attr = ( + PrintError => 1, + RaiseError => 1 +); + +my $dbh = DBI->connect( + "DBI:SQLite:dbname=$database", "", "", \%attr) + or die "Could not connect to database: $DBI::errstr"; + +my $sth_select_job = $dbh->prepare(qq{ + SELECT id, user_id, job_id, cluster_id, + start_time, stop_time, duration, num_nodes, + has_profile + FROM job + WHERE job_id=? + }); + +my $sth_update_job = $dbh->prepare(qq{ + UPDATE job + SET has_profile = ?, + mem_used_max = ?, + flops_any_avg = ?, + mem_bw_avg = ? + WHERE id=?; + }); + +my $sth_select_tag = $dbh->prepare(qq{ + SELECT id + FROM tag + WHERE tag_name=? + }); + +my $CMD = $ARGV[0]; + diff --git a/jobTagger/Makefile b/jobTagger/Makefile new file mode 100644 index 0000000..07d028e --- /dev/null +++ b/jobTagger/Makefile @@ -0,0 +1,78 @@ +#======================================================================================= +# +# Author: Jan Eitzinger (je), jan.eitzinger@fau.de +# Copyright (c) 2019 RRZE, University Erlangen-Nuremberg +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +#======================================================================================= +TAG = CLANG +#CONFIGURE BUILD SYSTEM +TARGET = jobTagger-$(TAG) +BUILD_DIR = ./$(TAG) +SRC_DIR = ./src +MAKE_DIR = ./ +Q ?= @ + +#DO NOT EDIT BELOW +include $(MAKE_DIR)/include_$(TAG).mk + +VPATH = $(SRC_DIR) +ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s,$(wildcard $(SRC_DIR)/*.c)) +OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o,$(wildcard $(SRC_DIR)/*.c)) +CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES) + + +${TARGET}: $(BUILD_DIR) $(OBJ) + @echo "===> LINKING $(TARGET)" + $(Q)${LINKER} ${LFLAGS} -o $(TARGET) $(OBJ) $(LIBS) + +asm: $(BUILD_DIR) $(ASM) + +$(BUILD_DIR)/%.o: %.c + @echo "===> COMPILE $@" + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + $(Q)$(GCC) $(CPPFLAGS) -MT $(@:.d=.o) -MM $< > $(BUILD_DIR)/$*.d + +$(BUILD_DIR)/%.s: %.c + @echo "===> GENERATE ASM $@" + $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ + +tags: + @echo "===> GENERATE TAGS" + $(Q)ctags -R + +$(BUILD_DIR): + @mkdir $(BUILD_DIR) + +ifeq ($(findstring $(MAKECMDGOALS),clean),) +-include $(OBJ:.o=.d) +endif + +.PHONY: clean distclean + +clean: + @echo "===> CLEAN" + @rm -rf $(BUILD_DIR) + @rm -f tags + +distclean: clean + @echo "===> DIST CLEAN" + @rm -f $(TARGET) + @rm -f tags diff --git a/jobTagger/include_CLANG.mk b/jobTagger/include_CLANG.mk new file mode 100644 index 0000000..82df2ba --- /dev/null +++ b/jobTagger/include_CLANG.mk @@ -0,0 +1,10 @@ +CC = clang +GCC = gcc +LINKER = $(CC) + +OPENMP = -fopenmp +CFLAGS = -Ofast -std=c99 $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE -DJSMN_PARENT_LINKS +INCLUDES = +LIBS = diff --git a/jobTagger/include_GCC.mk b/jobTagger/include_GCC.mk new file mode 100644 index 0000000..4ce8210 --- /dev/null +++ b/jobTagger/include_GCC.mk @@ -0,0 +1,13 @@ +CC = gcc +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -fopenmp +endif + +CFLAGS = -Ofast -ffreestanding -std=c99 $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE +INCLUDES = +LIBS = diff --git a/jobTagger/include_ICC.mk b/jobTagger/include_ICC.mk new file mode 100644 index 0000000..210e0c3 --- /dev/null +++ b/jobTagger/include_ICC.mk @@ -0,0 +1,13 @@ +CC = icc +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -qopenmp +endif + +CFLAGS = -qopt-report -Ofast -xHost -std=c99 -ffreestanding $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE +INCLUDES = +LIBS = diff --git a/jobTagger/src/affinity.c b/jobTagger/src/affinity.c new file mode 100644 index 0000000..19f4358 --- /dev/null +++ b/jobTagger/src/affinity.c @@ -0,0 +1,88 @@ +/* + * ======================================================================================= + * + * Author: Jan Eitzinger (je), jan.eitzinger@fau.de + * Copyright (c) 2019 RRZE, University Erlangen-Nuremberg + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * ======================================================================================= + */ + +#ifdef __linux__ +#ifdef _OPENMP +#include +#include +#include +#include +#include +#include +#include + +#define MAX_NUM_THREADS 128 +#define gettid() syscall(SYS_gettid) + +static int +getProcessorID(cpu_set_t* cpu_set) +{ + int processorId; + + for ( processorId = 0; processorId < MAX_NUM_THREADS; processorId++ ) + { + if ( CPU_ISSET(processorId,cpu_set) ) + { + break; + } + } + return processorId; +} + +int +affinity_getProcessorId() +{ + cpu_set_t cpu_set; + CPU_ZERO(&cpu_set); + sched_getaffinity(gettid(),sizeof(cpu_set_t), &cpu_set); + + return getProcessorID(&cpu_set); +} + +void +affinity_pinThread(int processorId) +{ + cpu_set_t cpuset; + pthread_t thread; + + thread = pthread_self(); + CPU_ZERO(&cpuset); + CPU_SET(processorId, &cpuset); + pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset); +} + +void +affinity_pinProcess(int processorId) +{ + cpu_set_t cpuset; + + CPU_ZERO(&cpuset); + CPU_SET(processorId, &cpuset); + sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); +} +#endif /*_OPENMP*/ +#endif /*__linux__*/ diff --git a/jobTagger/src/affinity.h b/jobTagger/src/affinity.h new file mode 100644 index 0000000..80f531e --- /dev/null +++ b/jobTagger/src/affinity.h @@ -0,0 +1,35 @@ +/* + * ======================================================================================= + * + * Author: Jan Eitzinger (je), jan.eitzinger@fau.de + * Copyright (c) 2019 RRZE, University Erlangen-Nuremberg + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * ======================================================================================= + */ + +#ifndef AFFINITY_H +#define AFFINITY_H + +extern int affinity_getProcessorId(); +extern void affinity_pinProcess(int); +extern void affinity_pinThread(int); + +#endif /*AFFINITY_H*/ diff --git a/jobTagger/src/allocate.c b/jobTagger/src/allocate.c new file mode 100644 index 0000000..2ad9236 --- /dev/null +++ b/jobTagger/src/allocate.c @@ -0,0 +1,58 @@ +/* + * ======================================================================================= + * + * Author: Jan Eitzinger (je), jan.eitzinger@fau.de + * Copyright (c) 2019 RRZE, University Erlangen-Nuremberg + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * ======================================================================================= + */ + +#include +#include +#include + +void* allocate (int alignment, size_t bytesize) +{ + int errorCode; + void* ptr; + + errorCode = posix_memalign(&ptr, alignment, bytesize); + + if (errorCode) { + if (errorCode == EINVAL) { + fprintf(stderr, + "Error: Alignment parameter is not a power of two\n"); + exit(EXIT_FAILURE); + } + if (errorCode == ENOMEM) { + fprintf(stderr, + "Error: Insufficient memory to fulfill the request\n"); + exit(EXIT_FAILURE); + } + } + + if (ptr == NULL) { + fprintf(stderr, "Error: posix_memalign failed!\n"); + exit(EXIT_FAILURE); + } + + return ptr; +} diff --git a/jobTagger/src/allocate.h b/jobTagger/src/allocate.h new file mode 100644 index 0000000..1e83fe0 --- /dev/null +++ b/jobTagger/src/allocate.h @@ -0,0 +1,33 @@ +/* + * ======================================================================================= + * + * Author: Jan Eitzinger (je), jan.eitzinger@fau.de + * Copyright (c) 2019 RRZE, University Erlangen-Nuremberg + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * ======================================================================================= + */ + +#ifndef __ALLOCATE_H_ +#define __ALLOCATE_H_ + +extern void* allocate (int alignment, size_t bytesize); + +#endif diff --git a/jobTagger/src/jsmn.h b/jobTagger/src/jsmn.h new file mode 100644 index 0000000..b95368a --- /dev/null +++ b/jobTagger/src/jsmn.h @@ -0,0 +1,468 @@ +/* + * MIT License + * + * Copyright (c) 2010 Serge Zaitsev + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef JSMN_H +#define JSMN_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef JSMN_STATIC +#define JSMN_API static +#else +#define JSMN_API extern +#endif + +/** + * JSON type identifier. Basic types are: + * o Object + * o Array + * o String + * o Other primitive: number, boolean (true/false) or null + */ +typedef enum { + JSMN_UNDEFINED = 0, + JSMN_OBJECT = 1, + JSMN_ARRAY = 2, + JSMN_STRING = 3, + JSMN_PRIMITIVE = 4 +} jsmntype_t; + +enum jsmnerr { + /* Not enough tokens were provided */ + JSMN_ERROR_NOMEM = -1, + /* Invalid character inside JSON string */ + JSMN_ERROR_INVAL = -2, + /* The string is not a full JSON packet, more bytes expected */ + JSMN_ERROR_PART = -3 +}; + +/** + * JSON token description. + * type type (object, array, string etc.) + * start start position in JSON data string + * end end position in JSON data string + */ +typedef struct { + jsmntype_t type; + int start; + int end; + int size; +#ifdef JSMN_PARENT_LINKS + int parent; +#endif +} jsmntok_t; + +/** + * JSON parser. Contains an array of token blocks available. Also stores + * the string being parsed now and current position in that string. + */ +typedef struct { + unsigned int pos; /* offset in the JSON string */ + unsigned int toknext; /* next token to allocate */ + int toksuper; /* superior token node, e.g. parent object or array */ +} jsmn_parser; + +/** + * Create JSON parser over an array of tokens + */ +JSMN_API void jsmn_init(jsmn_parser *parser); + +/** + * Run JSON parser. It parses a JSON data string into and array of tokens, each + * describing + * a single JSON object. + */ +JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, const size_t len, + jsmntok_t *tokens, const unsigned int num_tokens); + +#ifndef JSMN_HEADER +/** + * Allocates a fresh unused token from the token pool. + */ +static jsmntok_t *jsmn_alloc_token(jsmn_parser *parser, jsmntok_t *tokens, + const size_t num_tokens) { + jsmntok_t *tok; + if (parser->toknext >= num_tokens) { + return NULL; + } + tok = &tokens[parser->toknext++]; + tok->start = tok->end = -1; + tok->size = 0; +#ifdef JSMN_PARENT_LINKS + tok->parent = -1; +#endif + return tok; +} + +/** + * Fills token type and boundaries. + */ +static void jsmn_fill_token(jsmntok_t *token, const jsmntype_t type, + const int start, const int end) { + token->type = type; + token->start = start; + token->end = end; + token->size = 0; +} + +/** + * Fills next available token with JSON primitive. + */ +static int jsmn_parse_primitive(jsmn_parser *parser, const char *js, + const size_t len, jsmntok_t *tokens, + const size_t num_tokens) { + jsmntok_t *token; + int start; + + start = parser->pos; + + for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { + switch (js[parser->pos]) { +#ifndef JSMN_STRICT + /* In strict mode primitive must be followed by "," or "}" or "]" */ + case ':': +#endif + case '\t': + case '\r': + case '\n': + case ' ': + case ',': + case ']': + case '}': + goto found; + } + if (js[parser->pos] < 32 || js[parser->pos] >= 127) { + parser->pos = start; + return JSMN_ERROR_INVAL; + } + } +#ifdef JSMN_STRICT + /* In strict mode primitive must be followed by a comma/object/array */ + parser->pos = start; + return JSMN_ERROR_PART; +#endif + +found: + if (tokens == NULL) { + parser->pos--; + return 0; + } + token = jsmn_alloc_token(parser, tokens, num_tokens); + if (token == NULL) { + parser->pos = start; + return JSMN_ERROR_NOMEM; + } + jsmn_fill_token(token, JSMN_PRIMITIVE, start, parser->pos); +#ifdef JSMN_PARENT_LINKS + token->parent = parser->toksuper; +#endif + parser->pos--; + return 0; +} + +/** + * Fills next token with JSON string. + */ +static int jsmn_parse_string(jsmn_parser *parser, const char *js, + const size_t len, jsmntok_t *tokens, + const size_t num_tokens) { + jsmntok_t *token; + + int start = parser->pos; + + parser->pos++; + + /* Skip starting quote */ + for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { + char c = js[parser->pos]; + + /* Quote: end of string */ + if (c == '\"') { + if (tokens == NULL) { + return 0; + } + token = jsmn_alloc_token(parser, tokens, num_tokens); + if (token == NULL) { + parser->pos = start; + return JSMN_ERROR_NOMEM; + } + jsmn_fill_token(token, JSMN_STRING, start + 1, parser->pos); +#ifdef JSMN_PARENT_LINKS + token->parent = parser->toksuper; +#endif + return 0; + } + + /* Backslash: Quoted symbol expected */ + if (c == '\\' && parser->pos + 1 < len) { + int i; + parser->pos++; + switch (js[parser->pos]) { + /* Allowed escaped symbols */ + case '\"': + case '/': + case '\\': + case 'b': + case 'f': + case 'r': + case 'n': + case 't': + break; + /* Allows escaped symbol \uXXXX */ + case 'u': + parser->pos++; + for (i = 0; i < 4 && parser->pos < len && js[parser->pos] != '\0'; + i++) { + /* If it isn't a hex character we have an error */ + if (!((js[parser->pos] >= 48 && js[parser->pos] <= 57) || /* 0-9 */ + (js[parser->pos] >= 65 && js[parser->pos] <= 70) || /* A-F */ + (js[parser->pos] >= 97 && js[parser->pos] <= 102))) { /* a-f */ + parser->pos = start; + return JSMN_ERROR_INVAL; + } + parser->pos++; + } + parser->pos--; + break; + /* Unexpected symbol */ + default: + parser->pos = start; + return JSMN_ERROR_INVAL; + } + } + } + parser->pos = start; + return JSMN_ERROR_PART; +} + +/** + * Parse JSON string and fill tokens. + */ +JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, const size_t len, + jsmntok_t *tokens, const unsigned int num_tokens) { + int r; + int i; + jsmntok_t *token; + int count = parser->toknext; + + for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { + char c; + jsmntype_t type; + + c = js[parser->pos]; + switch (c) { + case '{': + case '[': + count++; + if (tokens == NULL) { + break; + } + token = jsmn_alloc_token(parser, tokens, num_tokens); + if (token == NULL) { + return JSMN_ERROR_NOMEM; + } + if (parser->toksuper != -1) { + jsmntok_t *t = &tokens[parser->toksuper]; +#ifdef JSMN_STRICT + /* In strict mode an object or array can't become a key */ + if (t->type == JSMN_OBJECT) { + return JSMN_ERROR_INVAL; + } +#endif + t->size++; +#ifdef JSMN_PARENT_LINKS + token->parent = parser->toksuper; +#endif + } + token->type = (c == '{' ? JSMN_OBJECT : JSMN_ARRAY); + token->start = parser->pos; + parser->toksuper = parser->toknext - 1; + break; + case '}': + case ']': + if (tokens == NULL) { + break; + } + type = (c == '}' ? JSMN_OBJECT : JSMN_ARRAY); +#ifdef JSMN_PARENT_LINKS + if (parser->toknext < 1) { + return JSMN_ERROR_INVAL; + } + token = &tokens[parser->toknext - 1]; + for (;;) { + if (token->start != -1 && token->end == -1) { + if (token->type != type) { + return JSMN_ERROR_INVAL; + } + token->end = parser->pos + 1; + parser->toksuper = token->parent; + break; + } + if (token->parent == -1) { + if (token->type != type || parser->toksuper == -1) { + return JSMN_ERROR_INVAL; + } + break; + } + token = &tokens[token->parent]; + } +#else + for (i = parser->toknext - 1; i >= 0; i--) { + token = &tokens[i]; + if (token->start != -1 && token->end == -1) { + if (token->type != type) { + return JSMN_ERROR_INVAL; + } + parser->toksuper = -1; + token->end = parser->pos + 1; + break; + } + } + /* Error if unmatched closing bracket */ + if (i == -1) { + return JSMN_ERROR_INVAL; + } + for (; i >= 0; i--) { + token = &tokens[i]; + if (token->start != -1 && token->end == -1) { + parser->toksuper = i; + break; + } + } +#endif + break; + case '\"': + r = jsmn_parse_string(parser, js, len, tokens, num_tokens); + if (r < 0) { + return r; + } + count++; + if (parser->toksuper != -1 && tokens != NULL) { + tokens[parser->toksuper].size++; + } + break; + case '\t': + case '\r': + case '\n': + case ' ': + break; + case ':': + parser->toksuper = parser->toknext - 1; + break; + case ',': + if (tokens != NULL && parser->toksuper != -1 && + tokens[parser->toksuper].type != JSMN_ARRAY && + tokens[parser->toksuper].type != JSMN_OBJECT) { +#ifdef JSMN_PARENT_LINKS + parser->toksuper = tokens[parser->toksuper].parent; +#else + for (i = parser->toknext - 1; i >= 0; i--) { + if (tokens[i].type == JSMN_ARRAY || tokens[i].type == JSMN_OBJECT) { + if (tokens[i].start != -1 && tokens[i].end == -1) { + parser->toksuper = i; + break; + } + } + } +#endif + } + break; +#ifdef JSMN_STRICT + /* In strict mode primitives are: numbers and booleans */ + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case 't': + case 'f': + case 'n': + /* And they must not be keys of the object */ + if (tokens != NULL && parser->toksuper != -1) { + const jsmntok_t *t = &tokens[parser->toksuper]; + if (t->type == JSMN_OBJECT || + (t->type == JSMN_STRING && t->size != 0)) { + return JSMN_ERROR_INVAL; + } + } +#else + /* In non-strict mode every unquoted value is a primitive */ + default: +#endif + r = jsmn_parse_primitive(parser, js, len, tokens, num_tokens); + if (r < 0) { + return r; + } + count++; + if (parser->toksuper != -1 && tokens != NULL) { + tokens[parser->toksuper].size++; + } + break; + +#ifdef JSMN_STRICT + /* Unexpected char in strict mode */ + default: + return JSMN_ERROR_INVAL; +#endif + } + } + + if (tokens != NULL) { + for (i = parser->toknext - 1; i >= 0; i--) { + /* Unmatched opened object or array */ + if (tokens[i].start != -1 && tokens[i].end == -1) { + return JSMN_ERROR_PART; + } + } + } + + return count; +} + +/** + * Creates a new parser based over a given buffer with an array of tokens + * available. + */ +JSMN_API void jsmn_init(jsmn_parser *parser) { + parser->pos = 0; + parser->toknext = 0; + parser->toksuper = -1; +} + +#endif /* JSMN_HEADER */ + +#ifdef __cplusplus +} +#endif + +#endif /* JSMN_H */ diff --git a/jobTagger/src/main.c b/jobTagger/src/main.c new file mode 100644 index 0000000..8cdcfd5 --- /dev/null +++ b/jobTagger/src/main.c @@ -0,0 +1,304 @@ +/* + * ======================================================================================= + * + * Author: Jan Eitzinger (je), jan.eitzinger@fau.de + * Copyright (c) 2019 RRZE, University Erlangen-Nuremberg + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * ======================================================================================= + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _OPENMP +#include +#endif + +#include "jsmn.h" +#include "timing.h" +#include "allocate.h" +#include "affinity.h" + +#define HLINE "----------------------------------------------------------------------------\n" + +#ifndef MIN +#define MIN(x,y) ((x)<(y)?(x):(y)) +#endif +#ifndef MAX +#define MAX(x,y) ((x)>(y)?(x):(y)) +#endif +#ifndef ABS +#define ABS(a) ((a) >= 0 ? (a) : -(a)) +#endif + +char* json_fetch(char* filepath) +{ + int fd = open(filepath, O_RDONLY); + if ( fd == -1) { + perror("Cannot open output file\n"); exit(1); + } + int len = lseek(fd, 0, SEEK_END); + void *data = mmap(0, len, PROT_READ, MAP_PRIVATE, fd, 0); + + return (char*) data; +} + +jsmntok_t * json_tokenise(char *js) +{ + jsmn_parser parser; + jsmn_init(&parser); + + unsigned int n = 4096; + jsmntok_t *tokens = malloc(sizeof(jsmntok_t) * n); + + int ret = jsmn_parse(&parser, js, strlen(js), tokens, n); + + while (ret == JSMN_ERROR_NOMEM) + { + n = n * 2 + 1; + tokens = realloc(tokens, sizeof(jsmntok_t) * n); + ret = jsmn_parse(&parser, js, strlen(js), tokens, n); + } + + if (ret == JSMN_ERROR_INVAL) { + printf("jsmn_parse: invalid JSON string"); + exit(EXIT_SUCCESS); + } + if (ret == JSMN_ERROR_PART) { + printf("jsmn_parse: truncated JSON string"); + exit(EXIT_SUCCESS); + } + + return tokens; +} + +int json_token_streq(char* js, jsmntok_t* t, char* s) +{ + return (strncmp(js + t->start, s, t->end - t->start) == 0 + && strlen(s) == (size_t) (t->end - t->start)); +} + +char* json_token_tostr(char* js, jsmntok_t* t) +{ + js[t->end] = '\0'; + return js + t->start; +} + +void print_token(jsmntok_t* t) +{ + char* type; + + switch ( t->type ){ + case JSMN_STRING: + type = "STRING"; + break; + case JSMN_OBJECT: + type = "OBJECT"; + break; + case JSMN_ARRAY: + type = "ARRAY"; + break; + case JSMN_PRIMITIVE: + type = "PRIMITIVE"; + break; + } + + printf("%s: S%d E%d C%d\n", type, t->start, t->end, t->size); + +} + +int main (int argc, char** argv) +{ + char* filepath; + + if ( argc > 1 ) { + filepath = argv[1]; + } else { + printf("Usage: %s \n",argv[0]); + exit(EXIT_SUCCESS); + } + + char* js = json_fetch(filepath); + jsmntok_t* tokens = json_tokenise(js); + + typedef enum { + START, + METRIC, METRIC_OBJECT, + SERIES, NODE_ARRAY, + NODE_OBJECT, + DATA, + SKIP, + STOP + } parse_state; + + parse_state state = START; + size_t node_tokens = 0; + size_t skip_tokens = 0; + size_t metrics = 0; + size_t nodes = 0; + size_t elements = 0; + + for (size_t i = 0, j = 1; j > 0; i++, j--) + { + jsmntok_t* t = &tokens[i]; + + if (t->type == JSMN_ARRAY || t->type == JSMN_OBJECT){ + j += t->size; + } + print_token(t); + + switch (state) + { + case START: + if (t->type != JSMN_OBJECT){ + printf("Invalid response: root element must be object."); + exit(EXIT_SUCCESS); + } + + state = METRIC; + break; + + case METRIC: + if (t->type != JSMN_STRING){ + printf("Invalid response: metric key must be a string."); + exit(EXIT_SUCCESS); + } + + printf("METRIC\n"); + state = METRIC_OBJECT; + object_tokens = t->size; + break; + + case METRIC_OBJECT: + printf("METRIC OBJECT %lu\n", object_tokens); + object_tokens--; + + if (t->type == JSMN_STRING && json_token_streq(js, t, "series")) { + state = SERIES; + } else { + state = SKIP; + if (t->type == JSMN_ARRAY || t->type == JSMN_OBJECT) { + skip_tokens = t->size; + } + } + + // Last object value + if (object_tokens == 0) { + state = METRIC; + } + + break; + + case SKIP: + skip_tokens--; + + printf("SKIP\n"); + if (t->type == JSMN_ARRAY || t->type == JSMN_OBJECT) { + skip_tokens += t->size; + } + + break; + + case SERIES: + if (t->type != JSMN_ARRAY) { + printf("Unknown series value: expected array."); + } + + printf("SERIES\n"); + nodes = t->size; + state = NODE_ARRAY; + + if (nodes == 0) { + state = METRIC_OBJECT; + } + + break; + + case NODE_ARRAY: + nodes--; + + printf("NODE_ARRAY\n"); + node_tokens = t->size; + state = NODE_OBJECT; + + // Last node object + if (nodes == 0) { + state = STOP; + } + + break; + + case NODE_OBJECT: + node_tokens--; + + printf("NODE_OBJECT\n"); + // Keys are odd-numbered tokens within the object + if (node_tokens % 2 == 1) + { + if (t->type == JSMN_STRING && json_token_streq(js, t, "data")) { + state = DATA; + } else { + state = SKIP; + if (t->type == JSMN_ARRAY || t->type == JSMN_OBJECT) { + skip_tokens = t->size; + } + } + } + + // Last object value + if (node_tokens == 0) { + state = NODE_ARRAY; + } + + break; + + case DATA: + if (t->type != JSMN_ARRAY || t->type != JSMN_STRING) { + printf("Unknown data value: expected string or array."); + } + if (t->type == JSMN_ARRAY) { + elements = t->size; + printf("%lu elements\n", elements ); + state = SKIP; + skip_tokens = elements; + } + + break; + + case STOP: + // Just consume the tokens + break; + + default: + printf("Invalid state %u", state); + } + } + + free(tokens); + return (EXIT_SUCCESS); +} diff --git a/jobTagger/src/timing.c b/jobTagger/src/timing.c new file mode 100644 index 0000000..2daf260 --- /dev/null +++ b/jobTagger/src/timing.c @@ -0,0 +1,48 @@ +/* + * ======================================================================================= + * + * Author: Jan Eitzinger (je), jan.eitzinger@fau.de + * Copyright (c) 2019 RRZE, University Erlangen-Nuremberg + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * ======================================================================================= + */ + +#include +#include + +double getTimeStamp() +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} + +double getTimeResolution() +{ + struct timespec ts; + clock_getres(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} + +double getTimeStamp_() +{ + return getTimeStamp(); +} diff --git a/jobTagger/src/timing.h b/jobTagger/src/timing.h new file mode 100644 index 0000000..b7260cb --- /dev/null +++ b/jobTagger/src/timing.h @@ -0,0 +1,35 @@ +/* + * ======================================================================================= + * + * Author: Jan Eitzinger (je), jan.eitzinger@fau.de + * Copyright (c) 2019 RRZE, University Erlangen-Nuremberg + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * ======================================================================================= + */ + +#ifndef __TIMING_H_ +#define __TIMING_H_ + +extern double getTimeStamp(); +extern double getTimeResolution(); +extern double getTimeStamp_(); + +#endif diff --git a/restruct.pl b/restruct.pl new file mode 100755 index 0000000..b40649a --- /dev/null +++ b/restruct.pl @@ -0,0 +1,36 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use utf8; + +use File::Copy; + +my $trunk = '/home/jan/prg/HPCJobDatabase'; +my $basedir = $ARGV[0]; +my $destdir = $ARGV[1]; + + +opendir my $dh, $basedir or die "can't open directory: $!"; +while ( readdir $dh ) { + use integer; + chomp; + next if $_ eq '.' or $_ eq '..'; + + my $jobID = $_; + my $srcPath = "$trunk/$basedir/$jobID"; + $jobID =~ s/\.eadm//; + + my $level1 = $jobID/1000; + my $level2 = $jobID%1000; + + my $dstPath = sprintf("%s/%s/%d/%03d", $trunk, $destdir, $level1, $level2); + # print "COPY from $srcPath to $dstPath\n"; + # print "$trunk/$destdir/$level1\n"; + if (not -d "$trunk/$destdir/$level1") { + mkdir "$trunk/$destdir/$level1"; + } + + move($srcPath, $dstPath); +} + diff --git a/syncDB.pl b/syncDB.pl index 1226c57..dc8441a 100755 --- a/syncDB.pl +++ b/syncDB.pl @@ -48,8 +48,7 @@ my $dbh = DBI->connect( my $sth_select_job = $dbh->prepare(qq{ SELECT id, user_id, job_id, cluster_id, start_time, stop_time, duration, num_nodes, - has_profile, mem_used_max, flops_any_avg, mem_bw_avg, - ib_bw_avg, file_bw_avg + has_profile FROM job WHERE job_id=? }); @@ -63,8 +62,6 @@ my $sth_update_job = $dbh->prepare(qq{ WHERE id=?; }); -my $jobcount = 0; -my $wrongjobcount = 0; my ($TS, $TE); my $counter = 0; @@ -80,7 +77,7 @@ while ( <$fh> ) { my $jobmeta_json = read_file("$basedir/$path1/$path2/meta.json"); my $job = decode_json $jobmeta_json; my @row = $dbh->selectrow_array($sth_select_job, undef, $jobID); - my ($db_id, $db_user_id, $db_job_id, $db_cluster_id, $db_start_time, $db_stop_time, $db_duration, $db_num_nodes); + my ($db_id, $db_user_id, $db_job_id, $db_cluster_id, $db_start_time, $db_stop_time, $db_duration, $db_num_nodes, $db_has_profile); # print Dumper($job); @@ -92,42 +89,48 @@ while ( <$fh> ) { $db_start_time, $db_stop_time, $db_duration, - $db_num_nodes) = @row; + $db_num_nodes, + $db_has_profile) = @row; - my $stats = $job->{statistics}; + if ($db_has_profile == 0) { - if ( $job->{user_id} ne $db_user_id ) { - print "jobID $jobID $job->{user_id} $db_user_id\n"; - $job->{user_id} = $db_user_id; - } + my $stats = $job->{statistics}; - # if ( $job->{start_time} != $db_start_time ) { - # print "start $jobID $job->{start_time} $db_start_time\n"; - # } - # if ( $job->{stop_time} != $db_stop_time ) { - # print "stop $jobID $job->{stop_time} $db_stop_time\n"; - # } - if ( $job->{duration} != $db_duration ) { - my $difference = $job->{duration} - $db_duration; - if ( abs($difference) > 120 ) { - print "duration $jobID $job->{duration} $db_duration $difference\n"; + if ( $job->{user_id} ne $db_user_id ) { + print "jobID $jobID $job->{user_id} $db_user_id\n"; + $job->{user_id} = $db_user_id; } + + # if ( $job->{start_time} != $db_start_time ) { + # print "start $jobID $job->{start_time} $db_start_time\n"; + # } + # if ( $job->{stop_time} != $db_stop_time ) { + # print "stop $jobID $job->{stop_time} $db_stop_time\n"; + # } + if ( $job->{duration} != $db_duration ) { + my $difference = $job->{duration} - $db_duration; + if ( abs($difference) > 120 ) { + print "####duration $jobID $job->{duration} $db_duration $difference\n"; + } + } + + if ( $job->{num_nodes} != $db_num_nodes ) { + print "####num nodes $jobID $job->{num_nodes} $db_num_nodes\n"; + } + + $sth_update_job->execute( + 1, + $stats->{mem_used}->{max}, + $stats->{flops_any}->{avg}, + $stats->{mem_bw}->{avg}, + $db_id + ); } - - $sth_update_job->execute( - 1, - $stats->{mem_used}->{max}, - $stats->{flops_any}->{avg}, - $stats->{mem_bw}->{avg}, - $db_id - ); - - $jobcount++; } else { print "$jobID NOT in DB!\n"; } - if ( $counter == 50 ) { + if ( $counter == 100 ) { $TE = time(); my $rate = $counter/($TE-$TS); $counter = 0; @@ -138,4 +141,3 @@ while ( <$fh> ) { $dbh->disconnect; close $fh; -print "$wrongjobcount of $jobcount need update\n"; diff --git a/tagController.pl b/tagController.pl new file mode 100755 index 0000000..2628ba1 --- /dev/null +++ b/tagController.pl @@ -0,0 +1,105 @@ +#!/usr/bin/env perl +# ======================================================================================= +# +# Author: Jan Eitzinger (je), jan.eitzinger@fau.de +# Copyright (c) 2019 RRZE, University Erlangen-Nuremberg +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# ======================================================================================= + +use strict; +use warnings; +use utf8; + +use File::Slurp; +use Data::Dumper; +use JSON::MaybeXS qw(encode_json decode_json); +use DBI; + +my $database = $ARGV[0]; +my $basedir = $ARGV[1]; + +my %attr = ( + PrintError => 1, + RaiseError => 1 +); + +my $dbh = DBI->connect( + "DBI:SQLite:dbname=$database", "", "", \%attr) + or die "Could not connect to database: $DBI::errstr"; + +my $sth_select_job = $dbh->prepare(qq{ + SELECT id, user_id, job_id, cluster_id, + start_time, stop_time, duration, num_nodes, + has_profile + FROM job + WHERE job_id=? + }); + +my $sth_update_job = $dbh->prepare(qq{ + UPDATE job + SET has_profile = ?, + mem_used_max = ?, + flops_any_avg = ?, + mem_bw_avg = ? + WHERE id=?; + }); + +my $sth_select_tag = $dbh->prepare(qq{ + SELECT id + FROM tag + WHERE tag_name=? + }); + + +my ($TS, $TE); +my $counter = 0; + +open(my $fhn, '>:encoding(UTF-8)', './jobIds-tagged.txt') + or die "Could not open file $!"; +open(my $fh, '<:encoding(UTF-8)', './jobIds.txt') + or die "Could not open file $!"; +$TS = time(); + +while ( <$fh> ) { + + my $line = $_; + my ($jobID, $path1, $path2) = split ' ', $line; + $counter++; + + my $json = read_file($jobDirectory.'/data.json'); + my $data = decode_json $json; + $json = read_file($jobDirectory.'/meta.json'); + my $meta = decode_json $json; + + my @flopsAny = $data->{flops_any}->{series}; + my @memBw = $data->{mem_bw}->{series}; + + if ( $counter == 20 ) { + $TE = time(); + my $rate = $counter/($TE-$TS); + $counter = 0; + print "Processing $rate jobs per second\n"; + $TS = $TE; + } +} +$dbh->disconnect; +close $fh; +close $fhn;