diff --git a/README.md b/README.md index dfafba6..ca9fea6 100644 --- a/README.md +++ b/README.md @@ -1,226 +1,9 @@ -# HPCJobDatabase -A standardized interface and reference implementation for HPC job data. -The DB and json schema specification is available in the [wiki](https://github.com/RRZE-HPC/HPCJobDatabase/wiki). - -# Dependencies - - * Getopt::Long - * Pod::Usage - * DateTime::Format::Strptime - * DBD::SQLite - # Setup -``` -sqlite3 jobDB < initDB.sql -``` +* Edit ```./graph/schema.graphqls``` +* Regenerate code: ```gqlgen generate``` +* Implement callbacks in ```graph/schema.resolvers.go``` -# Helper Scripts +# Run server -For all scripts apart from `acQuery.pl` the advice *use the source Luke* holds. - -Help text for acQuery: -``` -Usage: - acQuery.pl [options] -- - - Help Options: - --help Show help text - --man Show man page - --hasprofile Only show jobs with timerseries metric data - --mode Set the operation mode - --user Search for jobs of specific user - --project Search for jobs of specific project - --numnodes Specify range for number of nodes of job - --starttime Specify range for start time of jobs - --duration Specify duration range of jobs - --mem_used Specify range for average main memory capacity of job - --mem_bandwidth Specify range for average main memory bandwidth of job - --flops_any Specify range for average flop any rate of job - -Options: - --help Show a brief help information. - - --man Read the manual, with examples - - --hasprofile [true|false] Only show jobs with or without timerseries - metric data - - --mode [ids|query|count|list|stat|perf] Specify output mode. Mode can be - one of: - - ids - Print list of job ids matching conditions. One job id per - line. - - query - Print the query string and then exit. - count - Only output the number of jobs matching the conditions. - (Default mode) - - list - Output a record of every job matching the conditions. - - stat - Output job statistic for all jobs matching the - conditions. - - perf - Output job performance footprint statistic for all jobs - matching the conditions. - - --user Search job for a specific user id. - - --project Search job for a specific project. - - --duration Specify condition for job duration. This option takes two - arguments: If both arguments are positive integers the condition is - duration between first argument and second argument. If the second - argument is zero condition is duration smaller than first argument. If - first argument is zero condition is duration larger than second - argument. Duration can be in seconds, minutes (append m) or hours - (append h). - - --numnodes Specify condition for number of node range of job. This - option takes two arguments: If both arguments are positive integers the - condition is number of nodes between first argument and second argument. - If the second argument is zero condition is number of nodes smaller than - first argument. If first argument is zero condition is number of nodes - larger than second argument. - - --starttime Specify condition for the starttime of job. This option - takes two arguments: If both arguments are positive integers the - condition is start time between first argument and second argument. If - the second argument is zero condition is start time smaller than first - argument. If first argument is zero condition is start time larger than - second argument. Start time must be given as date in the following - format: %d.%m.%Y/%H:%M. - - --mem_used Specify condition for average main memory capacity used by - job. This option takes two arguments: If both arguments are positive - integers the condition is memory used is between first argument and - second argument. If the second argument is zero condition is memory used - is smaller than first argument. If first argument is zero condition is - memory used is larger than second argument. - - --mem_bandwidth Specify condition for average main memory bandwidth used - by job. This option takes two arguments: If both arguments are positive - integers the condition is memory bandwidth is between first argument and - second argument. If the second argument is zero condition is memory - bandwidth is smaller than first argument. If first argument is zero - condition is memory bandwidth is larger than second argument. - - --flops_any Specify condition for average flops any of job. This option - takes two arguments: If both arguments are positive integers the - condition is flops any is between first argument and second argument. If - the second argument is zero condition is flops any is smaller than first - argument. If first argument is zero condition is flops any is larger - than second argument. - -``` - -# Examples - -Query jobs with conditions: - -``` -[HPCJobDatabase] ./acQuery.pl --duration 20h 24h --starttime 01.08.2018/12:00 01.03.2019/12:00 -COUNT 6476 -``` - -Query jobs from alternative database file (default is jobDB): - -``` -[HPCJobDatabase] ./acQuery.pl --project project_30 --starttime 01.08.2018/12:00 01.03.2019/12:00 -- jobDB-anon-emmy -COUNT 21560 -``` - -Get job statistics output: - -``` -[HPCJobDatabase] ./acQuery.pl --project project_30 --mode stat --duration 0 20h --starttime 01.08.2018/12:00 01.03.2019/12:00 -- jobDB-anon-emmy -================================= -Job count: 747 -Total walltime [h]: 16334 -Total node hours [h]: 78966 - -Histogram: Number of nodes -nodes count -1 54 **** -2 1 -3 1 -4 36 **** -5 522 ******* -6 118 ***** -7 15 *** - -Histogram: Walltime -hours count -20 250 ****** -21 200 ****** -22 114 ***** -23 183 ****** -``` - -Get job performance statistics: - -``` -[HPCJobDatabase] ./acQuery.pl --project project_30 --mode perf --duration 0 20h --numnodes 1 4 --starttime 01.08.2018/12:00 01.03.2019/12:00 -- jobDB-anon-emmy -================================= -Job count: 92 -Jobs with performance profile: 48 -Total walltime [h]: 2070 -Total node hours [h]: 4332 - -Histogram: Mem used -Mem count -2 3 ** -3 4 ** -18 2 * -19 3 ** -20 2 * -21 1 -22 2 * -23 5 ** -24 2 * -25 1 -26 1 -27 3 ** -29 1 -30 2 * -31 1 -34 1 -35 1 -36 1 -41 1 -42 2 * -43 2 * -44 1 -49 1 -50 2 * -51 1 -52 1 -53 1 - -Histogram: Memory bandwidth -BW count -1 1 -2 9 *** -3 1 -4 1 -5 4 ** -6 2 * -7 10 *** -8 9 *** -9 11 *** - -Histogram: Flops any -flops count -1 3 ** -2 1 -3 4 ** -4 3 ** -5 9 *** -6 10 *** -7 11 *** -85 1 -225 1 -236 1 -240 2 * -244 2 * -``` +* ```go run server.go``` diff --git a/acSync.pl b/acSync.pl deleted file mode 100755 index e021368..0000000 --- a/acSync.pl +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env perl -# ======================================================================================= -# -# Author: Jan Eitzinger (je), jan.eitzinger@fau.de -# Copyright (c) 2019 RRZE, University Erlangen-Nuremberg -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# ======================================================================================= - -use strict; -use warnings; -use utf8; - -use File::Slurp; -use Data::Dumper; -use JSON::MaybeXS qw(encode_json decode_json); -use DBI; - -my $database = $ARGV[0]; -my $basedir = $ARGV[1]; - -my %attr = ( - PrintError => 1, - RaiseError => 1 -); - -my $dbh = DBI->connect( - "DBI:SQLite:dbname=$database", "", "", \%attr) - or die "Could not connect to database: $DBI::errstr"; - -my $sth_select_job = $dbh->prepare(qq{ - SELECT id, user_id, job_id, cluster_id, - start_time, stop_time, duration, num_nodes - FROM job - WHERE job_id=? - }); - -my $jobcount = 0; -my $wrongjobcount = 0; - -opendir my $dh, $basedir or die "can't open directory: $!"; -while ( readdir $dh ) { - chomp; - next if $_ eq '.' or $_ eq '..'; - - my $jobID = $_; - my $needsUpdate = 0; - - my $jobmeta_json = read_file("$basedir/$jobID/meta.json"); - my $job = decode_json $jobmeta_json; - my @row = $dbh->selectrow_array($sth_select_job, undef, $jobID); - - if ( @row ) { - - $jobcount++; - # print Dumper(@row); - my $duration_diff = abs($job->{duration} - $row[6]); - - if ( $duration_diff > 120 ) { - $needsUpdate = 1; - # print "$jobID DIFF DURATION $duration_diff\n"; - # print "CC $row[4] - $row[5]\n"; - # print "DB $job->{start_time} - $job->{stop_time}\n" - } - - if ( $row[7] != $job->{num_nodes} ){ - $needsUpdate = 1; - # print "$jobID DIFF NODES $row[7] $job->{num_nodes}\n"; - } - } else { - print "$jobID NOT in DB!\n"; - } - - if ( $needsUpdate ){ - $wrongjobcount++; - print "$jobID\n"; - } -} -closedir $dh or die "can't close directory: $!"; -$dbh->disconnect; - -print "$wrongjobcount of $jobcount need update\n"; diff --git a/anonDB.pl b/anonDB.pl deleted file mode 100755 index 38d19d1..0000000 --- a/anonDB.pl +++ /dev/null @@ -1,153 +0,0 @@ -#!/usr/bin/env perl -# ======================================================================================= -# -# Author: Jan Eitzinger (je), jan.eitzinger@fau.de -# Copyright (c) 2019 RRZE, University Erlangen-Nuremberg -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# ======================================================================================= - -use strict; -use warnings; -use utf8; - -use File::Slurp; -use Data::Dumper; -use JSON::MaybeXS qw(encode_json decode_json); -use DBI; - -my $database = $ARGV[0]; -my $basedir = $ARGV[1]; - -my %attr = ( - PrintError => 1, - RaiseError => 1 -); - -my $dbh = DBI->connect( - "DBI:SQLite:dbname=$database", "", "", \%attr) - or die "Could not connect to database: $DBI::errstr"; - -my $sth_select_all = $dbh->prepare(qq{ - SELECT id, user_id, project_id - FROM job; - }); - -my $sth_select_job = $dbh->prepare(qq{ - SELECT id, user_id, project_id - FROM job - WHERE job_id=?; - }); - -my $sth_update_job = $dbh->prepare(qq{ - UPDATE job - SET user_id = ?, - project_id = ? - WHERE id=?; - }); - -my $user_index = 0; my $project_index = 0; -my %user_lookup; my %project_lookup; -my %user_group; -my %row; - -# build lookups -$sth_select_all->execute; -$sth_select_all->bind_columns( \( @row{ @{$sth_select_all->{NAME_lc} } } )); - -while ($sth_select_all->fetch) { - my $user_id = $row{'user_id'}; - my $project_id = $row{'project_id'}; - - if ( not exists $user_lookup{$user_id}) { - $user_index++; - $user_lookup{$user_id} = $user_index; - $user_group{$user_id} = $project_id; - } - - if ( not exists $project_lookup{$project_id}) { - $project_index++; - $project_lookup{$project_id} = $project_index; - } -} - -write_file("user-conversion.json", encode_json \%user_lookup); -write_file("project-conversion.json", encode_json \%project_lookup); -print "$user_index total users\n"; -print "$project_index total projects\n"; - -# convert database -$sth_select_all->execute; -$sth_select_all->bind_columns( \( @row{ @{$sth_select_all->{NAME_lc} } } )); - -while ($sth_select_all->fetch) { - my $user_id = 'user_'.$user_lookup{$row{'user_id'}}; - my $project_id = 'project_'.$project_lookup{$row{'project_id'}}; - - # print "$row{'id'}: $user_id - $project_id\n"; - - $sth_update_job->execute( - $user_id, - $project_id, - $row{'id'} - ); -} - -open(my $fh, '<:encoding(UTF-8)', './jobIds.txt') - or die "Could not open file $!"; - -# convert job meta file -while ( <$fh> ) { - - my $line = $_; - my ($jobID, $path1, $path2) = split ' ', $line; - - my $json = read_file("$basedir/$path1/$path2/meta.json"); - my $job = decode_json $json; - - my $user = $job->{'user_id'}; - - # if ( $user =~ /^user_.*/ ) { - # print "$jobID $user\n"; - # } - - my $project; - - if ( exists $user_lookup{$user}) { - $project = $user_group{$user}; - $user = 'user_'.$user_lookup{$user}; - } else { - die "$user not in lookup hash!\n"; - } - - if ( exists $project_lookup{$project}) { - $project = 'project_'.$project_lookup{$project}; - } else { - die "$project not in lookup hash!\n"; - } - - $job->{user_id} = $user; - $job->{project_id} = $project; - $json = encode_json $job; - write_file("$basedir/$path1/$path2/meta.json", $json); -} -close $fh; - -$dbh->disconnect; diff --git a/extractJobIDs.pl b/extractJobIDs.pl deleted file mode 100755 index 3d50155..0000000 --- a/extractJobIDs.pl +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env perl -# ======================================================================================= -# -# Author: Jan Eitzinger (je), jan.eitzinger@fau.de -# Copyright (c) 2019 RRZE, University Erlangen-Nuremberg -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# ======================================================================================= - -use strict; -use warnings; -use utf8; - -my $basedir = $ARGV[0]; -open(my $fh, '>:encoding(UTF-8)', 'jobIds.txt') - or die "Could not open file $!"; - -opendir my $odh, $basedir or die "can't open directory: $!"; -my $count = 0; - -while ( readdir $odh ) { - chomp; - next if $_ eq '.' or $_ eq '..'; - - my $jobID1 = $_; - print "Open $jobID1\n"; - - opendir my $idh, "$basedir/$jobID1" or die "can't open directory: $!"; - - while ( readdir $idh ) { - chomp; - next if $_ eq '.' or $_ eq '..'; - my $jobID2 = $_; - - unless (-e "$basedir/$jobID1/$jobID2/data.json") { - print "$basedir/$jobID1/$jobID2/ File Doesn't Exist!\n"; - # rmdir "$basedir/$jobID1/$jobID2"; - $count++; - } else { - print $fh "$jobID1$jobID2.eadm $jobID1 $jobID2\n"; - } - - } - - closedir $idh or die "can't close directory: $!"; -} -closedir $odh or die "can't close directory: $!"; -close $fh; - -print "$count empty jobs!\n"; diff --git a/generatePlots.pl b/generatePlots.pl deleted file mode 100755 index 030fb7c..0000000 --- a/generatePlots.pl +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env perl - -use strict; -use warnings; -use utf8; - -use File::Slurp; -use Data::Dumper; -use JSON::MaybeXS qw(encode_json decode_json); - -my $jobDirectory = '../data'; - -sub gnuplotControl { - my $jobID = shift; - my $metricName = shift; - my $numNodes = shift; - my $unit = shift; - - my $gpMacros = <<"END"; -set terminal png size 1400,768 enhanced font ,12 -set output '$jobID-$metricName.png' -set xlabel 'runtime [s]' -set ylabel '[$unit]' -END - - $gpMacros .= "plot '$metricName.dat' u 2 w lines notitle"; - foreach my $col ( 3 ... $numNodes ){ - $gpMacros .= ", '$metricName.dat' u $col w lines notitle"; - } - - open(my $fh, '>:encoding(UTF-8)', './metric.plot') - or die "Could not open file $!"; - print $fh $gpMacros; - close $fh; - - system('gnuplot','metric.plot'); -} - -sub createPlot { - my $jobID = shift; - my $metricName = shift; - my $metric = shift; - my $unit = shift; - - my @lines; - - foreach my $node ( @$metric ) { - my $i = 0; - - foreach my $val ( @{$node->{data}} ){ - $lines[$i++] .= " $val"; - } - } - - open(my $fh, '>:encoding(UTF-8)', './'.$metricName.'.dat') - or die "Could not open file $!"; - - my $timestamp = 0; - - foreach my $line ( @lines ) { - print $fh $timestamp.$line."\n"; - $timestamp += 60; - } - - close $fh; - gnuplotControl($jobID, $metricName, $#$metric + 2, $unit); -} - -mkdir('./plots'); -chdir('./plots'); - -while ( <> ) { - my $jobID = $_; - $jobID =~ s/\.eadm//; - chomp $jobID; - - my $level1 = $jobID/1000; - my $level2 = $jobID%1000; - my $jobpath = sprintf("%s/%d/%03d", $jobDirectory, $level1, $level2); - - my $json = read_file($jobpath.'/data.json'); - my $data = decode_json $json; - $json = read_file($jobpath.'/meta.json'); - my $meta = decode_json $json; - - createPlot($jobID, 'flops_any', $data->{flops_any}->{series}, $data->{flops_any}->{unit}); - createPlot($jobID, 'mem_bw', $data->{mem_bw}->{series}, $data->{mem_bw}->{unit}); -} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..21d562e --- /dev/null +++ b/go.mod @@ -0,0 +1,12 @@ +module github.com/moebiusband/cc-jobarchive + +go 1.15 + +require ( + github.com/99designs/gqlgen v0.13.0 + github.com/gorilla/handlers v1.5.1 + github.com/gorilla/mux v1.6.1 + github.com/jmoiron/sqlx v1.3.1 // indirect + github.com/mattn/go-sqlite3 v1.14.6 + github.com/vektah/gqlparser/v2 v2.1.0 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..557d6cf --- /dev/null +++ b/go.sum @@ -0,0 +1,91 @@ +github.com/99designs/gqlgen v0.13.0 h1:haLTcUp3Vwp80xMVEg5KRNwzfUrgFdRmtBY8fuB8scA= +github.com/99designs/gqlgen v0.13.0/go.mod h1:NV130r6f4tpRWuAI+zsrSdooO/eWUv+Gyyoi3rEfXIk= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= +github.com/agnivade/levenshtein v1.0.3 h1:M5ZnqLOoZR8ygVq0FfkXsNOKzMCk0xRiow0R5+5VkQ0= +github.com/agnivade/levenshtein v1.0.3/go.mod h1:4SFRZbbXWLF4MU1T9Qg0pGgH3Pjs+t6ie5efyrwRJXs= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/trifles v0.0.0-20190318185328-a8d75aae118c/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= +github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-chi/chi v3.3.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= +github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= +github.com/gorilla/mux v1.6.1 h1:KOwqsTYZdeuMacU7CxjMNYEKeBvLbxW+psodrbcEa3A= +github.com/gorilla/mux v1.6.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/jmoiron/sqlx v1.3.1 h1:aLN7YINNZ7cYOPK3QC83dbM6KT0NMqVMw961TqrejlE= +github.com/jmoiron/sqlx v1.3.1/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/matryer/moq v0.0.0-20200106131100-75d0ddfc0007 h1:reVOUXwnhsYv/8UqjvhrMOu5CNT9UapHFLbQ2JcXsmg= +github.com/matryer/moq v0.0.0-20200106131100-75d0ddfc0007/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-sqlite3 v1.14.4 h1:4rQjbDxdu9fSgI/r3KN72G3c2goxknAqHHgPWWs8UlI= +github.com/mattn/go-sqlite3 v1.14.4/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= +github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= +github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mitchellh/mapstructure v0.0.0-20180203102830-a4e142e9c047 h1:zCoDWFD5nrJJVjbXiDZcVhOBSzKn3o9LgRLLMRNuru8= +github.com/mitchellh/mapstructure v0.0.0-20180203102830-a4e142e9c047/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/shurcooL/vfsgen v0.0.0-20180121065927-ffb13db8def0/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/urfave/cli/v2 v2.1.1 h1:Qt8FeAtxE/vfdrLmR3rxR6JRE0RoVmbXu8+6kZtYU4k= +github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= +github.com/vektah/dataloaden v0.2.1-0.20190515034641-a19b9a6e7c9e h1:+w0Zm/9gaWpEAyDlU1eKOuk5twTjAjuevXqcJJw8hrg= +github.com/vektah/dataloaden v0.2.1-0.20190515034641-a19b9a6e7c9e/go.mod h1:/HUdMve7rvxZma+2ZELQeNh88+003LL7Pf/CZ089j8U= +github.com/vektah/gqlparser/v2 v2.1.0 h1:uiKJ+T5HMGGQM2kRKQ8Pxw8+Zq9qhhZhz/lieYvCMns= +github.com/vektah/gqlparser/v2 v2.1.0/go.mod h1:SyUiHgLATUR8BiYURfTirrTcGpcE+4XkV2se04Px1Ms= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190515012406-7d7faa4812bd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20200114235610-7ae403b6b589 h1:rjUrONFu4kLchcZTfp3/96bR8bW8dIa8uz3cR5n0cgM= +golang.org/x/tools v0.0.0-20200114235610-7ae403b6b589/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +sourcegraph.com/sourcegraph/appdash v0.0.0-20180110180208-2cc67fd64755/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= +sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67/go.mod h1:L5q+DGLGOQFpo1snNEkLOJT2d1YTW66rWNzatr3He1k= diff --git a/gqlgen.yml b/gqlgen.yml new file mode 100644 index 0000000..51887f0 --- /dev/null +++ b/gqlgen.yml @@ -0,0 +1,60 @@ +# Where are all the schema files located? globs are supported eg src/**/*.graphqls +schema: + - graph/*.graphqls + +# Where should the generated server code go? +exec: + filename: graph/generated/generated.go + package: generated + +# Uncomment to enable federation +# federation: +# filename: graph/generated/federation.go +# package: generated + +# Where should any generated models go? +model: + filename: graph/model/models_gen.go + package: model + +# Where should the resolver implementations go? +resolver: + layout: follow-schema + dir: graph + package: graph + +# Optional: turn on use `gqlgen:"fieldName"` tags in your models +# struct_tag: json + +# Optional: turn on to use []Thing instead of []*Thing +# omit_slice_element_pointers: false + +# Optional: set to speed up generation time by not performing a final validation pass. +# skip_validation: true + +# gqlgen will search for any type names in the schema in these go packages +# if they match it will use them, otherwise it will generate them. +autobind: + - "fossil.moebiusband.org/jobaccounting-backend/graph/model" + +# This section declares type mapping between the GraphQL and go type systems +# +# The first line in each type will be used as defaults for resolver arguments and +# modelgen, the others will be allowed when binding to fields. Configure them to +# your liking +models: + ID: + model: + - github.com/99designs/gqlgen/graphql.ID + - github.com/99designs/gqlgen/graphql.Int + - github.com/99designs/gqlgen/graphql.Int64 + - github.com/99designs/gqlgen/graphql.Int32 + Int: + model: + - github.com/99designs/gqlgen/graphql.Int + - github.com/99designs/gqlgen/graphql.Int64 + - github.com/99designs/gqlgen/graphql.Int32 + Job: + model: "fossil.moebiusband.org/jobaccounting-backend/graph/model.Job" + Timestamp: + model: "fossil.moebiusband.org/jobaccounting-backend/graph/model.Timestamp" diff --git a/graph/model/models.go b/graph/model/models.go new file mode 100644 index 0000000..d4beb71 --- /dev/null +++ b/graph/model/models.go @@ -0,0 +1,25 @@ +package model + +import ( + "time" +) + +type Job struct { + ID string `json:"id"` + JobID string `json:"jobId" db:"job_id"` + UserID string `json:"userId" db:"user_id"` + ProjectID string `json:"projectId" db:"project_id"` + ClusterID string `json:"clusterId" db:"cluster_id"` + StartTime time.Time `json:"startTime" db:"start_time"` + Duration int `json:"duration" db:"duration"` + Walltime *int `json:"walltime" db:"walltime"` + Jobstate *string `json:"jobstate" db:"job_state"` + NumNodes int `json:"numNodes" db:"num_nodes"` + NodeList string `json:"nodelist" db:"node_list"` + HasProfile bool `json:"hasProfile" db:"has_profile"` + MemUsed_max *float64 `json:"memUsedMax" db:"mem_used_max"` + FlopsAny_avg *float64 `json:"flopsAnyAvg" db:"flops_any_avg"` + MemBw_avg *float64 `json:"memBwAvg" db:"mem_bw_avg"` + NetBw_avg *float64 `json:"netBwAvg" db:"net_bw_avg"` + FileBw_avg *float64 `json:"fileBwAvg" db:"file_bw_avg"` +} diff --git a/graph/resolver.go b/graph/resolver.go new file mode 100644 index 0000000..f318095 --- /dev/null +++ b/graph/resolver.go @@ -0,0 +1,248 @@ +package graph + +//go:generate go run github.com/99designs/gqlgen +import ( + "context" + "fmt" + "log" + "strings" + + "fossil.moebiusband.org/jobaccounting-backend/graph/generated" + "fossil.moebiusband.org/jobaccounting-backend/graph/model" + "github.com/jmoiron/sqlx" +) + +type Resolver struct { + DB *sqlx.DB +} + +func NewRootResolvers(db *sqlx.DB) generated.Config { + c := generated.Config{ + Resolvers: &Resolver{ + DB: db, + }, + } + + return c +} + +// Helper functions +func addStringCondition(conditions []string, field string, input *model.StringInput) []string { + if input.Eq != nil { + conditions = append(conditions, fmt.Sprintf("%s='%s'", field, *input.Eq)) + } + if input.StartsWith != nil { + conditions = append(conditions, fmt.Sprintf("%s LIKE '%s%%'", field, *input.StartsWith)) + } + if input.Contains != nil { + conditions = append(conditions, fmt.Sprintf("%s LIKE '%%%s%%'", field, *input.Contains)) + } + if input.EndsWith != nil { + conditions = append(conditions, fmt.Sprintf("%s LIKE '%%%s'", field, *input.EndsWith)) + } + + return conditions +} + +func addIntCondition(conditions []string, field string, input *model.IntRange) []string { + conditions = append(conditions, fmt.Sprintf("%s BETWEEN %d AND %d", field, input.From, input.To)) + return conditions +} + +func addTimeCondition(conditions []string, field string, input *model.TimeRange) []string { + conditions = append(conditions, fmt.Sprintf("%s BETWEEN %d AND %d", field, input.From.Unix(), input.To.Unix())) + return conditions +} + +func buildQueryConditions(filterList *model.JobFilterList) string { + var conditions []string + + for _, condition := range filterList.List { + if condition.JobID != nil { + conditions = addStringCondition(conditions, `job_id`, condition.JobID) + } + if condition.UserID != nil { + conditions = addStringCondition(conditions, `user_id`, condition.UserID) + } + if condition.ProjectID != nil { + conditions = addStringCondition(conditions, `project_id`, condition.ProjectID) + } + if condition.ClusterID != nil { + conditions = addStringCondition(conditions, `cluster_id`, condition.ClusterID) + } + if condition.StartTime != nil { + conditions = addTimeCondition(conditions, `start_time`, condition.StartTime) + } + if condition.Duration != nil { + conditions = addIntCondition(conditions, `duration`, condition.Duration) + } + if condition.NumNodes != nil { + conditions = addIntCondition(conditions, `num_nodes`, condition.NumNodes) + } + } + + return strings.Join(conditions, " AND ") +} + +// Queries + +func (r *queryResolver) JobByID( + ctx context.Context, + jobID string) (*model.Job, error) { + var job model.Job + qstr := `SELECT * from job ` + qstr += fmt.Sprintf("WHERE id=%s", jobID) + + row := r.DB.QueryRowx(qstr) + err := row.StructScan(&job) + if err != nil { + return nil, err + } + + return &job, nil +} + +func (r *queryResolver) Jobs( + ctx context.Context, + filterList *model.JobFilterList, + page *model.PageRequest, + orderBy *model.OrderByInput) (*model.JobResultList, error) { + + var jobs []*model.Job + var limit, offset int + var qc, ob string + + if page != nil { + limit = *page.Limit + offset = *page.Offset + } else { + limit = 20 + offset = 0 + } + + if filterList != nil { + qc = buildQueryConditions(filterList) + + if qc != "" { + qc = `WHERE ` + qc + } + } + + if orderBy != nil { + ob = fmt.Sprintf("ORDER BY %s %s", orderBy.Field, *orderBy.Order) + } + + qstr := `SELECT * ` + qstr += fmt.Sprintf("FROM job %s %s LIMIT %d OFFSET %d", qc, ob, limit, offset) + log.Printf("%s", qstr) + + rows, err := r.DB.Queryx(qstr) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var job model.Job + err := rows.StructScan(&job) + if err != nil { + fmt.Println(err) + } + jobs = append(jobs, &job) + } + + var count int + qstr = fmt.Sprintf("SELECT COUNT(*) FROM job %s", qc) + row := r.DB.QueryRow(qstr) + err = row.Scan(&count) + if err != nil { + return nil, err + } + + returnValue := model.JobResultList{ + jobs, + &offset, &limit, + &count} + + return &returnValue, nil +} + +func (r *queryResolver) JobsStatistics( + ctx context.Context, + filterList *model.JobFilterList) (*model.JobsStatistics, error) { + var qc string + + if filterList != nil { + qc = buildQueryConditions(filterList) + + if qc != "" { + qc = `WHERE ` + qc + } + } + + // TODO Change current node hours to core hours + qstr := `SELECT COUNT(*), SUM(duration)/3600, SUM(duration*num_nodes)/3600 ` + qstr += fmt.Sprintf("FROM job %s ", qc) + log.Printf("%s", qstr) + + var stats model.JobsStatistics + row := r.DB.QueryRow(qstr) + err := row.Scan(&stats.TotalJobs, &stats.TotalWalltime, &stats.TotalCoreHours) + if err != nil { + return nil, err + } + + qstr = `SELECT COUNT(*) ` + qstr += fmt.Sprintf("FROM job %s AND duration < 120", qc) + log.Printf("%s", qstr) + row = r.DB.QueryRow(qstr) + err = row.Scan(&stats.ShortJobs) + if err != nil { + return nil, err + } + + var histogram []*model.HistoPoint + // Node histogram + qstr = `SELECT num_nodes, COUNT(*) ` + qstr += fmt.Sprintf("FROM job %s GROUP BY 1", qc) + log.Printf("%s", qstr) + + rows, err := r.DB.Query(qstr) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var point model.HistoPoint + rows.Scan(&point.Count, &point.Value) + histogram = append(histogram, &point) + } + stats.HistNumNodes = histogram + + // Node histogram + qstr = `SELECT duration/3600, COUNT(*) ` + qstr += fmt.Sprintf("FROM job %s GROUP BY 1", qc) + log.Printf("%s", qstr) + + rows, err = r.DB.Query(qstr) + if err != nil { + return nil, err + } + defer rows.Close() + + histogram = nil + + for rows.Next() { + var point model.HistoPoint + rows.Scan(&point.Count, &point.Value) + histogram = append(histogram, &point) + } + stats.HistWalltime = histogram + + return &stats, nil +} + +func (r *Resolver) Query() generated.QueryResolver { return &queryResolver{r} } + +type queryResolver struct{ *Resolver } diff --git a/graph/schema.graphqls b/graph/schema.graphqls new file mode 100644 index 0000000..67bf2b2 --- /dev/null +++ b/graph/schema.graphqls @@ -0,0 +1,121 @@ +type Job { + id: ID! + jobId: String! + userId: String! + projectId: String! + clusterId: String! + startTime: Time! + duration: Int! + numNodes: Int! +} + +type Query { + jobById(jobId: String!): Job + jobs(filter: JobFilterList, page: PageRequest, order: OrderByInput): JobResultList! + jobsStatistics(filter: JobFilterList): JobsStatistics! +} + +type Mutation { + startJob(input: StartJobInput!): Job! + stopJob(input: StopJobInput!): Job! + addJob(input: AddJobInput!): Job! +} + +input StartJobInput { + jobId: String! + userId: String! + projectId: String! + clusterId: String! + startTime: Time! + numNodes: Int! +} + +input StopJobInput { + stopTime: Time! +} + +input AddJobInput { + jobId: String! + userId: String! + projectId: String! + clusterId: String! + startTime: Time! + duration: Int! + numNodes: Int! +} + + +input JobFilterList { + list: [JobFilter] +} + +input JobFilter { + jobId: StringInput + userId: StringInput + projectId: StringInput + clusterId: StringInput + duration: IntRange + numNodes: IntRange + startTime: TimeRange + hasProfile: Boolean +} + +input OrderByInput { + field: String! + order: SortDirectionEnum = ASC +} + +enum SortDirectionEnum { + DESC + ASC +} + +input StringInput { + eq: String + contains: String + startsWith: String + endsWith: String +} + +input IntRange { + from: Int! + to: Int! +} + +input FloatRange { + from: Float! + to: Float! +} + +input TimeRange { + from: Time! + to: Time! +} + +type JobResultList { + items: [Job]! + offset: Int + limit: Int + count: Int +} + +type HistoPoint { + count: Int! + value: Int! +} + +type JobsStatistics { + totalJobs: Int! + shortJobs: Int! + totalWalltime: Int! + totalCoreHours: Int! + histWalltime: [HistoPoint]! + histNumNodes: [HistoPoint]! +} + +input PageRequest { + itensPerPage: Int + page: Int +} + +scalar Time diff --git a/graph/schema.resolvers.go b/graph/schema.resolvers.go new file mode 100644 index 0000000..7f2540c --- /dev/null +++ b/graph/schema.resolvers.go @@ -0,0 +1,4 @@ +package graph + +// This file will be automatically regenerated based on the schema, any resolver implementations +// will be copied through when generating and any unknown code will be moved to the end. diff --git a/jobTag.pl b/jobTag.pl deleted file mode 100755 index 5baa4b3..0000000 --- a/jobTag.pl +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env perl -# ======================================================================================= -# -# Author: Jan Eitzinger (je), jan.eitzinger@fau.de -# Copyright (c) 2019 RRZE, University Erlangen-Nuremberg -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# ======================================================================================= - -use strict; -use warnings; -use utf8; - -use DBI; - -my $database = 'jobDB'; - -my %attr = ( - PrintError => 1, - RaiseError => 1 -); - -my $dbh = DBI->connect( - "DBI:SQLite:dbname=$database", "", "", \%attr) - or die "Could not connect to database: $DBI::errstr"; - -my $sth_select_tagged_jobs = $dbh->prepare(qq{ - SELECT j.* - FROM job j - JOIN jobtag jt ON j.id = jt.job_id - JOIN tag t ON jt.tag_id = t.id - WHERE t.name = ? - }); - -my $sth_select_job_tags = $dbh->prepare(qq{ - SELECT t.* - FROM tag t - JOIN jobtag jt ON t.id = jt.tag_id - JOIN job j ON jt.job_id = j.id - WHERE j.job_id = ? - }); - -my $sth_select_job = $dbh->prepare(qq{ - SELECT id - FROM job - WHERE job_id=? - }); - -my $sth_select_tag = $dbh->prepare(qq{ - SELECT id - FROM tag - WHERE name=? - }); - -my $sth_insert_tag = $dbh->prepare(qq{ - INSERT INTO tag(type,name) - VALUES(?,?) - }); - -my $sth_job_add_tag = $dbh->prepare(qq{ - INSERT INTO jobtag(job_id,tag_id) - VALUES(?,?) - }); - -my $sth_job_has_tag = $dbh->prepare(qq{ - SELECT id FROM job - WHERE job_id=? AND tag_id=? - }); - -my $CMD = $ARGV[0]; -my $JOB_ID = $ARGV[1]; -my $TAG_NAME = $ARGV[2]; - -my ($jid, $tid); - -# check if job exists -my @row = $dbh->selectrow_array($sth_select_job, undef, $JOB_ID); - -if ( @row ) { - $jid = $row[0]; -} else { - die "Job does not exist: $JOB_ID!\n"; -} - -# check if tag already exists -@row = $dbh->selectrow_array($sth_select_tag, undef, $TAG_NAME); - -if ( @row ) { - $tid = $row[0]; -} else { - print "Insert new tag: $TAG_NAME!\n"; - - $sth_insert_tag->execute('pathologic', $TAG_NAME); -} - -if ( $CMD eq 'ADD' ) { - @row = $dbh->selectrow_array($sth_job_has_tag, undef, $jid, $tid); - if ( @row ) { - die "Job already tagged!\n"; - } else { - print "Adding tag $TAG_NAME to job $JOB_ID!\n"; - $sth_job_add_tag($jid, $tid); - } -} -elsif ( $CMD eq 'RM' ) { - # elsif... -} -elsif ( $CMD eq 'LST' ) { - $sth_select_job_tags->execute; - my ($id, $type, $name); - - while(($id,$type,$name) = $sth->fetchrow()){ - print("$id, $type, $name\n"); - } - $sth_select_job_tags->finish(); -} -elsif ( $CMD eq 'LSJ' ) { - # elsif... -} -else { - die "Unknown command: $CMD!\n"; -} - -$dbh->disconnect(); diff --git a/jobTagger/Makefile b/jobTagger/Makefile deleted file mode 100644 index 07d028e..0000000 --- a/jobTagger/Makefile +++ /dev/null @@ -1,78 +0,0 @@ -#======================================================================================= -# -# Author: Jan Eitzinger (je), jan.eitzinger@fau.de -# Copyright (c) 2019 RRZE, University Erlangen-Nuremberg -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -#======================================================================================= -TAG = CLANG -#CONFIGURE BUILD SYSTEM -TARGET = jobTagger-$(TAG) -BUILD_DIR = ./$(TAG) -SRC_DIR = ./src -MAKE_DIR = ./ -Q ?= @ - -#DO NOT EDIT BELOW -include $(MAKE_DIR)/include_$(TAG).mk - -VPATH = $(SRC_DIR) -ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s,$(wildcard $(SRC_DIR)/*.c)) -OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o,$(wildcard $(SRC_DIR)/*.c)) -CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES) - - -${TARGET}: $(BUILD_DIR) $(OBJ) - @echo "===> LINKING $(TARGET)" - $(Q)${LINKER} ${LFLAGS} -o $(TARGET) $(OBJ) $(LIBS) - -asm: $(BUILD_DIR) $(ASM) - -$(BUILD_DIR)/%.o: %.c - @echo "===> COMPILE $@" - $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ - $(Q)$(GCC) $(CPPFLAGS) -MT $(@:.d=.o) -MM $< > $(BUILD_DIR)/$*.d - -$(BUILD_DIR)/%.s: %.c - @echo "===> GENERATE ASM $@" - $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ - -tags: - @echo "===> GENERATE TAGS" - $(Q)ctags -R - -$(BUILD_DIR): - @mkdir $(BUILD_DIR) - -ifeq ($(findstring $(MAKECMDGOALS),clean),) --include $(OBJ:.o=.d) -endif - -.PHONY: clean distclean - -clean: - @echo "===> CLEAN" - @rm -rf $(BUILD_DIR) - @rm -f tags - -distclean: clean - @echo "===> DIST CLEAN" - @rm -f $(TARGET) - @rm -f tags diff --git a/jobTagger/include_CLANG.mk b/jobTagger/include_CLANG.mk deleted file mode 100644 index 82df2ba..0000000 --- a/jobTagger/include_CLANG.mk +++ /dev/null @@ -1,10 +0,0 @@ -CC = clang -GCC = gcc -LINKER = $(CC) - -OPENMP = -fopenmp -CFLAGS = -Ofast -std=c99 $(OPENMP) -LFLAGS = $(OPENMP) -DEFINES = -D_GNU_SOURCE -DJSMN_PARENT_LINKS -INCLUDES = -LIBS = diff --git a/jobTagger/include_GCC.mk b/jobTagger/include_GCC.mk deleted file mode 100644 index 4ce8210..0000000 --- a/jobTagger/include_GCC.mk +++ /dev/null @@ -1,13 +0,0 @@ -CC = gcc -GCC = gcc -LINKER = $(CC) - -ifeq ($(ENABLE_OPENMP),true) -OPENMP = -fopenmp -endif - -CFLAGS = -Ofast -ffreestanding -std=c99 $(OPENMP) -LFLAGS = $(OPENMP) -DEFINES = -D_GNU_SOURCE -INCLUDES = -LIBS = diff --git a/jobTagger/include_ICC.mk b/jobTagger/include_ICC.mk deleted file mode 100644 index 210e0c3..0000000 --- a/jobTagger/include_ICC.mk +++ /dev/null @@ -1,13 +0,0 @@ -CC = icc -GCC = gcc -LINKER = $(CC) - -ifeq ($(ENABLE_OPENMP),true) -OPENMP = -qopenmp -endif - -CFLAGS = -qopt-report -Ofast -xHost -std=c99 -ffreestanding $(OPENMP) -LFLAGS = $(OPENMP) -DEFINES = -D_GNU_SOURCE -INCLUDES = -LIBS = diff --git a/jobTagger/src/affinity.c b/jobTagger/src/affinity.c deleted file mode 100644 index 19f4358..0000000 --- a/jobTagger/src/affinity.c +++ /dev/null @@ -1,88 +0,0 @@ -/* - * ======================================================================================= - * - * Author: Jan Eitzinger (je), jan.eitzinger@fau.de - * Copyright (c) 2019 RRZE, University Erlangen-Nuremberg - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - * ======================================================================================= - */ - -#ifdef __linux__ -#ifdef _OPENMP -#include -#include -#include -#include -#include -#include -#include - -#define MAX_NUM_THREADS 128 -#define gettid() syscall(SYS_gettid) - -static int -getProcessorID(cpu_set_t* cpu_set) -{ - int processorId; - - for ( processorId = 0; processorId < MAX_NUM_THREADS; processorId++ ) - { - if ( CPU_ISSET(processorId,cpu_set) ) - { - break; - } - } - return processorId; -} - -int -affinity_getProcessorId() -{ - cpu_set_t cpu_set; - CPU_ZERO(&cpu_set); - sched_getaffinity(gettid(),sizeof(cpu_set_t), &cpu_set); - - return getProcessorID(&cpu_set); -} - -void -affinity_pinThread(int processorId) -{ - cpu_set_t cpuset; - pthread_t thread; - - thread = pthread_self(); - CPU_ZERO(&cpuset); - CPU_SET(processorId, &cpuset); - pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset); -} - -void -affinity_pinProcess(int processorId) -{ - cpu_set_t cpuset; - - CPU_ZERO(&cpuset); - CPU_SET(processorId, &cpuset); - sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); -} -#endif /*_OPENMP*/ -#endif /*__linux__*/ diff --git a/jobTagger/src/affinity.h b/jobTagger/src/affinity.h deleted file mode 100644 index 80f531e..0000000 --- a/jobTagger/src/affinity.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * ======================================================================================= - * - * Author: Jan Eitzinger (je), jan.eitzinger@fau.de - * Copyright (c) 2019 RRZE, University Erlangen-Nuremberg - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - * ======================================================================================= - */ - -#ifndef AFFINITY_H -#define AFFINITY_H - -extern int affinity_getProcessorId(); -extern void affinity_pinProcess(int); -extern void affinity_pinThread(int); - -#endif /*AFFINITY_H*/ diff --git a/jobTagger/src/allocate.c b/jobTagger/src/allocate.c deleted file mode 100644 index 2ad9236..0000000 --- a/jobTagger/src/allocate.c +++ /dev/null @@ -1,58 +0,0 @@ -/* - * ======================================================================================= - * - * Author: Jan Eitzinger (je), jan.eitzinger@fau.de - * Copyright (c) 2019 RRZE, University Erlangen-Nuremberg - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - * ======================================================================================= - */ - -#include -#include -#include - -void* allocate (int alignment, size_t bytesize) -{ - int errorCode; - void* ptr; - - errorCode = posix_memalign(&ptr, alignment, bytesize); - - if (errorCode) { - if (errorCode == EINVAL) { - fprintf(stderr, - "Error: Alignment parameter is not a power of two\n"); - exit(EXIT_FAILURE); - } - if (errorCode == ENOMEM) { - fprintf(stderr, - "Error: Insufficient memory to fulfill the request\n"); - exit(EXIT_FAILURE); - } - } - - if (ptr == NULL) { - fprintf(stderr, "Error: posix_memalign failed!\n"); - exit(EXIT_FAILURE); - } - - return ptr; -} diff --git a/jobTagger/src/allocate.h b/jobTagger/src/allocate.h deleted file mode 100644 index 1e83fe0..0000000 --- a/jobTagger/src/allocate.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * ======================================================================================= - * - * Author: Jan Eitzinger (je), jan.eitzinger@fau.de - * Copyright (c) 2019 RRZE, University Erlangen-Nuremberg - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - * ======================================================================================= - */ - -#ifndef __ALLOCATE_H_ -#define __ALLOCATE_H_ - -extern void* allocate (int alignment, size_t bytesize); - -#endif diff --git a/jobTagger/src/jsmn.h b/jobTagger/src/jsmn.h deleted file mode 100644 index b95368a..0000000 --- a/jobTagger/src/jsmn.h +++ /dev/null @@ -1,468 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2010 Serge Zaitsev - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#ifndef JSMN_H -#define JSMN_H - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef JSMN_STATIC -#define JSMN_API static -#else -#define JSMN_API extern -#endif - -/** - * JSON type identifier. Basic types are: - * o Object - * o Array - * o String - * o Other primitive: number, boolean (true/false) or null - */ -typedef enum { - JSMN_UNDEFINED = 0, - JSMN_OBJECT = 1, - JSMN_ARRAY = 2, - JSMN_STRING = 3, - JSMN_PRIMITIVE = 4 -} jsmntype_t; - -enum jsmnerr { - /* Not enough tokens were provided */ - JSMN_ERROR_NOMEM = -1, - /* Invalid character inside JSON string */ - JSMN_ERROR_INVAL = -2, - /* The string is not a full JSON packet, more bytes expected */ - JSMN_ERROR_PART = -3 -}; - -/** - * JSON token description. - * type type (object, array, string etc.) - * start start position in JSON data string - * end end position in JSON data string - */ -typedef struct { - jsmntype_t type; - int start; - int end; - int size; -#ifdef JSMN_PARENT_LINKS - int parent; -#endif -} jsmntok_t; - -/** - * JSON parser. Contains an array of token blocks available. Also stores - * the string being parsed now and current position in that string. - */ -typedef struct { - unsigned int pos; /* offset in the JSON string */ - unsigned int toknext; /* next token to allocate */ - int toksuper; /* superior token node, e.g. parent object or array */ -} jsmn_parser; - -/** - * Create JSON parser over an array of tokens - */ -JSMN_API void jsmn_init(jsmn_parser *parser); - -/** - * Run JSON parser. It parses a JSON data string into and array of tokens, each - * describing - * a single JSON object. - */ -JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, const size_t len, - jsmntok_t *tokens, const unsigned int num_tokens); - -#ifndef JSMN_HEADER -/** - * Allocates a fresh unused token from the token pool. - */ -static jsmntok_t *jsmn_alloc_token(jsmn_parser *parser, jsmntok_t *tokens, - const size_t num_tokens) { - jsmntok_t *tok; - if (parser->toknext >= num_tokens) { - return NULL; - } - tok = &tokens[parser->toknext++]; - tok->start = tok->end = -1; - tok->size = 0; -#ifdef JSMN_PARENT_LINKS - tok->parent = -1; -#endif - return tok; -} - -/** - * Fills token type and boundaries. - */ -static void jsmn_fill_token(jsmntok_t *token, const jsmntype_t type, - const int start, const int end) { - token->type = type; - token->start = start; - token->end = end; - token->size = 0; -} - -/** - * Fills next available token with JSON primitive. - */ -static int jsmn_parse_primitive(jsmn_parser *parser, const char *js, - const size_t len, jsmntok_t *tokens, - const size_t num_tokens) { - jsmntok_t *token; - int start; - - start = parser->pos; - - for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { - switch (js[parser->pos]) { -#ifndef JSMN_STRICT - /* In strict mode primitive must be followed by "," or "}" or "]" */ - case ':': -#endif - case '\t': - case '\r': - case '\n': - case ' ': - case ',': - case ']': - case '}': - goto found; - } - if (js[parser->pos] < 32 || js[parser->pos] >= 127) { - parser->pos = start; - return JSMN_ERROR_INVAL; - } - } -#ifdef JSMN_STRICT - /* In strict mode primitive must be followed by a comma/object/array */ - parser->pos = start; - return JSMN_ERROR_PART; -#endif - -found: - if (tokens == NULL) { - parser->pos--; - return 0; - } - token = jsmn_alloc_token(parser, tokens, num_tokens); - if (token == NULL) { - parser->pos = start; - return JSMN_ERROR_NOMEM; - } - jsmn_fill_token(token, JSMN_PRIMITIVE, start, parser->pos); -#ifdef JSMN_PARENT_LINKS - token->parent = parser->toksuper; -#endif - parser->pos--; - return 0; -} - -/** - * Fills next token with JSON string. - */ -static int jsmn_parse_string(jsmn_parser *parser, const char *js, - const size_t len, jsmntok_t *tokens, - const size_t num_tokens) { - jsmntok_t *token; - - int start = parser->pos; - - parser->pos++; - - /* Skip starting quote */ - for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { - char c = js[parser->pos]; - - /* Quote: end of string */ - if (c == '\"') { - if (tokens == NULL) { - return 0; - } - token = jsmn_alloc_token(parser, tokens, num_tokens); - if (token == NULL) { - parser->pos = start; - return JSMN_ERROR_NOMEM; - } - jsmn_fill_token(token, JSMN_STRING, start + 1, parser->pos); -#ifdef JSMN_PARENT_LINKS - token->parent = parser->toksuper; -#endif - return 0; - } - - /* Backslash: Quoted symbol expected */ - if (c == '\\' && parser->pos + 1 < len) { - int i; - parser->pos++; - switch (js[parser->pos]) { - /* Allowed escaped symbols */ - case '\"': - case '/': - case '\\': - case 'b': - case 'f': - case 'r': - case 'n': - case 't': - break; - /* Allows escaped symbol \uXXXX */ - case 'u': - parser->pos++; - for (i = 0; i < 4 && parser->pos < len && js[parser->pos] != '\0'; - i++) { - /* If it isn't a hex character we have an error */ - if (!((js[parser->pos] >= 48 && js[parser->pos] <= 57) || /* 0-9 */ - (js[parser->pos] >= 65 && js[parser->pos] <= 70) || /* A-F */ - (js[parser->pos] >= 97 && js[parser->pos] <= 102))) { /* a-f */ - parser->pos = start; - return JSMN_ERROR_INVAL; - } - parser->pos++; - } - parser->pos--; - break; - /* Unexpected symbol */ - default: - parser->pos = start; - return JSMN_ERROR_INVAL; - } - } - } - parser->pos = start; - return JSMN_ERROR_PART; -} - -/** - * Parse JSON string and fill tokens. - */ -JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, const size_t len, - jsmntok_t *tokens, const unsigned int num_tokens) { - int r; - int i; - jsmntok_t *token; - int count = parser->toknext; - - for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { - char c; - jsmntype_t type; - - c = js[parser->pos]; - switch (c) { - case '{': - case '[': - count++; - if (tokens == NULL) { - break; - } - token = jsmn_alloc_token(parser, tokens, num_tokens); - if (token == NULL) { - return JSMN_ERROR_NOMEM; - } - if (parser->toksuper != -1) { - jsmntok_t *t = &tokens[parser->toksuper]; -#ifdef JSMN_STRICT - /* In strict mode an object or array can't become a key */ - if (t->type == JSMN_OBJECT) { - return JSMN_ERROR_INVAL; - } -#endif - t->size++; -#ifdef JSMN_PARENT_LINKS - token->parent = parser->toksuper; -#endif - } - token->type = (c == '{' ? JSMN_OBJECT : JSMN_ARRAY); - token->start = parser->pos; - parser->toksuper = parser->toknext - 1; - break; - case '}': - case ']': - if (tokens == NULL) { - break; - } - type = (c == '}' ? JSMN_OBJECT : JSMN_ARRAY); -#ifdef JSMN_PARENT_LINKS - if (parser->toknext < 1) { - return JSMN_ERROR_INVAL; - } - token = &tokens[parser->toknext - 1]; - for (;;) { - if (token->start != -1 && token->end == -1) { - if (token->type != type) { - return JSMN_ERROR_INVAL; - } - token->end = parser->pos + 1; - parser->toksuper = token->parent; - break; - } - if (token->parent == -1) { - if (token->type != type || parser->toksuper == -1) { - return JSMN_ERROR_INVAL; - } - break; - } - token = &tokens[token->parent]; - } -#else - for (i = parser->toknext - 1; i >= 0; i--) { - token = &tokens[i]; - if (token->start != -1 && token->end == -1) { - if (token->type != type) { - return JSMN_ERROR_INVAL; - } - parser->toksuper = -1; - token->end = parser->pos + 1; - break; - } - } - /* Error if unmatched closing bracket */ - if (i == -1) { - return JSMN_ERROR_INVAL; - } - for (; i >= 0; i--) { - token = &tokens[i]; - if (token->start != -1 && token->end == -1) { - parser->toksuper = i; - break; - } - } -#endif - break; - case '\"': - r = jsmn_parse_string(parser, js, len, tokens, num_tokens); - if (r < 0) { - return r; - } - count++; - if (parser->toksuper != -1 && tokens != NULL) { - tokens[parser->toksuper].size++; - } - break; - case '\t': - case '\r': - case '\n': - case ' ': - break; - case ':': - parser->toksuper = parser->toknext - 1; - break; - case ',': - if (tokens != NULL && parser->toksuper != -1 && - tokens[parser->toksuper].type != JSMN_ARRAY && - tokens[parser->toksuper].type != JSMN_OBJECT) { -#ifdef JSMN_PARENT_LINKS - parser->toksuper = tokens[parser->toksuper].parent; -#else - for (i = parser->toknext - 1; i >= 0; i--) { - if (tokens[i].type == JSMN_ARRAY || tokens[i].type == JSMN_OBJECT) { - if (tokens[i].start != -1 && tokens[i].end == -1) { - parser->toksuper = i; - break; - } - } - } -#endif - } - break; -#ifdef JSMN_STRICT - /* In strict mode primitives are: numbers and booleans */ - case '-': - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - case 't': - case 'f': - case 'n': - /* And they must not be keys of the object */ - if (tokens != NULL && parser->toksuper != -1) { - const jsmntok_t *t = &tokens[parser->toksuper]; - if (t->type == JSMN_OBJECT || - (t->type == JSMN_STRING && t->size != 0)) { - return JSMN_ERROR_INVAL; - } - } -#else - /* In non-strict mode every unquoted value is a primitive */ - default: -#endif - r = jsmn_parse_primitive(parser, js, len, tokens, num_tokens); - if (r < 0) { - return r; - } - count++; - if (parser->toksuper != -1 && tokens != NULL) { - tokens[parser->toksuper].size++; - } - break; - -#ifdef JSMN_STRICT - /* Unexpected char in strict mode */ - default: - return JSMN_ERROR_INVAL; -#endif - } - } - - if (tokens != NULL) { - for (i = parser->toknext - 1; i >= 0; i--) { - /* Unmatched opened object or array */ - if (tokens[i].start != -1 && tokens[i].end == -1) { - return JSMN_ERROR_PART; - } - } - } - - return count; -} - -/** - * Creates a new parser based over a given buffer with an array of tokens - * available. - */ -JSMN_API void jsmn_init(jsmn_parser *parser) { - parser->pos = 0; - parser->toknext = 0; - parser->toksuper = -1; -} - -#endif /* JSMN_HEADER */ - -#ifdef __cplusplus -} -#endif - -#endif /* JSMN_H */ diff --git a/jobTagger/src/main.c b/jobTagger/src/main.c deleted file mode 100644 index 8cdcfd5..0000000 --- a/jobTagger/src/main.c +++ /dev/null @@ -1,304 +0,0 @@ -/* - * ======================================================================================= - * - * Author: Jan Eitzinger (je), jan.eitzinger@fau.de - * Copyright (c) 2019 RRZE, University Erlangen-Nuremberg - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - * ======================================================================================= - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef _OPENMP -#include -#endif - -#include "jsmn.h" -#include "timing.h" -#include "allocate.h" -#include "affinity.h" - -#define HLINE "----------------------------------------------------------------------------\n" - -#ifndef MIN -#define MIN(x,y) ((x)<(y)?(x):(y)) -#endif -#ifndef MAX -#define MAX(x,y) ((x)>(y)?(x):(y)) -#endif -#ifndef ABS -#define ABS(a) ((a) >= 0 ? (a) : -(a)) -#endif - -char* json_fetch(char* filepath) -{ - int fd = open(filepath, O_RDONLY); - if ( fd == -1) { - perror("Cannot open output file\n"); exit(1); - } - int len = lseek(fd, 0, SEEK_END); - void *data = mmap(0, len, PROT_READ, MAP_PRIVATE, fd, 0); - - return (char*) data; -} - -jsmntok_t * json_tokenise(char *js) -{ - jsmn_parser parser; - jsmn_init(&parser); - - unsigned int n = 4096; - jsmntok_t *tokens = malloc(sizeof(jsmntok_t) * n); - - int ret = jsmn_parse(&parser, js, strlen(js), tokens, n); - - while (ret == JSMN_ERROR_NOMEM) - { - n = n * 2 + 1; - tokens = realloc(tokens, sizeof(jsmntok_t) * n); - ret = jsmn_parse(&parser, js, strlen(js), tokens, n); - } - - if (ret == JSMN_ERROR_INVAL) { - printf("jsmn_parse: invalid JSON string"); - exit(EXIT_SUCCESS); - } - if (ret == JSMN_ERROR_PART) { - printf("jsmn_parse: truncated JSON string"); - exit(EXIT_SUCCESS); - } - - return tokens; -} - -int json_token_streq(char* js, jsmntok_t* t, char* s) -{ - return (strncmp(js + t->start, s, t->end - t->start) == 0 - && strlen(s) == (size_t) (t->end - t->start)); -} - -char* json_token_tostr(char* js, jsmntok_t* t) -{ - js[t->end] = '\0'; - return js + t->start; -} - -void print_token(jsmntok_t* t) -{ - char* type; - - switch ( t->type ){ - case JSMN_STRING: - type = "STRING"; - break; - case JSMN_OBJECT: - type = "OBJECT"; - break; - case JSMN_ARRAY: - type = "ARRAY"; - break; - case JSMN_PRIMITIVE: - type = "PRIMITIVE"; - break; - } - - printf("%s: S%d E%d C%d\n", type, t->start, t->end, t->size); - -} - -int main (int argc, char** argv) -{ - char* filepath; - - if ( argc > 1 ) { - filepath = argv[1]; - } else { - printf("Usage: %s \n",argv[0]); - exit(EXIT_SUCCESS); - } - - char* js = json_fetch(filepath); - jsmntok_t* tokens = json_tokenise(js); - - typedef enum { - START, - METRIC, METRIC_OBJECT, - SERIES, NODE_ARRAY, - NODE_OBJECT, - DATA, - SKIP, - STOP - } parse_state; - - parse_state state = START; - size_t node_tokens = 0; - size_t skip_tokens = 0; - size_t metrics = 0; - size_t nodes = 0; - size_t elements = 0; - - for (size_t i = 0, j = 1; j > 0; i++, j--) - { - jsmntok_t* t = &tokens[i]; - - if (t->type == JSMN_ARRAY || t->type == JSMN_OBJECT){ - j += t->size; - } - print_token(t); - - switch (state) - { - case START: - if (t->type != JSMN_OBJECT){ - printf("Invalid response: root element must be object."); - exit(EXIT_SUCCESS); - } - - state = METRIC; - break; - - case METRIC: - if (t->type != JSMN_STRING){ - printf("Invalid response: metric key must be a string."); - exit(EXIT_SUCCESS); - } - - printf("METRIC\n"); - state = METRIC_OBJECT; - object_tokens = t->size; - break; - - case METRIC_OBJECT: - printf("METRIC OBJECT %lu\n", object_tokens); - object_tokens--; - - if (t->type == JSMN_STRING && json_token_streq(js, t, "series")) { - state = SERIES; - } else { - state = SKIP; - if (t->type == JSMN_ARRAY || t->type == JSMN_OBJECT) { - skip_tokens = t->size; - } - } - - // Last object value - if (object_tokens == 0) { - state = METRIC; - } - - break; - - case SKIP: - skip_tokens--; - - printf("SKIP\n"); - if (t->type == JSMN_ARRAY || t->type == JSMN_OBJECT) { - skip_tokens += t->size; - } - - break; - - case SERIES: - if (t->type != JSMN_ARRAY) { - printf("Unknown series value: expected array."); - } - - printf("SERIES\n"); - nodes = t->size; - state = NODE_ARRAY; - - if (nodes == 0) { - state = METRIC_OBJECT; - } - - break; - - case NODE_ARRAY: - nodes--; - - printf("NODE_ARRAY\n"); - node_tokens = t->size; - state = NODE_OBJECT; - - // Last node object - if (nodes == 0) { - state = STOP; - } - - break; - - case NODE_OBJECT: - node_tokens--; - - printf("NODE_OBJECT\n"); - // Keys are odd-numbered tokens within the object - if (node_tokens % 2 == 1) - { - if (t->type == JSMN_STRING && json_token_streq(js, t, "data")) { - state = DATA; - } else { - state = SKIP; - if (t->type == JSMN_ARRAY || t->type == JSMN_OBJECT) { - skip_tokens = t->size; - } - } - } - - // Last object value - if (node_tokens == 0) { - state = NODE_ARRAY; - } - - break; - - case DATA: - if (t->type != JSMN_ARRAY || t->type != JSMN_STRING) { - printf("Unknown data value: expected string or array."); - } - if (t->type == JSMN_ARRAY) { - elements = t->size; - printf("%lu elements\n", elements ); - state = SKIP; - skip_tokens = elements; - } - - break; - - case STOP: - // Just consume the tokens - break; - - default: - printf("Invalid state %u", state); - } - } - - free(tokens); - return (EXIT_SUCCESS); -} diff --git a/jobTagger/src/timing.c b/jobTagger/src/timing.c deleted file mode 100644 index 2daf260..0000000 --- a/jobTagger/src/timing.c +++ /dev/null @@ -1,48 +0,0 @@ -/* - * ======================================================================================= - * - * Author: Jan Eitzinger (je), jan.eitzinger@fau.de - * Copyright (c) 2019 RRZE, University Erlangen-Nuremberg - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - * ======================================================================================= - */ - -#include -#include - -double getTimeStamp() -{ - struct timespec ts; - clock_gettime(CLOCK_MONOTONIC, &ts); - return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; -} - -double getTimeResolution() -{ - struct timespec ts; - clock_getres(CLOCK_MONOTONIC, &ts); - return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; -} - -double getTimeStamp_() -{ - return getTimeStamp(); -} diff --git a/jobTagger/src/timing.h b/jobTagger/src/timing.h deleted file mode 100644 index b7260cb..0000000 --- a/jobTagger/src/timing.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * ======================================================================================= - * - * Author: Jan Eitzinger (je), jan.eitzinger@fau.de - * Copyright (c) 2019 RRZE, University Erlangen-Nuremberg - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - * ======================================================================================= - */ - -#ifndef __TIMING_H_ -#define __TIMING_H_ - -extern double getTimeStamp(); -extern double getTimeResolution(); -extern double getTimeStamp_(); - -#endif diff --git a/json-schema/cluster.schema.json b/json-schema/cluster.schema.json deleted file mode 100644 index 43baae1..0000000 --- a/json-schema/cluster.schema.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "HPC Cluster description", - "description": "Meta data information of a HPC cluster", - "type": "object", - "properties":{ - "cluster_id": { - "description": "The unique identifier of a cluster", - "type": "string" - }, - "processor_type": { - "description": "Processor type", - "type": "string" - }, - "sockets_per_node": { - "description": "Number of sockets per node", - "type": "integer" - }, - "cores_per_socket": { - "description": "Number of cores per socket", - "type": "integer" - }, - "threads_per_core": { - "description": "Number of SMT threads per core", - "type": "integer" - }, - "flop_rate_scalar": { - "description": "Theorethical node peak flop rate for scalar code in GFlops/s", - "type": "integer" - }, - "flop_rate_simd": { - "description": "Theorethical node peak flop rate for SIMD code in GFlops/s", - "type": "integer" - }, - "memory_bandwidth": { - "description": "Theorethical node peak memory bandwidth in GB/s", - "type": "integer" - } - }, - "required":[ - "cluster_id", - "processor_type", - "sockets_per_node", - "cores_per_socket", - "threads_per_core", - "flop_rate_scalar", - "flop_rate_simd", - "memory_bandwidth" - ] -} diff --git a/json-schema/job-data.schema.json b/json-schema/job-data.schema.json deleted file mode 100644 index 37f66be..0000000 --- a/json-schema/job-data.schema.json +++ /dev/null @@ -1,135 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Job metric data", - "description": "Meta data information of a HPC job", - "type": "object", - "properties": { - "mem_used": { - "description": "Memory capacity used (required)", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "flops_any": { - "description": "Total flop rate with DP flops scaled up (required)", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "mem_bw": { - "description": "Main memory bandwidth (required)", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "net_bw": { - "description": "Total fast interconnect network bandwidth (required)", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "file_bw": { - "description": "Total file IO bandwidth (required)", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "ipc": { - "description": "Instructions executed per cycle", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "cpu_used": { - "description": "CPU core utilization", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "flops_dp": { - "description": "Double precision flop rate", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "flops_sp": { - "description": "Single precision flops rate", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "rapl_power": { - "description": "CPU power consumption", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "gpu_used": { - "description": "GPU utilization", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "gpu_mem_used": { - "description": "GPU memory capacity used", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "gpu_power": { - "description": "GPU power consumption", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "clock": { - "description": "Average core frequency", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "eth_read_bw": { - "description": "Ethernet read bandwidth", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "eth_write_bw": { - "description": "Ethernet write bandwidth", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "lustre_read_bw": { - "description": "Lustre read bandwidth", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "lustre_write_bw": { - "description": "Lustre write bandwidth", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "lustre_read_req": { - "description": "Lustre read requests", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "lustre_write_req": { - "description": "Lustre write requests", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "lustre_inodes": { - "description": "Lustre inodes used", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "lustre_accesses": { - "description": "Lustre open and close", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "lustre_fsync": { - "description": "Lustre fsync", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "lustre_create": { - "description": "Lustre create", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "lustre_open": { - "description": "Lustre open", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "lustre_close": { - "description": "Lustre close", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "lustre_seek": { - "description": "Lustre seek", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "ib_read_bw": { - "description": "Infiniband read bandwidth", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "ib_write_bw": { - "description": "Infiniband write bandwidth", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - }, - "ib_congestion": { - "description": "Infiniband congestion", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-metric-data.schema.json" - } - }, - "required": [ - "mem_used", - "flops_any", - "mem_bw", - "net_bw", - "file_bw" - ] -} diff --git a/json-schema/job-meta.schema.json b/json-schema/job-meta.schema.json deleted file mode 100644 index 2434f43..0000000 --- a/json-schema/job-meta.schema.json +++ /dev/null @@ -1,238 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Job meta data", - "description": "Meta data information of a HPC job", - "type": "object", - "properties": { - "job_id": { - "description": "The unique identifier of a job", - "type": "string" - }, - "user_id": { - "description": "The unique identifier of a user", - "type": "string" - }, - "project_id": { - "description": "The unique identifier of a project", - "type": "string" - }, - "cluster_id": { - "description": "The unique identifier of a cluster", - "type": "string" - }, - "num_nodes": { - "description": "Number of nodes used", - "type": "integer", - "exclusiveMinimum": 0 - }, - "exclusive": { - "description": "Job uses only exclusive nodes", - "type": "boolean" - }, - "walltime": { - "description": "Requested walltime of job in seconds", - "type": "integer", - "exclusiveMinimum": 0 - }, - "job_state": { - "description": "Final state of job", - "type": "string", - "enum": [ - "completed", - "failed", - "canceled", - "timeout" - ] - }, - "start_time": { - "description": "Start epoch time stamp in seconds", - "type": "integer", - "exclusiveMinimum": 0 - }, - "stop_time": { - "description": "Stop epoch time stamp in seconds", - "type": "integer", - "exclusiveMinimum": 0 - }, - "duration": { - "description": "Duration of job in seconds", - "type": "integer", - "exclusiveMinimum": 0 - }, - "nodes": { - "description": "List of nodes", - "type": "array", - "items": { - "type": "string" - }, - "minItems": 1, - "uniqueItems": true - }, - "tags": { - "description": "List of tags", - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "type": { - "type": "string" - } - }, - "required": [ - "name", - "type" - ] - }, - "uniqueItems": true - }, - "statistics": { - "description": "Job statistic data", - "type": "object", - "properties": { - "mem_used": { - "description": "Memory capacity used (required)", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "flops_any": { - "description": "Total flop rate with DP flops scaled up (required)", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "mem_bw": { - "description": "Main memory bandwidth (required)", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "net_bw": { - "description": "Total fast interconnect network bandwidth (required)", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "file_bw": { - "description": "Total file IO bandwidth (required)", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "ipc": { - "description": "Instructions executed per cycle", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "cpu_used": { - "description": "CPU core utilization", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "flops_dp": { - "description": "Double precision flop rate", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "flops_sp": { - "description": "Single precision flops rate", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "rapl_power": { - "description": "CPU power consumption", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "gpu_used": { - "description": "GPU utilization", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "gpu_mem_used": { - "description": "GPU memory capacity used", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "gpu_power": { - "description": "GPU power consumption", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "clock": { - "description": "Average core frequency", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "eth_read_bw": { - "description": "Ethernet read bandwidth", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "eth_write_bw": { - "description": "Ethernet write bandwidth", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "lustre_read_bw": { - "description": "Lustre read bandwidth", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "lustre_write_bw": { - "description": "Lustre write bandwidth", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "lustre_read_req": { - "description": "Lustre read requests", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "lustre_write_req": { - "description": "Lustre write requests", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "lustre_inodes": { - "description": "Lustre inodes used", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "lustre_accesses": { - "description": "Lustre open and close", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "lustre_fsync": { - "description": "Lustre fsync", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "lustre_create": { - "description": "Lustre create", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "lustre_open": { - "description": "Lustre open", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "lustre_close": { - "description": "Lustre close", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "lustre_seek": { - "description": "Lustre seek", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "ib_read_bw": { - "description": "Infiniband read bandwidth", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "ib_write_bw": { - "description": "Infiniband write bandwidth", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - }, - "ib_congestion": { - "description": "Infiniband congestion", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-statistic.schema.json" - } - }, - "required": [ - "mem_used", - "flops_any", - "mem_bw", - "net_bw", - "file_bw" - ] - } - }, - "required": [ - "job_id", - "user_id", - "project_id", - "cluster_id", - "num_nodes", - "start_time", - "stop_time", - "duration", - "nodes", - "tags", - "statistics" - ] -} diff --git a/json-schema/job-metric-data.schema.json b/json-schema/job-metric-data.schema.json deleted file mode 100644 index 8d0cbb5..0000000 --- a/json-schema/job-metric-data.schema.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Job metric data", - "description": "Metric data of a HPC job", - "type": "object", - "properties": { - "unit": { - "description": "", - "type": "string" - }, - "scope": { - "description": "", - "type": "string", - "enum": [ - "node", - "cpu", - "socket" - ] - }, - "timestep": { - "description": "Measurement interval in seconds", - "type": "integer" - }, - "series": { - "description": "", - "type": "array", - "items": { - "type": "object", - "properties": { - "node_id": { - "type": "string" - }, - "id": { - "type": "integer" - }, - "statistics": { - "type": "object", - "properties": { - "avg": { - "description": "Series average", - "type": "number", - "minimum": 0 - }, - "min": { - "description": "Series minimum", - "type": "number", - "minimum": 0 - }, - "max": { - "description": "Series maximum", - "type": "number", - "minimum": 0 - } - }, - "required": [ - "avg", - "min", - "max" - ] - }, - "data": { - "type": "array", - "items": { - "type": "number", - "minimum": 0 - }, - "minItems": 1 - } - }, - "required": [ - "node_id", - "data" - ] - } - } - }, - "required": [ - "unit", - "scope", - "timestep", - "series" - ] -} diff --git a/json-schema/job-statistic.schema.json b/json-schema/job-statistic.schema.json deleted file mode 100644 index 04d5e93..0000000 --- a/json-schema/job-statistic.schema.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Job statistics", - "description": "Format specification for job metric statistics", - "type": "object", - "properties": { - "unit": { - "description": "Metric unit", - "#ref": "https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/unit.schema.json" - }, - "avg": { - "description": "Job metric average", - "type": "number", - "minimum": 0 - }, - "min": { - "description": "Job metric minimum", - "type": "number", - "minimum": 0 - }, - "max": { - "description": "Job metric maximum", - "type": "number", - "minimum": 0 - } - }, - "required": [ - "unit", - "avg", - "min", - "max" - ] -} diff --git a/json-schema/unit.schema.json b/json-schema/unit.schema.json deleted file mode 100644 index 1917361..0000000 --- a/json-schema/unit.schema.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Metric unit", - "description": "Format specification for job metric units", - "type": "object", - "properties": { - "base_unit": { - "description": "Metric base unit", - "type": "string", - "enum": [ - "B", - "F", - "B/s", - "F/s", - "CPI", - "IPC", - "Hz", - ] - }, - "prefix": { - "description": "Unit prefix", - "type": "string", - "enum": [ - "K", - "M", - "G", - "T", - "P", - "E" - ] - } - }, - "required": [ - "base_unit" - ] -} diff --git a/restruct.pl b/restruct.pl deleted file mode 100755 index b40649a..0000000 --- a/restruct.pl +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env perl - -use strict; -use warnings; -use utf8; - -use File::Copy; - -my $trunk = '/home/jan/prg/HPCJobDatabase'; -my $basedir = $ARGV[0]; -my $destdir = $ARGV[1]; - - -opendir my $dh, $basedir or die "can't open directory: $!"; -while ( readdir $dh ) { - use integer; - chomp; - next if $_ eq '.' or $_ eq '..'; - - my $jobID = $_; - my $srcPath = "$trunk/$basedir/$jobID"; - $jobID =~ s/\.eadm//; - - my $level1 = $jobID/1000; - my $level2 = $jobID%1000; - - my $dstPath = sprintf("%s/%s/%d/%03d", $trunk, $destdir, $level1, $level2); - # print "COPY from $srcPath to $dstPath\n"; - # print "$trunk/$destdir/$level1\n"; - if (not -d "$trunk/$destdir/$level1") { - mkdir "$trunk/$destdir/$level1"; - } - - move($srcPath, $dstPath); -} - diff --git a/server.go b/server.go new file mode 100644 index 0000000..591007a --- /dev/null +++ b/server.go @@ -0,0 +1,43 @@ +package main + +import ( + "log" + "net/http" + "os" + + "github.com/99designs/gqlgen/graphql/handler" + "github.com/99designs/gqlgen/graphql/playground" + "github.com/gorilla/handlers" + "github.com/gorilla/mux" + "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" + "github.com/moebiusband/cc-jobarchive/generated" + "github.com/moebiusband/cc-jobarchive/graph" +) + +const defaultPort = "8080" + +func main() { + port := os.Getenv("PORT") + if port == "" { + port = defaultPort + } + + db, err := sqlx.Open("sqlite3", "./job.db") + if err != nil { + log.Fatal(err) + } + defer db.Close() + + r := mux.NewRouter() + loggedRouter := handlers.LoggingHandler(os.Stdout, r) + srv := handler.NewDefaultServer(generated.NewExecutableSchema(generated.Config{Resolvers: &graph.Resolver{DB: db}})) + r.HandleFunc("/", playground.Handler("GraphQL playground", "/query")) + r.Handle("/query", srv) + + log.Printf("connect to http://localhost:%s/ for GraphQL playground", port) + log.Fatal(http.ListenAndServe("127.0.0.1:8080", + handlers.CORS(handlers.AllowedHeaders([]string{"X-Requested-With", "Content-Type", "Authorization"}), + handlers.AllowedMethods([]string{"GET", "POST", "HEAD", "OPTIONS"}), + handlers.AllowedOrigins([]string{"*"}))(loggedRouter))) +} diff --git a/syncDB.pl b/syncDB.pl deleted file mode 100755 index dc8441a..0000000 --- a/syncDB.pl +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env perl -# ======================================================================================= -# -# Author: Jan Eitzinger (je), jan.eitzinger@fau.de -# Copyright (c) 2019 RRZE, University Erlangen-Nuremberg -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# ======================================================================================= - -use strict; -use warnings; -use utf8; - -use File::Slurp; -use Data::Dumper; -use JSON::MaybeXS qw(encode_json decode_json); -use DBI; - -my $database = $ARGV[0]; -my $basedir = $ARGV[1]; - -my %attr = ( - PrintError => 1, - RaiseError => 1 -); - -my $dbh = DBI->connect( - "DBI:SQLite:dbname=$database", "", "", \%attr) - or die "Could not connect to database: $DBI::errstr"; - -my $sth_select_job = $dbh->prepare(qq{ - SELECT id, user_id, job_id, cluster_id, - start_time, stop_time, duration, num_nodes, - has_profile - FROM job - WHERE job_id=? - }); - -my $sth_update_job = $dbh->prepare(qq{ - UPDATE job - SET has_profile = ?, - mem_used_max = ?, - flops_any_avg = ?, - mem_bw_avg = ? - WHERE id=?; - }); - -my ($TS, $TE); -my $counter = 0; - -open(my $fh, '<:encoding(UTF-8)', './jobIds.txt') - or die "Could not open file $!"; -$TS = time(); - -while ( <$fh> ) { - - my ($jobID, $path1, $path2) = split ' ', $_; - $counter++; - - my $jobmeta_json = read_file("$basedir/$path1/$path2/meta.json"); - my $job = decode_json $jobmeta_json; - my @row = $dbh->selectrow_array($sth_select_job, undef, $jobID); - my ($db_id, $db_user_id, $db_job_id, $db_cluster_id, $db_start_time, $db_stop_time, $db_duration, $db_num_nodes, $db_has_profile); - - # print Dumper($job); - - if ( @row ) { - ($db_id, - $db_user_id, - $db_job_id, - $db_cluster_id, - $db_start_time, - $db_stop_time, - $db_duration, - $db_num_nodes, - $db_has_profile) = @row; - - if ($db_has_profile == 0) { - - my $stats = $job->{statistics}; - - if ( $job->{user_id} ne $db_user_id ) { - print "jobID $jobID $job->{user_id} $db_user_id\n"; - $job->{user_id} = $db_user_id; - } - - # if ( $job->{start_time} != $db_start_time ) { - # print "start $jobID $job->{start_time} $db_start_time\n"; - # } - # if ( $job->{stop_time} != $db_stop_time ) { - # print "stop $jobID $job->{stop_time} $db_stop_time\n"; - # } - if ( $job->{duration} != $db_duration ) { - my $difference = $job->{duration} - $db_duration; - if ( abs($difference) > 120 ) { - print "####duration $jobID $job->{duration} $db_duration $difference\n"; - } - } - - if ( $job->{num_nodes} != $db_num_nodes ) { - print "####num nodes $jobID $job->{num_nodes} $db_num_nodes\n"; - } - - $sth_update_job->execute( - 1, - $stats->{mem_used}->{max}, - $stats->{flops_any}->{avg}, - $stats->{mem_bw}->{avg}, - $db_id - ); - } - } else { - print "$jobID NOT in DB!\n"; - } - - if ( $counter == 100 ) { - $TE = time(); - my $rate = $counter/($TE-$TS); - $counter = 0; - print "Processing $rate jobs per second\n"; - $TS = $TE; - } -} -$dbh->disconnect; -close $fh; - diff --git a/tagController.pl b/tagController.pl deleted file mode 100755 index e9a33e4..0000000 --- a/tagController.pl +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env perl -# ======================================================================================= -# -# Author: Jan Eitzinger (je), jan.eitzinger@fau.de -# Copyright (c) 2019 RRZE, University Erlangen-Nuremberg -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# ======================================================================================= - -use strict; -use warnings; -use utf8; - -use File::Slurp; -use Data::Dumper; -use JSON::MaybeXS qw(encode_json decode_json); - -my $basedir = $ARGV[0]; -my $basedir = './data'; - - -my ($TS, $TE); -my $counter = 0; - -open(my $fhn, '>:encoding(UTF-8)', './jobIds-tagged.txt') - or die "Could not open file $!"; -open(my $fh, '<:encoding(UTF-8)', './jobIds.txt') - or die "Could not open file $!"; -$TS = time(); - -while ( <$fh> ) { - - my $line = $_; - my ($jobID, $system) = split '.', $line; - $counter++; - - # my $json = read_file($jobDirectory.'/data.json'); - # my $data = decode_json $json; - my $json = read_file($jobDirectory.'/meta.json'); - my $meta = decode_json $json; - - my $footprint = $meta->{statistics}; - - if ( $footprint->{flops_any}->{max} < 2.0 and $footprint->{mem_bw}->{max} < 2.0 ){ - print $fhn $jobID; - } - - if ( $counter == 20 ) { - $TE = time(); - my $rate = $counter/($TE-$TS); - $counter = 0; - print "Processing $rate jobs per second\n"; - $TS = $TE; - } -} -close $fh; -close $fhn; diff --git a/utils/README.md b/utils/README.md new file mode 100644 index 0000000..dfafba6 --- /dev/null +++ b/utils/README.md @@ -0,0 +1,226 @@ +# HPCJobDatabase +A standardized interface and reference implementation for HPC job data. +The DB and json schema specification is available in the [wiki](https://github.com/RRZE-HPC/HPCJobDatabase/wiki). + +# Dependencies + + * Getopt::Long + * Pod::Usage + * DateTime::Format::Strptime + * DBD::SQLite + +# Setup + +``` +sqlite3 jobDB < initDB.sql +``` + +# Helper Scripts + +For all scripts apart from `acQuery.pl` the advice *use the source Luke* holds. + +Help text for acQuery: +``` +Usage: + acQuery.pl [options] -- + + Help Options: + --help Show help text + --man Show man page + --hasprofile Only show jobs with timerseries metric data + --mode Set the operation mode + --user Search for jobs of specific user + --project Search for jobs of specific project + --numnodes Specify range for number of nodes of job + --starttime Specify range for start time of jobs + --duration Specify duration range of jobs + --mem_used Specify range for average main memory capacity of job + --mem_bandwidth Specify range for average main memory bandwidth of job + --flops_any Specify range for average flop any rate of job + +Options: + --help Show a brief help information. + + --man Read the manual, with examples + + --hasprofile [true|false] Only show jobs with or without timerseries + metric data + + --mode [ids|query|count|list|stat|perf] Specify output mode. Mode can be + one of: + + ids - Print list of job ids matching conditions. One job id per + line. + + query - Print the query string and then exit. + count - Only output the number of jobs matching the conditions. + (Default mode) + + list - Output a record of every job matching the conditions. + + stat - Output job statistic for all jobs matching the + conditions. + + perf - Output job performance footprint statistic for all jobs + matching the conditions. + + --user Search job for a specific user id. + + --project Search job for a specific project. + + --duration Specify condition for job duration. This option takes two + arguments: If both arguments are positive integers the condition is + duration between first argument and second argument. If the second + argument is zero condition is duration smaller than first argument. If + first argument is zero condition is duration larger than second + argument. Duration can be in seconds, minutes (append m) or hours + (append h). + + --numnodes Specify condition for number of node range of job. This + option takes two arguments: If both arguments are positive integers the + condition is number of nodes between first argument and second argument. + If the second argument is zero condition is number of nodes smaller than + first argument. If first argument is zero condition is number of nodes + larger than second argument. + + --starttime Specify condition for the starttime of job. This option + takes two arguments: If both arguments are positive integers the + condition is start time between first argument and second argument. If + the second argument is zero condition is start time smaller than first + argument. If first argument is zero condition is start time larger than + second argument. Start time must be given as date in the following + format: %d.%m.%Y/%H:%M. + + --mem_used Specify condition for average main memory capacity used by + job. This option takes two arguments: If both arguments are positive + integers the condition is memory used is between first argument and + second argument. If the second argument is zero condition is memory used + is smaller than first argument. If first argument is zero condition is + memory used is larger than second argument. + + --mem_bandwidth Specify condition for average main memory bandwidth used + by job. This option takes two arguments: If both arguments are positive + integers the condition is memory bandwidth is between first argument and + second argument. If the second argument is zero condition is memory + bandwidth is smaller than first argument. If first argument is zero + condition is memory bandwidth is larger than second argument. + + --flops_any Specify condition for average flops any of job. This option + takes two arguments: If both arguments are positive integers the + condition is flops any is between first argument and second argument. If + the second argument is zero condition is flops any is smaller than first + argument. If first argument is zero condition is flops any is larger + than second argument. + +``` + +# Examples + +Query jobs with conditions: + +``` +[HPCJobDatabase] ./acQuery.pl --duration 20h 24h --starttime 01.08.2018/12:00 01.03.2019/12:00 +COUNT 6476 +``` + +Query jobs from alternative database file (default is jobDB): + +``` +[HPCJobDatabase] ./acQuery.pl --project project_30 --starttime 01.08.2018/12:00 01.03.2019/12:00 -- jobDB-anon-emmy +COUNT 21560 +``` + +Get job statistics output: + +``` +[HPCJobDatabase] ./acQuery.pl --project project_30 --mode stat --duration 0 20h --starttime 01.08.2018/12:00 01.03.2019/12:00 -- jobDB-anon-emmy +================================= +Job count: 747 +Total walltime [h]: 16334 +Total node hours [h]: 78966 + +Histogram: Number of nodes +nodes count +1 54 **** +2 1 +3 1 +4 36 **** +5 522 ******* +6 118 ***** +7 15 *** + +Histogram: Walltime +hours count +20 250 ****** +21 200 ****** +22 114 ***** +23 183 ****** +``` + +Get job performance statistics: + +``` +[HPCJobDatabase] ./acQuery.pl --project project_30 --mode perf --duration 0 20h --numnodes 1 4 --starttime 01.08.2018/12:00 01.03.2019/12:00 -- jobDB-anon-emmy +================================= +Job count: 92 +Jobs with performance profile: 48 +Total walltime [h]: 2070 +Total node hours [h]: 4332 + +Histogram: Mem used +Mem count +2 3 ** +3 4 ** +18 2 * +19 3 ** +20 2 * +21 1 +22 2 * +23 5 ** +24 2 * +25 1 +26 1 +27 3 ** +29 1 +30 2 * +31 1 +34 1 +35 1 +36 1 +41 1 +42 2 * +43 2 * +44 1 +49 1 +50 2 * +51 1 +52 1 +53 1 + +Histogram: Memory bandwidth +BW count +1 1 +2 9 *** +3 1 +4 1 +5 4 ** +6 2 * +7 10 *** +8 9 *** +9 11 *** + +Histogram: Flops any +flops count +1 3 ** +2 1 +3 4 ** +4 3 ** +5 9 *** +6 10 *** +7 11 *** +85 1 +225 1 +236 1 +240 2 * +244 2 * +``` diff --git a/acImportPBS.pl b/utils/acImportPBS.pl similarity index 100% rename from acImportPBS.pl rename to utils/acImportPBS.pl diff --git a/acImportSlurm.pl b/utils/acImportSlurm.pl similarity index 100% rename from acImportSlurm.pl rename to utils/acImportSlurm.pl diff --git a/acQuery.pl b/utils/acQuery.pl similarity index 92% rename from acQuery.pl rename to utils/acQuery.pl index ed3a87f..7a96eed 100755 --- a/acQuery.pl +++ b/utils/acQuery.pl @@ -2,7 +2,7 @@ # ======================================================================================= # # Author: Jan Eitzinger (je), jan.eitzinger@fau.de -# Copyright (c) 2019 RRZE, University Erlangen-Nuremberg +# Copyright (c) 2020 RRZE, University Erlangen-Nuremberg # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -31,6 +31,7 @@ use utf8; use Data::Dumper; use Getopt::Long; use Pod::Usage; +use DateTime; use DateTime::Format::Strptime; use DBI; @@ -40,7 +41,7 @@ my ($add, $from, $to); my $dateParser = DateTime::Format::Strptime->new( - pattern => '%d.%m.%Y/%H:%M', + pattern => '%d.%m.%Y/%H:%M', time_zone => 'Europe/Berlin', on_error => 'undef' ); @@ -50,6 +51,7 @@ my $man = 0; my $hasprofile = ''; my $mode = 'count'; my $user = ''; +my $jobID = ''; my $project = ''; my @numnodes; my @starttime; @@ -59,18 +61,19 @@ my @mem_bandwidth; my @flops_any; GetOptions ( - 'help' => \$help, - 'man' => \$man, - 'hasprofile=s' => \$hasprofile, - 'mode=s' => \$mode, - 'user=s' => \$user, - 'project=s' => \$project, - 'numnodes=i{2}' => \@numnodes, - 'starttime=s{2}' => \@starttime, - 'duration=s{2}' => \@duration, - 'mem_used=i{2}' => \@mem_used, - 'mem_bandwidth=i{2}' => \@mem_bandwidth, - 'flops_any=i{2}' => \@flops_any + 'help' => \$help, + 'man' => \$man, + 'hasprofile=s' => \$hasprofile, + 'mode=s' => \$mode, + 'user=s' => \$user, + 'job=s' => \$jobID, + 'project=s' => \$project, + 'numnodes=i{2}' => \@numnodes, + 'starttime=s{2}' => \@starttime, + 'duration=s{2}' => \@duration, + 'mem_used=i{2}' => \@mem_used, + 'mem_bandwidth=i{2}' => \@mem_bandwidth, + 'flops_any=i{2}' => \@flops_any ) or pod2usage(2); my %attr = ( @@ -249,14 +252,17 @@ sub printJobStat { sub printJob { my $job = shift; + my $durationHours = sprintf("%.2f", $job->{duration}/3600); + my $startDatetime = DateTime->from_epoch(epoch=>$job->{start_time}, time_zone => 'Europe/Berlin',); + my $stopDatetime = DateTime->from_epoch(epoch=>$job->{stop_time}, time_zone => 'Europe/Berlin',); my $jobString = <<"END_JOB"; ================================= JobId: $job->{job_id} UserId: $job->{user_id} Number of nodes: $job->{num_nodes} -From $job->{start_time} to $job->{stop_time} -Duration $job->{duration} +From $startDatetime to $stopDatetime +Duration $durationHours hours END_JOB print $jobString; @@ -265,6 +271,17 @@ END_JOB pod2usage(1) if $help; pod2usage(-verbose => 2) if $man; +if ( $jobID ) { + my $sth = $dbh->prepare("SELECT * FROM job WHERE job_id=\'$jobID\'"); + $sth->execute; + my %row; + $sth->bind_columns( \( @row{ @{$sth->{NAME_lc} } } )); + while ($sth->fetch) { + printJob(\%row); + } + exit; +} + # build query conditions if ( $user ) { push @conditions, "user_id=\'$user\'"; @@ -274,7 +291,6 @@ if ( $project ) { push @conditions, "project_id=\'$project\'"; } - if ( @numnodes ) { ($add, $from, $to) = processRange($numnodes[0], $numnodes[1]); buildCondition('num_nodes'); @@ -384,6 +400,7 @@ acQuery.pl - Wrapper script to access sqlite job database. --man Show man page --hasprofile Only show jobs with timerseries metric data --mode Set the operation mode + --job Search for a specific job --user Search for jobs of specific user --project Search for jobs of specific project --numnodes Specify range for number of nodes of job diff --git a/initDB.sql b/utils/initDB.sql similarity index 100% rename from initDB.sql rename to utils/initDB.sql