ft: new config and interface for correctness/compile/run, temporary disabled dissect, cleaned dev

src/auto_grader/cli.py

@@ -2,6 +2,8 @@ import click
 import os
 import shutil
 import tomllib
+import importlib.resources as resources
+
 
 from auto_grader.utils.display import SUCCESS_BOX, WARNING_BOX
 from auto_grader.commands.test.run import run
@@ -10,7 +12,7 @@ from auto_grader.commands.test.dissect import dissect
 from auto_grader.commands.test.correctness import correctness
 from auto_grader.commands.archives import list_assign, unpack
 
-import importlib.resources as resources
+from auto_grader.utils.config_parser import grdr_config
 
 
 def parse_config(config_path):
@@ -44,7 +46,6 @@ def init(path):
 @click.option('-o', '--output-path', default='./roots', type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True))
 @click.option('--yes', is_flag=True, default=False, help='Skip confirmation prompt.')
 def collect(number, path, output_path, yes):
-
     students = os.listdir(path)
     src_paths = [
         f"{path}/{student}/assignment_{number}/" for student in students]
@@ -76,10 +77,10 @@ def collect(number, path, output_path, yes):
                 handle.write(f"origin:\n\t{os.path.relpath(root)}\n")
 
 
-@click.command()
+# @click.command()
-@click.option('-p', '--path', default='', type=click.Path(exists=True, file_okay=True, dir_okay=True, resolve_path=True))
+# @click.option('-p', '--path', default='./solutions/assignment_4/.grdr_config.toml', type=click.Path(exists=False, file_okay=True, dir_okay=True, resolve_path=True))
-def dev(path):
+# def dev(path):
-    pass
+#     grdr_config(path)
 
 
 @click.group()
@@ -98,7 +99,7 @@ def test():
 
 
 cli.add_command(init)
-cli.add_command(dev)
+# cli.add_command(dev)
 cli.add_command(archives)
 cli.add_command(roots)
 cli.add_command(test)

src/auto_grader/commands/test/compile.py

@@ -1,9 +1,11 @@
 import click
+from click.core import ParameterSource
 import os
 
 from auto_grader.types import StudentVarType
 from auto_grader.utils.display import indent_text, ERROR_BOX, WARNING_BOX
 from auto_grader.utils import build_root, clean_root
+from auto_grader.utils.config_parser import grdr_config
 
 
 @click.command()
@@ -12,6 +14,10 @@ from auto_grader.utils import build_root, clean_root
               type=click.Path(exists=True, file_okay=False,
                               dir_okay=True, resolve_path=True),
               help='Path to directory conaining student roots in format (student)/assignment_[NUMBER]/root_(...).')
+@click.option('--solutions-path', default='./solutions',
+              type=click.Path(exists=False, file_okay=False,
+                              dir_okay=True, resolve_path=True),
+              help='Path to directory conaining reference solutions roots in format (solutions-dir)/assignment_[NUMBER]/ .')
 @click.option('-s', '--student', default='', type=StudentVarType(),
               help='Specify wich student to run the test for. If omitted test is run for all found students.')
 @click.option('-c', '--command', default='make', type=click.STRING,
@@ -21,10 +27,27 @@ from auto_grader.utils import build_root, clean_root
 @click.option('-t', '--timeout', default=10, type=click.INT,
               help='Sets timeout of both clean and build commands, defaults to 10s.')
 @click.option('--verbose', is_flag=True, default=False,
-              help='Sets timeout of both clean and build commands, defaults to 10s.')
+              help='Display the results of each attempted test as a detailed summary')
-def compile(number, path, command, student, timeout, clean_command, verbose):
+@click.option('--config-path', default=None,
+              type=click.Path(exists=False, file_okay=True,
+                              dir_okay=False, resolve_path=True),
+              help='Custom configuration path.'
+              'By default searches in the corresponding solution directory for a .grdf_config.toml.'
+              'If it points to a non existing path it will trated as an empty config.')
+@click.pass_context
+def compile(context: click.core.Context, number, path, solutions_path, command, student, timeout, clean_command, verbose, config_path):
     current_wd = os.getcwd()
     students = os.listdir(path)
+    if config_path == None:
+        config_path = solutions_path+f'/assignment_{number}/.grdr_config.toml'
+    config = grdr_config(config_path)
+    if context.get_parameter_source('command') != ParameterSource.COMMANDLINE and config.compile_config['command'] != None:
+        command = config.compile_config['command']
+    if context.get_parameter_source('clean-command') != ParameterSource.COMMANDLINE and config.compile_config['clean-command'] != None:
+        clean_command = config.compile_config['clean-command']
+    if context.get_parameter_source('timeout') != ParameterSource.COMMANDLINE and config.compile_config['timeout'] != None:
+        timeout = config.compile_config['timeout']
+
     if student and student not in students:
         click.echo(
             f"{ERROR_BOX}: No student {student} in {os.path.relpath(path)}.")

src/auto_grader/commands/test/correctness.py

@@ -1,32 +1,141 @@
 import click
+from click.core import ParameterSource
+
 import os
-import subprocess
-import shlex
-import tomllib
 import numpy as np
-from auto_grader.utils.display import ERROR_BOX, WARNING_BOX
+
+from auto_grader.utils import clean_root, build_root, run_root
+from auto_grader.utils.display import ERROR_BOX, WARNING_BOX, indent_text
+from auto_grader.utils.config_parser import grdr_config
+from auto_grader.types import StudentVarType, run_log
 
 
-@click.command()
+@click.command(help='Builds and Runs full student root. Then compares file if in numpy txt matrix format.\n' +
+               'Command run is: \n\n' +
+               '[COMMAND] [PROCOPT] [NUMPROC] [EXECUTABLE-PREFIX]<exe + flags autodetermined by config>\n\n' +
+               'Example with defaults:\n\n' +
+               'mpirun -n 4 <exe + flags>')
 @click.argument('number', required=True, type=click.INT)
-@click.option('-p', '--path', default='./roots', type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True))
+@click.option('-p', '--path', default='./roots',
-@click.option('-c', '--compile-command', default='make', type=click.STRING)
+              type=click.Path(exists=True, file_okay=False,
-@click.option('-r', '--run-command', default='mpirun', type=click.STRING)
+                              dir_okay=True, resolve_path=True),
-@click.option('-n', '--nproc-flag', default="-n 5", type=click.STRING)
+              help='Path to directory conaining student roots in format (student)/assignment_[NUMBER]/root_(...).')
-@click.option('-v', '--verbose', is_flag=True, default=False)
+@click.option('--tol', default=1e-1,
-@click.option('-s', '--solution-path', default='./solutions', type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True))
+              type=click.FLOAT,
-def correctness(number, solution_path, path, compile_command, run_command, nproc_flag, verbose):
+              help='Tollerance passed to numpy.allclose for result comparison')
+@click.option('-c', '--run-command', default='mpirun', type=click.STRING,
+              help='Overrides command that is used to run roots. The default command is `mpirun`.')
+@click.option('--procopt', default='-n ', type=click.STRING,
+              help='Indicates the option flag to be used to specify process number from [COMMAND]. The default value is `-n `.')
+@click.option('-n', '--numproc', default=4, type=click.INT,
+              help='Selects the number of mpi processes used to run the root. The default value is `4`.')
+@click.option('-t', '--run-timeout', default=10, type=click.INT,
+              help='Sets timeout for the run, defaults to 10s.')
+@click.option('--solutions-path', default='./solutions',
+              type=click.Path(exists=True, file_okay=False,
+                              dir_okay=True, resolve_path=True),
+              help='Path to directory conaining solution root in format assignment_[NUMBER]/. Solution must contain configuration grdr_config.toml')
+@click.option('-s', '--student', default='', type=StudentVarType(),
+              help='Specify wich student to run the test for. If omitted test is run for all found students.')
+@click.option('--executable-prefix', default='', type=click.STRING,
+              help='This prefix is appended to the executable name when the run command is launched (some launchers require `./`. The default value is ``.')
+@click.option('--compile-command', default='make', type=click.STRING,
+              help='Overrides command that is run to build roots. The default command is `make`.')
+@click.option('--clean-command', default='make distclean', type=click.STRING,
+              help='Overrides command that is run to clean roots. The default command is `make distclean`.')
+@click.option('--compile-timeout', default=10, type=click.INT,
+              help='Sets timeout of both clean and build commands, defaults to 10s.')
+@click.option('--verbose', is_flag=True, default=False,
+              help='Display the results of each attempted test as a detailed summary')
+@click.pass_context
+def correctness(
+        context,
+        number,
+        path,
+        tol,
+        run_command,
+        procopt,
+        numproc,
+        run_timeout,
+        student,
+        solutions_path,
+        executable_prefix,
+        compile_command,
+        clean_command,
+        compile_timeout,
+        verbose
+):
+
+    current_wd = os.getcwd()
     students = os.listdir(path)
+    config_path = solutions_path+f'/assignment_{number}/.grdr_config.toml'
+    config = grdr_config(config_path)
+
+    # --  Parse run options ---------------------------------------------------#
+    if context.get_parameter_source('run-command') != ParameterSource.COMMANDLINE and config.run_config['command'] != None:
+        run_command = config.run_config['command']
+
+    if context.get_parameter_source('procopt') != ParameterSource.COMMANDLINE and config.run_config['procopt'] != None:
+        procopt = config.run_config['procopt']
+
+    if context.get_parameter_source('numproc') != ParameterSource.COMMANDLINE and config.run_config['numproc'] != None:
+        numproc = config.run_config['numproc']
+
+    if context.get_parameter_source('run-timeout') != ParameterSource.COMMANDLINE and config.run_config['timeout'] != None:
+        run_timeout = config.run_config['timeout']
+
+    if len(config.run_config['_signatures']) == 0:
+        click.echo(
+            f"{ERROR_BOX}: No signature detected in configuration file. A signature list, even if empty (signature = []) us required)")
+
+    # --  Parse compile options  ----------------------------------------------#
+    if context.get_parameter_source('compile-command') != ParameterSource.COMMANDLINE and config.compile_config['command'] != None:
+        compile_command = config.compile_config['command']
+
+    if context.get_parameter_source('clean-command') != ParameterSource.COMMANDLINE and config.compile_config['clean-command'] != None:
+        clean_command = config.compile_config['clean-command']
+
+    if context.get_parameter_source('compile-timeout') != ParameterSource.COMMANDLINE and config.compile_config['timeout'] != None:
+        compile_timeout = config.compile_config['timeout']
+
+    # -------------------------------------------------------------------------#
+    if config.correctness_config["ref-solution"] == None:
+        click.echo(
+            f"{ERROR_BOX}: No ref-solution detected in configuration file.")
+        return
+    reference_solution = np.empty((0))
+    try:
+        reference_solution = np.loadtxt(
+            config.correctness_config["ref-solution"])
+    except Exception as e:
+        click.echo(
+            f"{ERROR_BOX}: Failed to load reference solution. Except:{e}")
+        return
+
+    if config.correctness_config["local-solution"] == None:
+        click.echo(
+            f"{ERROR_BOX}: No local-solution detected in configuration file.")
+        return
+    local_solution_path: str = config.correctness_config["local-solution"]
+    if config.correctness_config["signature"] == None:
+        click.echo(
+            f"{ERROR_BOX}: No signature detected in configuration file.")
+        return
+
+    if student and student not in students:
+        click.echo(
+            f"{ERROR_BOX}: No student {student} in {os.path.relpath(path)}.")
+        return
+
+    if student:
+        students = [student]
     asg_dirs = [os.path.join(
         path, student, f"assignment_{number}") for student in students]
 
     for asg_dir, student in zip(asg_dirs, students):
-
-        current_wd = os.getcwd()
-
         if not os.path.exists(asg_dir) or not os.path.isdir(asg_dir):
             click.echo(
-                f"{WARNING_BOX}: No roots could be identified for {student}/assignment_{number}")
+                f"{WARNING_BOX}: No {student}/assignment_{number} was found, skipping...")
             continue
         local_roots = [os.path.join(asg_dir, dir)
                        for dir in os.listdir(asg_dir) if 'root_' in dir and os.path.isdir(os.path.join(asg_dir, dir))]
@@ -34,123 +143,94 @@ def correctness(number, solution_path, path, compile_command, run_command, nproc
             click.echo(
                 f"{WARNING_BOX}: No roots could be identified for {student}/assignment_{number}")
             continue
+        click.echo(f"[{student}]:")
 
-        if verbose:
+        command_prefix = f'{run_command} {procopt}{numproc} {executable_prefix}'
-            click.echo(f"Testing compile for {student}:assignment_{number}")
-        else:
-            click.echo(f"({student}:assignment_{number}):")
 
         for root in local_roots:
-            os.chdir(current_wd)
-            if verbose:
-                click.echo(f"\t[{os.path.basename(root)}]: ", nl=False)
-                click.echo(f"entering {os.path.relpath(root)}")
             os.chdir(root)
-            result = subprocess.run(
+            executables = [file for file in os.listdir(
-                shlex.split("make distclean"),  capture_output=True, text=True)
+                os.getcwd()) if os.path.isfile(file) and os.access(file, os.X_OK)]
-
+            for executable in executables:
-            result = subprocess.run(
+                os.remove(executable)
-                shlex.split(compile_command),  capture_output=True, text=True)
-
-            if verbose:
-                if (result.returncode != 0):
-                    click.echo(
-                        f"\t\033[0;91m[{os.path.basename(root)}]\033[0m: ", nl=False)
-                else:
-                    click.echo(
-                        f"\t\033[0;92m[{os.path.basename(root)}]\033[0m: ", nl=False)
-                click.echo(f"stdout: {result.stdout}")
-                click.echo(f"stderr: {result.stderr}")
-            else:
-                if (result.returncode != 0):
-                    click.echo(
-                        f"\t\033[0;91m[{os.path.basename(root)}]\033[0m: ", nl=False)
-                    click.echo(f"stderr: {result.stderr[:69]:69s}...")
-
-                else:
-                    click.echo(
-                        f"\t\033[0;92m[{os.path.basename(root)}]\033[0m: SUCCESS")
-
-            executables = [os.path.join(root, file)
-                           for file in os.listdir(root) if 'exe-' in file and os.path.isfile(os.path.join(root, file))]
-
-            sol_as_path = os.path.join(solution_path, f"assignment_{number}")
-            config_path = os.path.join(sol_as_path, "test_config.toml")
-            if not os.path.isdir(sol_as_path):
-                click.echo(
-                    f"{ERROR_BOX}: No reference solution for assignment_{number} found.", err=True)
-                return
-            if not os.path.exists(config_path) or not os.path.isfile(config_path):
-                click.echo(
-                    f"{ERROR_BOX}: No test configuration for assignment_{number} found.", err=True)
-                return
-
-            config = dict()
-            with open(config_path, 'rb') as handle:
-                config = tomllib.load(handle)
-
-            if not 'ref_solution' in config["correctness"]:
-                click.echo(
-                    f"{ERROR_BOX}: No function dissect config found for assignment_{number}.", err=True)
-                return
-
-            ref_solution = config['correctness']['ref_solution']
-            ref_solution_path = os.path.join(sol_as_path, ref_solution)
-
-            if not 'ref_param' in config["correctness"]:
-                click.echo(
-                    f"{ERROR_BOX}: No function dissect config found for assignment_{number}.", err=True)
-                return
-            ref_param = config['correctness']['ref_param']
-            ref_param_path = os.path.join(sol_as_path, ref_param)
-            local_run_command = " ".join([run_command, nproc_flag,
-                                          executables[0], ref_param_path])
-            local_solution_path = os.path.join(root, 'p.dat')
 
             if os.path.exists(local_solution_path):
                 os.remove(local_solution_path)
 
+            clean_log = clean_root(root, clean_command)
+            if not clean_log.run_success or not clean_log.cmd_return_code == 0:
+                click.echo(
+                    f"{indent_text('[CLEAN]:' + clean_log.oneline(type_as_prefix=False), 4)}")
+                continue
+
+            build_log = build_root(root, compile_command,
+                                   timeout=compile_timeout)
+            if not build_log.run_success or not build_log.cmd_return_code == 0:
+                click.echo(
+                    f"{indent_text('[BUILD]:' + build_log.oneline(type_as_prefix=False), 4)}")
+                continue
+            executables = [file for file in os.listdir(
+                os.getcwd()) if os.path.isfile(file) and os.access(file, os.X_OK)]
+
+            local_command = command_prefix + executables[0] + ' '
+            rn_log = run_root(root, local_command +
+                              config.correctness_config['signature'], timeout=run_timeout)
+            if not rn_log.run_success or not rn_log.cmd_return_code == 0:
+                click.echo(
+                    f"{indent_text('[RUN]:' + rn_log.oneline(type_as_prefix=False), 4)}")
+                continue
+
+            run_success = True
+            summary = ''
+            cmd_return_code = 0
+            cmd_stderr = ''
+
+            if run_success and not os.path.exists(local_solution_path):
+                run_success = False
+                summary = f'Output file {local_solution_path} was not produced by run'
+                cmd_return_code = 1
+
+            local_solution = np.empty((0))
+            if run_success:
                 try:
-                result = subprocess.run(
+                    local_solution = np.loadtxt(local_solution_path)
-                    shlex.split(local_run_command),  capture_output=True, text=True, timeout=10)
+                except Exception as e:
-            except subprocess.TimeoutExpired:
+                    run_success = False
-                click.echo("Run Timeout")
+                    summary = f'Failed to load output file {local_solution_path}. Except: {e}'
-                result.returncode = 1
+                    cmd_return_code = 1
 
-            if verbose:
+            if run_success:
-                if (result.returncode != 0):
+                if cmd_return_code == 0 and reference_solution.shape != local_solution.shape:
-                    click.echo(
+                    cmd_return_code = 1
-                        f"\t\033[0;91m[{os.path.basename(root)}]\033[0m: ", nl=False)
+                    cmd_stderr = f'Correctness comparison failed due to missmatching data shapes. ref: {reference_solution.shape}, student: {local_solution.shape}'
-                    continue
+                if cmd_return_code == 0 and not np.allclose(local_solution, reference_solution, atol=tol):
-                else:
+                    cmd_return_code = 1
-                    click.echo(
+                    cmd_stderr = f'Correctness comparison failed due to missmatching data. Diff max|ref - student| = { np.max(np.abs(reference_solution- local_solution))}'
-                        f"\t\033[0;92m[{os.path.basename(root)}]\033[0m: ", nl=False)
-                click.echo(f"stdout: {result.stdout}")
-                click.echo(f"stderr: {result.stderr}")
-            else:
-                if (result.returncode != 0):
-                    click.echo(
-                        f"\t\033[0;91m[{os.path.basename(root)}]\033[0m: ", nl=False)
-                    click.echo(f"stderr: {result.stderr[:69]:69s}...")
-                    continue
-                else:
-                    click.echo(
-                        f"\t\033[0;92m[{os.path.basename(root)}]\033[0m: SUCCESS")
 
-            if not os.path.exists(local_solution_path):
+            log = run_log(
-                data_files = [file for file in os.listdir(
+                'CORRECTNESS',
-                    root) if '.dat' in file]
+                f"PHONEY: compare {config.correctness_config['ref-solution']}{os.path.abspath(local_solution_path)}",
-                print(data_files)
+                os.getcwd(),
-                continue
+                run_success,
+                summary,
+                cmd_return_code,
+                '',
+                cmd_stderr)
 
-            local_solution_data = np.loadtxt(local_solution_path, usecols=True)
+            if not verbose:
-            ref_solution_data = np.loadtxt(ref_solution_path, usecols=True)
+                click.echo(
-
+                    f"{indent_text(log.oneline(type_as_prefix=False), 4)}")
-            if local_solution_data.shape != ref_solution_data.shape:
-                click.echo("Correctness Test Failed, Dimenstions Not Matching")
-                continue
-            elif not np.allclose(local_solution_data, ref_solution_data, rtol=1e-1):
-                click.echo("Correctness Test Failed: Data Not Matching")
-                continue
             else:
-                click.echo("Correctness Test Passed")
+                click.echo(f"{indent_text(log.as_str(),4)}")
+
+            if os.path.exists(local_solution_path):
+                os.remove(local_solution_path)
+
+            for executable in executables:
+                os.remove(executable)
+            clean_log = clean_root(root, clean_command)
+            if not clean_log.run_success or not clean_log.cmd_return_code == 0:
+                click.echo(
+                    f"{indent_text('[CLEAN]:' + clean_log.oneline(type_as_prefix=False), 4)}")
+                continue
+        os.chdir(current_wd)

src/auto_grader/commands/test/dissect.py

@@ -112,7 +112,8 @@ def _extract_prototypes_to_header(filename, functions, header_filename):
 @click.option('-s', '--solution-path', default='./solutions', type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True))
 @click.option('-p', '--path', default='./roots', type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True))
 def dissect(number, solution_path, path):
-
+    print('Under Construction')
+    return
     sol_as_path = os.path.join(solution_path, f"assignment_{number}")
     config_path = os.path.join(sol_as_path, "test_config.toml")
 
@@ -257,6 +258,6 @@ def dissect(number, solution_path, path):
                             f"\t[{os.path.basename(root)}]:[{function}]:[CORRECT]:{SUCCESS_BOX}")
                     else:
                         click.echo(
-                            f"\t[{os.path.basename(root)}]:[{function}]:[CORRECT]:{ERROR_BOX}")
+                            f"\t[{os.path.basename(root)}]:[CORRECT]:{ERROR_BOX}")
                         continue
                 os.chdir(current_wd)

src/auto_grader/commands/test/run.py

@@ -1,111 +1,162 @@
 import click
+from click.core import ParameterSource
+import os
+
 from auto_grader.types import StudentVarType
+from auto_grader.utils.display import ERROR_BOX, WARNING_BOX, indent_text
+from auto_grader.utils import clean_root, build_root, run_root
+from auto_grader.utils.config_parser import grdr_config
 
 
 @click.command(help='Builds and Runs full student root. The run command is served as \n\n' +
-               '[COMMAND] [PROCOPT] [NUMPROC] [EXECUTABLE-PREFIX]<exe + exeflags autodetermined>\n\n' +
+               '[COMMAND] [PROCOPT] [NUMPROC] [EXECUTABLE-PREFIX]<exe + flags autodetermined by config>\n\n' +
                'Example with defaults:\n\n' +
-               'mpirun -n 4 exe <exe-opts>')
+               'mpirun -n 4 <exe + flags>')
 @click.argument('number', required=True, type=click.INT)
 @click.option('-p', '--path', default='./roots',
               type=click.Path(exists=True, file_okay=False,
                               dir_okay=True, resolve_path=True),
               help='Path to directory conaining student roots in format (student)/assignment_[NUMBER]/root_(...).')
-@click.option('--solution-path', default='./solutions',
-              type=click.Path(exists=True, file_okay=False,
-                              dir_okay=True, resolve_path=True),
-              help='Path to directory conaining solution root in format assignment_[NUMBER]/. Solution must contain configuration test_config.toml defining ')
-@click.option('-s', '--student', default='', type=StudentVarType(),
-              help='Specify wich student to run the test for. If omitted test is run for all found students.')
 @click.option('-c', '--command', default='mpirun', type=click.STRING,
               help='Overrides command that is used to run roots. The default command is `mpirun`.')
 @click.option('--procopt', default='-n ', type=click.STRING,
               help='Indicates the option flag to be used to specify process number from [COMMAND]. The default value is `-n `.')
 @click.option('-n', '--numproc', default=4, type=click.INT,
               help='Selects the number of mpi processes used to run the root. The default value is `4`.')
+@click.option('-t', '--timeout', default=10, type=click.INT,
+              help='Sets timeout for the run, defaults to 10s.')
+@click.option('--solutions-path', default='./solutions',
+              type=click.Path(exists=True, file_okay=False,
+                              dir_okay=True, resolve_path=True),
+              help='Path to directory conaining solution root in format assignment_[NUMBER]/. Solution must contain configuration grdr_config.toml')
+@click.option('-s', '--student', default='', type=StudentVarType(),
+              help='Specify wich student to run the test for. If omitted test is run for all found students.')
 @click.option('--executable-prefix', default='', type=click.STRING,
               help='This prefix is appended to the executable name when the run command is launched (some launchers require `./`. The default value is ``.')
 @click.option('--compile-command', default='make', type=click.STRING,
               help='Overrides command that is run to build roots. The default command is `make`.')
 @click.option('--clean-command', default='make distclean', type=click.STRING,
-              help='Overrides command that is run to build roots. The default command is `make distclean`.')
+              help='Overrides command that is run to clean roots. The default command is `make distclean`.')
-@click.option('-t', '--timeout', default=10, type=click.INT,
+@click.option('--compile-timeout', default=10, type=click.INT,
               help='Sets timeout of both clean and build commands, defaults to 10s.')
-@click.option('-e', '--exe-name', default='', type=click.STRING,
+@click.option('--verbose', is_flag=True, default=False,
-              help='Specify an executable name instead of determining it at runtime.')
+              help='Display the results of each attempted test as a detailed summary')
 @click.pass_context
 def run(
         context,
         number,
         path,
-        solution_path,
-        student,
         command,
         procopt,
         numproc,
-        executable_prefix,
         timeout,
+        student,
+        solutions_path,
+        executable_prefix,
         compile_command,
         clean_command,
-        exe_name
+        compile_timeout,
+        verbose
 ):
-    print("NOT IMPLEMENTED")
+    current_wd = os.getcwd()
-    pass
+    students = os.listdir(path)
-    # current_wd = os.getcwd()
+    config_path = solutions_path+f'/assignment_{number}/.grdr_config.toml'
-    # students = os.listdir(path)
+    config = grdr_config(config_path)
-    # if student and student not in students:
+
-    #     click.echo(
+    # --  Parse run options ---------------------------------------------------#
-    #         f"{ERROR_BOX}: No student {student} in {os.path.relpath(path)}.")
+    if context.get_parameter_source('command') != ParameterSource.COMMANDLINE and config.run_config['command'] != None:
-    #     return
+        command = config.run_config['command']
-    # if student:
+
-    #     students = [student]
+    if context.get_parameter_source('procopt') != ParameterSource.COMMANDLINE and config.run_config['procopt'] != None:
-    # asg_dirs = [os.path.join(
+        procopt = config.run_config['procopt']
-    #     path, student, f"assignment_{number}") for student in students]
+
-    #
+    if context.get_parameter_source('numproc') != ParameterSource.COMMANDLINE and config.run_config['numproc'] != None:
-    # for asg_dir, student in zip(asg_dirs, students):
+        numproc = config.run_config['numproc']
-    #
+
-    #     if not os.path.exists(asg_dir) or not os.path.isdir(asg_dir):
+    if context.get_parameter_source('timeout') != ParameterSource.COMMANDLINE and config.run_config['timeout'] != None:
-    #         click.echo(
+        timeout = config.run_config['timeout']
-    #             f"{WARNING_BOX}: No {student}/assignment_{number} was found, skipping...")
+
-    #         continue
+    if len(config.run_config['_signatures']) == 0:
-    #
+        click.echo(
-    #     local_roots = [os.path.join(asg_dir, dir)
+            f"{ERROR_BOX}: No signature detected in configuration file. A signature list, even if empty (signature = []) us required)")
-    #                    for dir in os.listdir(asg_dir) if 'root_' in dir and os.path.isdir(os.path.join(asg_dir, dir))]
+        return
-    #     if len(local_roots) == 0:
+
-    #         click.echo(
+    # --  Parse compile options  ----------------------------------------------#
-    #             f"{WARNING_BOX}: No roots could be identified for {student}/assignment_{number}")
+    if context.get_parameter_source('compile-command') != ParameterSource.COMMANDLINE and config.compile_config['command'] != None:
-    #         continue
+        compile_command = config.compile_config['command']
-    #     click.echo(f"[{student}]:")
+
-    #
+    if context.get_parameter_source('clean-command') != ParameterSource.COMMANDLINE and config.compile_config['clean-command'] != None:
-    #     for root in local_roots:
+        clean_command = config.compile_config['clean-command']
-    #         os.chdir(root)
+
-    #
+    if context.get_parameter_source('compile-timeout') != ParameterSource.COMMANDLINE and config.compile_config['timeout'] != None:
-    #         clean_log = clean_root(root, clean_command)
+        compile_timeout = config.compile_config['timeout']
-    #         if not clean_log.run_success or not clean_log.cmd_return_code == 0:
+
-    #             click.echo(
+    # -------------------------------------------------------------------------#
-    #                 f"{indent_text('[CLEAN]:' + clean_log.oneline(type_as_prefix=False), 4)}")
+
-    #             continue
+    if student and student not in students:
-    #
+        click.echo(
-    #         executables = [file for file in os.listdir(
+            f"{ERROR_BOX}: No student {student} in {os.path.relpath(path)}.")
-    #             root) if os.path.isfile(file) and file.startswith('exe-')]
+        return
-    #         for executable in executables:
+
-    #             os.remove(executable)
+    if student:
-    #
+        students = [student]
-    #         build_log = build_root(root, compile_command, timeout=timeout)
+    asg_dirs = [os.path.join(
-    #         if not build_log.run_success or not build_log.cmd_return_code == 0:
+        path, student, f"assignment_{number}") for student in students]
-    #             click.echo(f"{indent_text(build_log.oneline(), 4)}")
+    for asg_dir, student in zip(asg_dirs, students):
-    #             continue
+        if not os.path.exists(asg_dir) or not os.path.isdir(asg_dir):
-    #         executables = [file for file in os.listdir(
+            click.echo(
-    #             root) if os.path.isfile(file) and file.startswith('exe-')]
+                f"{WARNING_BOX}: No {student}/assignment_{number} was found, skipping...")
-    #
+            continue
-    #         if len(executables) == 0:
+        local_roots = [os.path.join(asg_dir, dir)
-    #             warn_string = f"[{os.path.basename(root)}]:{color_string('WARNING', 'br-yellow')} Build successfull but no exe-* was found, maybe non standard name? Skipping ..."
+                       for dir in os.listdir(asg_dir) if 'root_' in dir and os.path.isdir(os.path.join(asg_dir, dir))]
-    #             click.echo(f"{indent_text(warn_string, 4)}")
+        if len(local_roots) == 0:
-    #             continue
+            click.echo(
-    #
+                f"{WARNING_BOX}: No roots could be identified for {student}/assignment_{number}")
-    #         clean_log = clean_root(root, clean_command)
+            continue
-    #         if not clean_log.run_success or not clean_log.cmd_return_code == 0:
+        click.echo(f"[{student}]:")
-    #             click.echo(
+
-    #                 f"{indent_text('[CLEAN]:' + clean_log.oneline(type_as_prefix=False), 4)}")
+        command_prefix = f'{command} {procopt}{numproc} {executable_prefix}'
-    #             continue
+
-    #     os.chdir(current_wd)
+        for root in local_roots:
+            os.chdir(root)
+
+            executables = [file for file in os.listdir(
+                os.getcwd()) if os.path.isfile(file) and os.access(file, os.X_OK)]
+            for executable in executables:
+                os.remove(executable)
+
+            clean_log = clean_root(root, clean_command)
+            if not clean_log.run_success or not clean_log.cmd_return_code == 0:
+                click.echo(
+                    f"{indent_text('[CLEAN]:' + clean_log.oneline(type_as_prefix=False), 4)}")
+                continue
+
+            build_log = build_root(root, compile_command,
+                                   timeout=compile_timeout)
+            if not build_log.run_success or not build_log.cmd_return_code == 0:
+                click.echo(
+                    f"{indent_text('[BUILD]:' + build_log.oneline(type_as_prefix=False), 4)}")
+                continue
+
+            executables = [file for file in os.listdir(
+                os.getcwd()) if os.path.isfile(file) and os.access(file, os.X_OK)]
+            local_command = command_prefix + executables[0] + ' '
+            for signature in config.run_config['_signatures']:
+                root_postfix = ''
+                if len(config.run_config['_signatures']) > 1:
+                    root_postfix = f"[{signature.strip()}]"
+                log = run_root(root, local_command +
+                               signature, timeout=timeout)
+                if not verbose:
+                    click.echo(
+                        f"{indent_text(log.oneline(type_as_prefix=False,root_postfix=root_postfix), 4)}")
+                else:
+                    click.echo(f"{indent_text(log.as_str(),4)}")
+            for executable in executables:
+                os.remove(executable)
+            clean_log = clean_root(root, clean_command)
+            if not clean_log.run_success or not clean_log.cmd_return_code == 0:
+                click.echo(
+                    f"{indent_text('[CLEAN]:' + clean_log.oneline(type_as_prefix=False), 4)}")
+                continue
+        os.chdir(current_wd)

src/auto_grader/types/__init__.py

@@ -63,7 +63,7 @@ class run_log:
             final_string, indent=indent, indent_char=indent_char)
         return final_string
 
-    def oneline(self, type_as_prefix=True, color=True):
+    def oneline(self, type_as_prefix: bool = True, color: bool = True, root_postfix: str = ''):
         color_func = (
             (lambda s, c: color_string(s, c))
             if color
@@ -72,7 +72,7 @@ class run_log:
         oneline = ""
         if type_as_prefix:
             oneline += f"[{self.ttype}]:"
-        oneline += f"[{os.path.basename(self.run_abs_dir)}]: "
+        oneline += f"[{os.path.basename(self.run_abs_dir)}]{root_postfix}: "
         if self.run_success and self.cmd_return_code == 0:
             oneline += f"{color_func('SUCCESS', 'br-green')}"
         if not self.run_success:

src/auto_grader/utils/__init__.py

@@ -37,7 +37,7 @@ def run_root(root_path: str, run_cmd: str, timeout: int = 10):
         exception_summary = str(e)
 
     os.chdir(cwd)
-    log = run_log(test_type.build, test_type.run, os.path.abspath(root_path), run_result,
+    log = run_log(test_type.run, run_cmd, os.path.abspath(root_path), run_result,
                   exception_summary, result.returncode, result.stdout, result.stderr)
     return log
 

src/auto_grader/utils/config_parser.py

@@ -0,0 +1,81 @@
+import tomllib as tml
+import os
+from itertools import product
+
+
+class grdr_config():
+    def __init__(self, path: str):
+        self.config_path = path
+        self.compile_config = \
+            {
+                "command": None,
+                "clean-command": None,
+                "timeout": None
+            }
+
+        self.run_config = \
+            {
+                "command": None,
+                "procopt": None,
+                "numproc": None,
+                "timeout": None,
+                "signature": None,
+                "_signatures": [],
+            }
+
+        self.correctness_config = \
+            {
+                "ref-solution": None,
+                "local-solution": None,
+                "signature": None,
+            }
+
+        if os.path.exists(path) and os.path.isfile(path):
+            self.parse_config(path)
+
+    def parse_config(self, path: str):
+        raw_config = {}
+        try:
+            try:
+                with open(path, 'rb') as config_file:
+                    raw_config = tml.load(config_file)
+            except tml.TOMLDecodeError as e:
+                print(
+                    f"Failed parsing config at {os.path.abspath(path)}, trown: {e}")
+                return
+        except OSError as e:
+            print(
+                f"Unknow error opening config at {os.path.abspath(path)}, trown: {e}")
+            return
+        if 'compile' in raw_config:
+            for key in self.compile_config:
+                if key in raw_config['compile']:
+                    self.compile_config[key] = raw_config['compile'][key]
+
+        if 'run' in raw_config:
+            for key in self.run_config:
+                if key in raw_config['run'] and key != '_signatures':
+                    self.run_config[key] = raw_config['run'][key]
+
+        if self.run_config['signature'] != None:
+            if len(self.run_config['signature']) != 0:
+                found_ct = 0
+                total_product = []
+                # this sucks for performance if inputs get more complicated
+                for key in self.run_config['signature']:
+                    if key in raw_config['run']['params']:
+                        found_ct += 1
+                        val = raw_config['run']['params'][key]
+                        if type(val) != list:
+                            val = [val]
+                        total_product.append(val)
+                total_product = product(*total_product)
+                for sequence in total_product:
+                    self.run_config['_signatures'].append(' '.join(sequence))
+            else:
+                self.run_config['_signatures'].append('')
+
+        if 'correctness' in raw_config:
+            for key in self.correctness_config:
+                if key in raw_config['correctness']:
+                    self.correctness_config[key] = raw_config['correctness'][key]