diff --git a/src/auto_grader/cli.py b/src/auto_grader/cli.py index 672172f..497ac00 100644 --- a/src/auto_grader/cli.py +++ b/src/auto_grader/cli.py @@ -2,6 +2,8 @@ import click import os import shutil import tomllib +import importlib.resources as resources + from auto_grader.utils.display import SUCCESS_BOX, WARNING_BOX from auto_grader.commands.test.run import run @@ -10,7 +12,7 @@ from auto_grader.commands.test.dissect import dissect from auto_grader.commands.test.correctness import correctness from auto_grader.commands.archives import list_assign, unpack -import importlib.resources as resources +from auto_grader.utils.config_parser import grdr_config def parse_config(config_path): @@ -44,7 +46,6 @@ def init(path): @click.option('-o', '--output-path', default='./roots', type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True)) @click.option('--yes', is_flag=True, default=False, help='Skip confirmation prompt.') def collect(number, path, output_path, yes): - students = os.listdir(path) src_paths = [ f"{path}/{student}/assignment_{number}/" for student in students] @@ -76,10 +77,10 @@ def collect(number, path, output_path, yes): handle.write(f"origin:\n\t{os.path.relpath(root)}\n") -@click.command() -@click.option('-p', '--path', default='', type=click.Path(exists=True, file_okay=True, dir_okay=True, resolve_path=True)) -def dev(path): - pass +# @click.command() +# @click.option('-p', '--path', default='./solutions/assignment_4/.grdr_config.toml', type=click.Path(exists=False, file_okay=True, dir_okay=True, resolve_path=True)) +# def dev(path): +# grdr_config(path) @click.group() @@ -98,7 +99,7 @@ def test(): cli.add_command(init) -cli.add_command(dev) +# cli.add_command(dev) cli.add_command(archives) cli.add_command(roots) cli.add_command(test) diff --git a/src/auto_grader/commands/test/compile.py b/src/auto_grader/commands/test/compile.py index 01ea727..bf5bd35 100644 --- a/src/auto_grader/commands/test/compile.py +++ b/src/auto_grader/commands/test/compile.py @@ -1,9 +1,11 @@ import click +from click.core import ParameterSource import os from auto_grader.types import StudentVarType from auto_grader.utils.display import indent_text, ERROR_BOX, WARNING_BOX from auto_grader.utils import build_root, clean_root +from auto_grader.utils.config_parser import grdr_config @click.command() @@ -12,6 +14,10 @@ from auto_grader.utils import build_root, clean_root type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), help='Path to directory conaining student roots in format (student)/assignment_[NUMBER]/root_(...).') +@click.option('--solutions-path', default='./solutions', + type=click.Path(exists=False, file_okay=False, + dir_okay=True, resolve_path=True), + help='Path to directory conaining reference solutions roots in format (solutions-dir)/assignment_[NUMBER]/ .') @click.option('-s', '--student', default='', type=StudentVarType(), help='Specify wich student to run the test for. If omitted test is run for all found students.') @click.option('-c', '--command', default='make', type=click.STRING, @@ -21,10 +27,27 @@ from auto_grader.utils import build_root, clean_root @click.option('-t', '--timeout', default=10, type=click.INT, help='Sets timeout of both clean and build commands, defaults to 10s.') @click.option('--verbose', is_flag=True, default=False, - help='Sets timeout of both clean and build commands, defaults to 10s.') -def compile(number, path, command, student, timeout, clean_command, verbose): + help='Display the results of each attempted test as a detailed summary') +@click.option('--config-path', default=None, + type=click.Path(exists=False, file_okay=True, + dir_okay=False, resolve_path=True), + help='Custom configuration path.' + 'By default searches in the corresponding solution directory for a .grdf_config.toml.' + 'If it points to a non existing path it will trated as an empty config.') +@click.pass_context +def compile(context: click.core.Context, number, path, solutions_path, command, student, timeout, clean_command, verbose, config_path): current_wd = os.getcwd() students = os.listdir(path) + if config_path == None: + config_path = solutions_path+f'/assignment_{number}/.grdr_config.toml' + config = grdr_config(config_path) + if context.get_parameter_source('command') != ParameterSource.COMMANDLINE and config.compile_config['command'] != None: + command = config.compile_config['command'] + if context.get_parameter_source('clean-command') != ParameterSource.COMMANDLINE and config.compile_config['clean-command'] != None: + clean_command = config.compile_config['clean-command'] + if context.get_parameter_source('timeout') != ParameterSource.COMMANDLINE and config.compile_config['timeout'] != None: + timeout = config.compile_config['timeout'] + if student and student not in students: click.echo( f"{ERROR_BOX}: No student {student} in {os.path.relpath(path)}.") diff --git a/src/auto_grader/commands/test/correctness.py b/src/auto_grader/commands/test/correctness.py index cce7aed..4bdcac8 100644 --- a/src/auto_grader/commands/test/correctness.py +++ b/src/auto_grader/commands/test/correctness.py @@ -1,32 +1,141 @@ import click +from click.core import ParameterSource + import os -import subprocess -import shlex -import tomllib import numpy as np -from auto_grader.utils.display import ERROR_BOX, WARNING_BOX + +from auto_grader.utils import clean_root, build_root, run_root +from auto_grader.utils.display import ERROR_BOX, WARNING_BOX, indent_text +from auto_grader.utils.config_parser import grdr_config +from auto_grader.types import StudentVarType, run_log -@click.command() +@click.command(help='Builds and Runs full student root. Then compares file if in numpy txt matrix format.\n' + + 'Command run is: \n\n' + + '[COMMAND] [PROCOPT] [NUMPROC] [EXECUTABLE-PREFIX]\n\n' + + 'Example with defaults:\n\n' + + 'mpirun -n 4 ') @click.argument('number', required=True, type=click.INT) -@click.option('-p', '--path', default='./roots', type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True)) -@click.option('-c', '--compile-command', default='make', type=click.STRING) -@click.option('-r', '--run-command', default='mpirun', type=click.STRING) -@click.option('-n', '--nproc-flag', default="-n 5", type=click.STRING) -@click.option('-v', '--verbose', is_flag=True, default=False) -@click.option('-s', '--solution-path', default='./solutions', type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True)) -def correctness(number, solution_path, path, compile_command, run_command, nproc_flag, verbose): +@click.option('-p', '--path', default='./roots', + type=click.Path(exists=True, file_okay=False, + dir_okay=True, resolve_path=True), + help='Path to directory conaining student roots in format (student)/assignment_[NUMBER]/root_(...).') +@click.option('--tol', default=1e-1, + type=click.FLOAT, + help='Tollerance passed to numpy.allclose for result comparison') +@click.option('-c', '--run-command', default='mpirun', type=click.STRING, + help='Overrides command that is used to run roots. The default command is `mpirun`.') +@click.option('--procopt', default='-n ', type=click.STRING, + help='Indicates the option flag to be used to specify process number from [COMMAND]. The default value is `-n `.') +@click.option('-n', '--numproc', default=4, type=click.INT, + help='Selects the number of mpi processes used to run the root. The default value is `4`.') +@click.option('-t', '--run-timeout', default=10, type=click.INT, + help='Sets timeout for the run, defaults to 10s.') +@click.option('--solutions-path', default='./solutions', + type=click.Path(exists=True, file_okay=False, + dir_okay=True, resolve_path=True), + help='Path to directory conaining solution root in format assignment_[NUMBER]/. Solution must contain configuration grdr_config.toml') +@click.option('-s', '--student', default='', type=StudentVarType(), + help='Specify wich student to run the test for. If omitted test is run for all found students.') +@click.option('--executable-prefix', default='', type=click.STRING, + help='This prefix is appended to the executable name when the run command is launched (some launchers require `./`. The default value is ``.') +@click.option('--compile-command', default='make', type=click.STRING, + help='Overrides command that is run to build roots. The default command is `make`.') +@click.option('--clean-command', default='make distclean', type=click.STRING, + help='Overrides command that is run to clean roots. The default command is `make distclean`.') +@click.option('--compile-timeout', default=10, type=click.INT, + help='Sets timeout of both clean and build commands, defaults to 10s.') +@click.option('--verbose', is_flag=True, default=False, + help='Display the results of each attempted test as a detailed summary') +@click.pass_context +def correctness( + context, + number, + path, + tol, + run_command, + procopt, + numproc, + run_timeout, + student, + solutions_path, + executable_prefix, + compile_command, + clean_command, + compile_timeout, + verbose +): + + current_wd = os.getcwd() students = os.listdir(path) + config_path = solutions_path+f'/assignment_{number}/.grdr_config.toml' + config = grdr_config(config_path) + + # -- Parse run options ---------------------------------------------------# + if context.get_parameter_source('run-command') != ParameterSource.COMMANDLINE and config.run_config['command'] != None: + run_command = config.run_config['command'] + + if context.get_parameter_source('procopt') != ParameterSource.COMMANDLINE and config.run_config['procopt'] != None: + procopt = config.run_config['procopt'] + + if context.get_parameter_source('numproc') != ParameterSource.COMMANDLINE and config.run_config['numproc'] != None: + numproc = config.run_config['numproc'] + + if context.get_parameter_source('run-timeout') != ParameterSource.COMMANDLINE and config.run_config['timeout'] != None: + run_timeout = config.run_config['timeout'] + + if len(config.run_config['_signatures']) == 0: + click.echo( + f"{ERROR_BOX}: No signature detected in configuration file. A signature list, even if empty (signature = []) us required)") + + # -- Parse compile options ----------------------------------------------# + if context.get_parameter_source('compile-command') != ParameterSource.COMMANDLINE and config.compile_config['command'] != None: + compile_command = config.compile_config['command'] + + if context.get_parameter_source('clean-command') != ParameterSource.COMMANDLINE and config.compile_config['clean-command'] != None: + clean_command = config.compile_config['clean-command'] + + if context.get_parameter_source('compile-timeout') != ParameterSource.COMMANDLINE and config.compile_config['timeout'] != None: + compile_timeout = config.compile_config['timeout'] + + # -------------------------------------------------------------------------# + if config.correctness_config["ref-solution"] == None: + click.echo( + f"{ERROR_BOX}: No ref-solution detected in configuration file.") + return + reference_solution = np.empty((0)) + try: + reference_solution = np.loadtxt( + config.correctness_config["ref-solution"]) + except Exception as e: + click.echo( + f"{ERROR_BOX}: Failed to load reference solution. Except:{e}") + return + + if config.correctness_config["local-solution"] == None: + click.echo( + f"{ERROR_BOX}: No local-solution detected in configuration file.") + return + local_solution_path: str = config.correctness_config["local-solution"] + if config.correctness_config["signature"] == None: + click.echo( + f"{ERROR_BOX}: No signature detected in configuration file.") + return + + if student and student not in students: + click.echo( + f"{ERROR_BOX}: No student {student} in {os.path.relpath(path)}.") + return + + if student: + students = [student] asg_dirs = [os.path.join( path, student, f"assignment_{number}") for student in students] for asg_dir, student in zip(asg_dirs, students): - - current_wd = os.getcwd() - if not os.path.exists(asg_dir) or not os.path.isdir(asg_dir): click.echo( - f"{WARNING_BOX}: No roots could be identified for {student}/assignment_{number}") + f"{WARNING_BOX}: No {student}/assignment_{number} was found, skipping...") continue local_roots = [os.path.join(asg_dir, dir) for dir in os.listdir(asg_dir) if 'root_' in dir and os.path.isdir(os.path.join(asg_dir, dir))] @@ -34,123 +143,94 @@ def correctness(number, solution_path, path, compile_command, run_command, nproc click.echo( f"{WARNING_BOX}: No roots could be identified for {student}/assignment_{number}") continue + click.echo(f"[{student}]:") - if verbose: - click.echo(f"Testing compile for {student}:assignment_{number}") - else: - click.echo(f"({student}:assignment_{number}):") + command_prefix = f'{run_command} {procopt}{numproc} {executable_prefix}' for root in local_roots: - os.chdir(current_wd) - if verbose: - click.echo(f"\t[{os.path.basename(root)}]: ", nl=False) - click.echo(f"entering {os.path.relpath(root)}") os.chdir(root) - result = subprocess.run( - shlex.split("make distclean"), capture_output=True, text=True) - - result = subprocess.run( - shlex.split(compile_command), capture_output=True, text=True) - - if verbose: - if (result.returncode != 0): - click.echo( - f"\t\033[0;91m[{os.path.basename(root)}]\033[0m: ", nl=False) - else: - click.echo( - f"\t\033[0;92m[{os.path.basename(root)}]\033[0m: ", nl=False) - click.echo(f"stdout: {result.stdout}") - click.echo(f"stderr: {result.stderr}") - else: - if (result.returncode != 0): - click.echo( - f"\t\033[0;91m[{os.path.basename(root)}]\033[0m: ", nl=False) - click.echo(f"stderr: {result.stderr[:69]:69s}...") - - else: - click.echo( - f"\t\033[0;92m[{os.path.basename(root)}]\033[0m: SUCCESS") - - executables = [os.path.join(root, file) - for file in os.listdir(root) if 'exe-' in file and os.path.isfile(os.path.join(root, file))] - - sol_as_path = os.path.join(solution_path, f"assignment_{number}") - config_path = os.path.join(sol_as_path, "test_config.toml") - if not os.path.isdir(sol_as_path): - click.echo( - f"{ERROR_BOX}: No reference solution for assignment_{number} found.", err=True) - return - if not os.path.exists(config_path) or not os.path.isfile(config_path): - click.echo( - f"{ERROR_BOX}: No test configuration for assignment_{number} found.", err=True) - return - - config = dict() - with open(config_path, 'rb') as handle: - config = tomllib.load(handle) - - if not 'ref_solution' in config["correctness"]: - click.echo( - f"{ERROR_BOX}: No function dissect config found for assignment_{number}.", err=True) - return - - ref_solution = config['correctness']['ref_solution'] - ref_solution_path = os.path.join(sol_as_path, ref_solution) - - if not 'ref_param' in config["correctness"]: - click.echo( - f"{ERROR_BOX}: No function dissect config found for assignment_{number}.", err=True) - return - ref_param = config['correctness']['ref_param'] - ref_param_path = os.path.join(sol_as_path, ref_param) - local_run_command = " ".join([run_command, nproc_flag, - executables[0], ref_param_path]) - local_solution_path = os.path.join(root, 'p.dat') + executables = [file for file in os.listdir( + os.getcwd()) if os.path.isfile(file) and os.access(file, os.X_OK)] + for executable in executables: + os.remove(executable) if os.path.exists(local_solution_path): os.remove(local_solution_path) - try: - result = subprocess.run( - shlex.split(local_run_command), capture_output=True, text=True, timeout=10) - except subprocess.TimeoutExpired: - click.echo("Run Timeout") - result.returncode = 1 + clean_log = clean_root(root, clean_command) + if not clean_log.run_success or not clean_log.cmd_return_code == 0: + click.echo( + f"{indent_text('[CLEAN]:' + clean_log.oneline(type_as_prefix=False), 4)}") + continue - if verbose: - if (result.returncode != 0): - click.echo( - f"\t\033[0;91m[{os.path.basename(root)}]\033[0m: ", nl=False) - continue - else: - click.echo( - f"\t\033[0;92m[{os.path.basename(root)}]\033[0m: ", nl=False) - click.echo(f"stdout: {result.stdout}") - click.echo(f"stderr: {result.stderr}") + build_log = build_root(root, compile_command, + timeout=compile_timeout) + if not build_log.run_success or not build_log.cmd_return_code == 0: + click.echo( + f"{indent_text('[BUILD]:' + build_log.oneline(type_as_prefix=False), 4)}") + continue + executables = [file for file in os.listdir( + os.getcwd()) if os.path.isfile(file) and os.access(file, os.X_OK)] + + local_command = command_prefix + executables[0] + ' ' + rn_log = run_root(root, local_command + + config.correctness_config['signature'], timeout=run_timeout) + if not rn_log.run_success or not rn_log.cmd_return_code == 0: + click.echo( + f"{indent_text('[RUN]:' + rn_log.oneline(type_as_prefix=False), 4)}") + continue + + run_success = True + summary = '' + cmd_return_code = 0 + cmd_stderr = '' + + if run_success and not os.path.exists(local_solution_path): + run_success = False + summary = f'Output file {local_solution_path} was not produced by run' + cmd_return_code = 1 + + local_solution = np.empty((0)) + if run_success: + try: + local_solution = np.loadtxt(local_solution_path) + except Exception as e: + run_success = False + summary = f'Failed to load output file {local_solution_path}. Except: {e}' + cmd_return_code = 1 + + if run_success: + if cmd_return_code == 0 and reference_solution.shape != local_solution.shape: + cmd_return_code = 1 + cmd_stderr = f'Correctness comparison failed due to missmatching data shapes. ref: {reference_solution.shape}, student: {local_solution.shape}' + if cmd_return_code == 0 and not np.allclose(local_solution, reference_solution, atol=tol): + cmd_return_code = 1 + cmd_stderr = f'Correctness comparison failed due to missmatching data. Diff max|ref - student| = { np.max(np.abs(reference_solution- local_solution))}' + + log = run_log( + 'CORRECTNESS', + f"PHONEY: compare {config.correctness_config['ref-solution']}{os.path.abspath(local_solution_path)}", + os.getcwd(), + run_success, + summary, + cmd_return_code, + '', + cmd_stderr) + + if not verbose: + click.echo( + f"{indent_text(log.oneline(type_as_prefix=False), 4)}") else: - if (result.returncode != 0): - click.echo( - f"\t\033[0;91m[{os.path.basename(root)}]\033[0m: ", nl=False) - click.echo(f"stderr: {result.stderr[:69]:69s}...") - continue - else: - click.echo( - f"\t\033[0;92m[{os.path.basename(root)}]\033[0m: SUCCESS") + click.echo(f"{indent_text(log.as_str(),4)}") - if not os.path.exists(local_solution_path): - data_files = [file for file in os.listdir( - root) if '.dat' in file] - print(data_files) - continue + if os.path.exists(local_solution_path): + os.remove(local_solution_path) - local_solution_data = np.loadtxt(local_solution_path, usecols=True) - ref_solution_data = np.loadtxt(ref_solution_path, usecols=True) - - if local_solution_data.shape != ref_solution_data.shape: - click.echo("Correctness Test Failed, Dimenstions Not Matching") + for executable in executables: + os.remove(executable) + clean_log = clean_root(root, clean_command) + if not clean_log.run_success or not clean_log.cmd_return_code == 0: + click.echo( + f"{indent_text('[CLEAN]:' + clean_log.oneline(type_as_prefix=False), 4)}") continue - elif not np.allclose(local_solution_data, ref_solution_data, rtol=1e-1): - click.echo("Correctness Test Failed: Data Not Matching") - continue - else: - click.echo("Correctness Test Passed") + os.chdir(current_wd) diff --git a/src/auto_grader/commands/test/dissect.py b/src/auto_grader/commands/test/dissect.py index f5b962f..894e9cb 100644 --- a/src/auto_grader/commands/test/dissect.py +++ b/src/auto_grader/commands/test/dissect.py @@ -112,7 +112,8 @@ def _extract_prototypes_to_header(filename, functions, header_filename): @click.option('-s', '--solution-path', default='./solutions', type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True)) @click.option('-p', '--path', default='./roots', type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True)) def dissect(number, solution_path, path): - + print('Under Construction') + return sol_as_path = os.path.join(solution_path, f"assignment_{number}") config_path = os.path.join(sol_as_path, "test_config.toml") @@ -257,6 +258,6 @@ def dissect(number, solution_path, path): f"\t[{os.path.basename(root)}]:[{function}]:[CORRECT]:{SUCCESS_BOX}") else: click.echo( - f"\t[{os.path.basename(root)}]:[{function}]:[CORRECT]:{ERROR_BOX}") + f"\t[{os.path.basename(root)}]:[CORRECT]:{ERROR_BOX}") continue os.chdir(current_wd) diff --git a/src/auto_grader/commands/test/run.py b/src/auto_grader/commands/test/run.py index 5cbaef0..a52158a 100644 --- a/src/auto_grader/commands/test/run.py +++ b/src/auto_grader/commands/test/run.py @@ -1,111 +1,162 @@ import click +from click.core import ParameterSource +import os + from auto_grader.types import StudentVarType +from auto_grader.utils.display import ERROR_BOX, WARNING_BOX, indent_text +from auto_grader.utils import clean_root, build_root, run_root +from auto_grader.utils.config_parser import grdr_config @click.command(help='Builds and Runs full student root. The run command is served as \n\n' + - '[COMMAND] [PROCOPT] [NUMPROC] [EXECUTABLE-PREFIX]\n\n' + + '[COMMAND] [PROCOPT] [NUMPROC] [EXECUTABLE-PREFIX]\n\n' + 'Example with defaults:\n\n' + - 'mpirun -n 4 exe ') + 'mpirun -n 4 ') @click.argument('number', required=True, type=click.INT) @click.option('-p', '--path', default='./roots', type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), help='Path to directory conaining student roots in format (student)/assignment_[NUMBER]/root_(...).') -@click.option('--solution-path', default='./solutions', - type=click.Path(exists=True, file_okay=False, - dir_okay=True, resolve_path=True), - help='Path to directory conaining solution root in format assignment_[NUMBER]/. Solution must contain configuration test_config.toml defining ') -@click.option('-s', '--student', default='', type=StudentVarType(), - help='Specify wich student to run the test for. If omitted test is run for all found students.') @click.option('-c', '--command', default='mpirun', type=click.STRING, help='Overrides command that is used to run roots. The default command is `mpirun`.') @click.option('--procopt', default='-n ', type=click.STRING, help='Indicates the option flag to be used to specify process number from [COMMAND]. The default value is `-n `.') @click.option('-n', '--numproc', default=4, type=click.INT, help='Selects the number of mpi processes used to run the root. The default value is `4`.') +@click.option('-t', '--timeout', default=10, type=click.INT, + help='Sets timeout for the run, defaults to 10s.') +@click.option('--solutions-path', default='./solutions', + type=click.Path(exists=True, file_okay=False, + dir_okay=True, resolve_path=True), + help='Path to directory conaining solution root in format assignment_[NUMBER]/. Solution must contain configuration grdr_config.toml') +@click.option('-s', '--student', default='', type=StudentVarType(), + help='Specify wich student to run the test for. If omitted test is run for all found students.') @click.option('--executable-prefix', default='', type=click.STRING, help='This prefix is appended to the executable name when the run command is launched (some launchers require `./`. The default value is ``.') @click.option('--compile-command', default='make', type=click.STRING, help='Overrides command that is run to build roots. The default command is `make`.') @click.option('--clean-command', default='make distclean', type=click.STRING, - help='Overrides command that is run to build roots. The default command is `make distclean`.') -@click.option('-t', '--timeout', default=10, type=click.INT, + help='Overrides command that is run to clean roots. The default command is `make distclean`.') +@click.option('--compile-timeout', default=10, type=click.INT, help='Sets timeout of both clean and build commands, defaults to 10s.') -@click.option('-e', '--exe-name', default='', type=click.STRING, - help='Specify an executable name instead of determining it at runtime.') +@click.option('--verbose', is_flag=True, default=False, + help='Display the results of each attempted test as a detailed summary') @click.pass_context def run( context, number, path, - solution_path, - student, command, procopt, numproc, - executable_prefix, timeout, + student, + solutions_path, + executable_prefix, compile_command, clean_command, - exe_name + compile_timeout, + verbose ): - print("NOT IMPLEMENTED") - pass - # current_wd = os.getcwd() - # students = os.listdir(path) - # if student and student not in students: - # click.echo( - # f"{ERROR_BOX}: No student {student} in {os.path.relpath(path)}.") - # return - # if student: - # students = [student] - # asg_dirs = [os.path.join( - # path, student, f"assignment_{number}") for student in students] - # - # for asg_dir, student in zip(asg_dirs, students): - # - # if not os.path.exists(asg_dir) or not os.path.isdir(asg_dir): - # click.echo( - # f"{WARNING_BOX}: No {student}/assignment_{number} was found, skipping...") - # continue - # - # local_roots = [os.path.join(asg_dir, dir) - # for dir in os.listdir(asg_dir) if 'root_' in dir and os.path.isdir(os.path.join(asg_dir, dir))] - # if len(local_roots) == 0: - # click.echo( - # f"{WARNING_BOX}: No roots could be identified for {student}/assignment_{number}") - # continue - # click.echo(f"[{student}]:") - # - # for root in local_roots: - # os.chdir(root) - # - # clean_log = clean_root(root, clean_command) - # if not clean_log.run_success or not clean_log.cmd_return_code == 0: - # click.echo( - # f"{indent_text('[CLEAN]:' + clean_log.oneline(type_as_prefix=False), 4)}") - # continue - # - # executables = [file for file in os.listdir( - # root) if os.path.isfile(file) and file.startswith('exe-')] - # for executable in executables: - # os.remove(executable) - # - # build_log = build_root(root, compile_command, timeout=timeout) - # if not build_log.run_success or not build_log.cmd_return_code == 0: - # click.echo(f"{indent_text(build_log.oneline(), 4)}") - # continue - # executables = [file for file in os.listdir( - # root) if os.path.isfile(file) and file.startswith('exe-')] - # - # if len(executables) == 0: - # warn_string = f"[{os.path.basename(root)}]:{color_string('WARNING', 'br-yellow')} Build successfull but no exe-* was found, maybe non standard name? Skipping ..." - # click.echo(f"{indent_text(warn_string, 4)}") - # continue - # - # clean_log = clean_root(root, clean_command) - # if not clean_log.run_success or not clean_log.cmd_return_code == 0: - # click.echo( - # f"{indent_text('[CLEAN]:' + clean_log.oneline(type_as_prefix=False), 4)}") - # continue - # os.chdir(current_wd) + current_wd = os.getcwd() + students = os.listdir(path) + config_path = solutions_path+f'/assignment_{number}/.grdr_config.toml' + config = grdr_config(config_path) + + # -- Parse run options ---------------------------------------------------# + if context.get_parameter_source('command') != ParameterSource.COMMANDLINE and config.run_config['command'] != None: + command = config.run_config['command'] + + if context.get_parameter_source('procopt') != ParameterSource.COMMANDLINE and config.run_config['procopt'] != None: + procopt = config.run_config['procopt'] + + if context.get_parameter_source('numproc') != ParameterSource.COMMANDLINE and config.run_config['numproc'] != None: + numproc = config.run_config['numproc'] + + if context.get_parameter_source('timeout') != ParameterSource.COMMANDLINE and config.run_config['timeout'] != None: + timeout = config.run_config['timeout'] + + if len(config.run_config['_signatures']) == 0: + click.echo( + f"{ERROR_BOX}: No signature detected in configuration file. A signature list, even if empty (signature = []) us required)") + return + + # -- Parse compile options ----------------------------------------------# + if context.get_parameter_source('compile-command') != ParameterSource.COMMANDLINE and config.compile_config['command'] != None: + compile_command = config.compile_config['command'] + + if context.get_parameter_source('clean-command') != ParameterSource.COMMANDLINE and config.compile_config['clean-command'] != None: + clean_command = config.compile_config['clean-command'] + + if context.get_parameter_source('compile-timeout') != ParameterSource.COMMANDLINE and config.compile_config['timeout'] != None: + compile_timeout = config.compile_config['timeout'] + + # -------------------------------------------------------------------------# + + if student and student not in students: + click.echo( + f"{ERROR_BOX}: No student {student} in {os.path.relpath(path)}.") + return + + if student: + students = [student] + asg_dirs = [os.path.join( + path, student, f"assignment_{number}") for student in students] + for asg_dir, student in zip(asg_dirs, students): + if not os.path.exists(asg_dir) or not os.path.isdir(asg_dir): + click.echo( + f"{WARNING_BOX}: No {student}/assignment_{number} was found, skipping...") + continue + local_roots = [os.path.join(asg_dir, dir) + for dir in os.listdir(asg_dir) if 'root_' in dir and os.path.isdir(os.path.join(asg_dir, dir))] + if len(local_roots) == 0: + click.echo( + f"{WARNING_BOX}: No roots could be identified for {student}/assignment_{number}") + continue + click.echo(f"[{student}]:") + + command_prefix = f'{command} {procopt}{numproc} {executable_prefix}' + + for root in local_roots: + os.chdir(root) + + executables = [file for file in os.listdir( + os.getcwd()) if os.path.isfile(file) and os.access(file, os.X_OK)] + for executable in executables: + os.remove(executable) + + clean_log = clean_root(root, clean_command) + if not clean_log.run_success or not clean_log.cmd_return_code == 0: + click.echo( + f"{indent_text('[CLEAN]:' + clean_log.oneline(type_as_prefix=False), 4)}") + continue + + build_log = build_root(root, compile_command, + timeout=compile_timeout) + if not build_log.run_success or not build_log.cmd_return_code == 0: + click.echo( + f"{indent_text('[BUILD]:' + build_log.oneline(type_as_prefix=False), 4)}") + continue + + executables = [file for file in os.listdir( + os.getcwd()) if os.path.isfile(file) and os.access(file, os.X_OK)] + local_command = command_prefix + executables[0] + ' ' + for signature in config.run_config['_signatures']: + root_postfix = '' + if len(config.run_config['_signatures']) > 1: + root_postfix = f"[{signature.strip()}]" + log = run_root(root, local_command + + signature, timeout=timeout) + if not verbose: + click.echo( + f"{indent_text(log.oneline(type_as_prefix=False,root_postfix=root_postfix), 4)}") + else: + click.echo(f"{indent_text(log.as_str(),4)}") + for executable in executables: + os.remove(executable) + clean_log = clean_root(root, clean_command) + if not clean_log.run_success or not clean_log.cmd_return_code == 0: + click.echo( + f"{indent_text('[CLEAN]:' + clean_log.oneline(type_as_prefix=False), 4)}") + continue + os.chdir(current_wd) diff --git a/src/auto_grader/types/__init__.py b/src/auto_grader/types/__init__.py index 619b121..5a36631 100644 --- a/src/auto_grader/types/__init__.py +++ b/src/auto_grader/types/__init__.py @@ -63,7 +63,7 @@ class run_log: final_string, indent=indent, indent_char=indent_char) return final_string - def oneline(self, type_as_prefix=True, color=True): + def oneline(self, type_as_prefix: bool = True, color: bool = True, root_postfix: str = ''): color_func = ( (lambda s, c: color_string(s, c)) if color @@ -72,7 +72,7 @@ class run_log: oneline = "" if type_as_prefix: oneline += f"[{self.ttype}]:" - oneline += f"[{os.path.basename(self.run_abs_dir)}]: " + oneline += f"[{os.path.basename(self.run_abs_dir)}]{root_postfix}: " if self.run_success and self.cmd_return_code == 0: oneline += f"{color_func('SUCCESS', 'br-green')}" if not self.run_success: diff --git a/src/auto_grader/utils/__init__.py b/src/auto_grader/utils/__init__.py index cd54546..f51acfd 100644 --- a/src/auto_grader/utils/__init__.py +++ b/src/auto_grader/utils/__init__.py @@ -37,7 +37,7 @@ def run_root(root_path: str, run_cmd: str, timeout: int = 10): exception_summary = str(e) os.chdir(cwd) - log = run_log(test_type.build, test_type.run, os.path.abspath(root_path), run_result, + log = run_log(test_type.run, run_cmd, os.path.abspath(root_path), run_result, exception_summary, result.returncode, result.stdout, result.stderr) return log diff --git a/src/auto_grader/utils/config_parser.py b/src/auto_grader/utils/config_parser.py new file mode 100644 index 0000000..f2f9b6e --- /dev/null +++ b/src/auto_grader/utils/config_parser.py @@ -0,0 +1,81 @@ +import tomllib as tml +import os +from itertools import product + + +class grdr_config(): + def __init__(self, path: str): + self.config_path = path + self.compile_config = \ + { + "command": None, + "clean-command": None, + "timeout": None + } + + self.run_config = \ + { + "command": None, + "procopt": None, + "numproc": None, + "timeout": None, + "signature": None, + "_signatures": [], + } + + self.correctness_config = \ + { + "ref-solution": None, + "local-solution": None, + "signature": None, + } + + if os.path.exists(path) and os.path.isfile(path): + self.parse_config(path) + + def parse_config(self, path: str): + raw_config = {} + try: + try: + with open(path, 'rb') as config_file: + raw_config = tml.load(config_file) + except tml.TOMLDecodeError as e: + print( + f"Failed parsing config at {os.path.abspath(path)}, trown: {e}") + return + except OSError as e: + print( + f"Unknow error opening config at {os.path.abspath(path)}, trown: {e}") + return + if 'compile' in raw_config: + for key in self.compile_config: + if key in raw_config['compile']: + self.compile_config[key] = raw_config['compile'][key] + + if 'run' in raw_config: + for key in self.run_config: + if key in raw_config['run'] and key != '_signatures': + self.run_config[key] = raw_config['run'][key] + + if self.run_config['signature'] != None: + if len(self.run_config['signature']) != 0: + found_ct = 0 + total_product = [] + # this sucks for performance if inputs get more complicated + for key in self.run_config['signature']: + if key in raw_config['run']['params']: + found_ct += 1 + val = raw_config['run']['params'][key] + if type(val) != list: + val = [val] + total_product.append(val) + total_product = product(*total_product) + for sequence in total_product: + self.run_config['_signatures'].append(' '.join(sequence)) + else: + self.run_config['_signatures'].append('') + + if 'correctness' in raw_config: + for key in self.correctness_config: + if key in raw_config['correctness']: + self.correctness_config[key] = raw_config['correctness'][key]