import click from click.core import ParameterSource import os import numpy as np from auto_grader.utils import clean_root, build_root, run_root from auto_grader.utils.display import ERROR_BOX, WARNING_BOX, indent_text from auto_grader.utils.config_parser import grdr_config from auto_grader.types import StudentVarType, run_log @click.command(help='Builds and Runs full student root. Then compares file if in numpy txt matrix format.\n' + 'Command run is: \n\n' + '[COMMAND] [PROCOPT] [NUMPROC] [EXECUTABLE-PREFIX]\n\n' + 'Example with defaults:\n\n' + 'mpirun -n 4 ') @click.argument('number', required=True, type=click.INT) @click.option('-p', '--path', default='./roots', type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), help='Path to directory conaining student roots in format (student)/assignment_[NUMBER]/root_(...).') @click.option('--tol', default=1e-1, type=click.FLOAT, help='Tollerance passed to numpy.allclose for result comparison') @click.option('-c', '--run-command', default='mpirun', type=click.STRING, help='Overrides command that is used to run roots. The default command is `mpirun`.') @click.option('--procopt', default='-n ', type=click.STRING, help='Indicates the option flag to be used to specify process number from [COMMAND]. The default value is `-n `.') @click.option('-n', '--numproc', default=4, type=click.INT, help='Selects the number of mpi processes used to run the root. The default value is `4`.') @click.option('-t', '--run-timeout', default=10, type=click.INT, help='Sets timeout for the run, defaults to 10s.') @click.option('--solutions-path', default='./solutions', type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), help='Path to directory conaining solution root in format assignment_[NUMBER]/. Solution must contain configuration grdr_config.toml') @click.option('-s', '--student', default='', type=StudentVarType(), help='Specify wich student to run the test for. If omitted test is run for all found students.') @click.option('--executable-prefix', default='', type=click.STRING, help='This prefix is appended to the executable name when the run command is launched (some launchers require `./`. The default value is ``.') @click.option('--compile-command', default='make', type=click.STRING, help='Overrides command that is run to build roots. The default command is `make`.') @click.option('--clean-command', default='make distclean', type=click.STRING, help='Overrides command that is run to clean roots. The default command is `make distclean`.') @click.option('--compile-timeout', default=10, type=click.INT, help='Sets timeout of both clean and build commands, defaults to 10s.') @click.option('--verbose', is_flag=True, default=False, help='Display the results of each attempted test as a detailed summary') @click.pass_context def correctness( context, number, path, tol, run_command, procopt, numproc, run_timeout, student, solutions_path, executable_prefix, compile_command, clean_command, compile_timeout, verbose ): current_wd = os.getcwd() students = os.listdir(path) config_path = solutions_path+f'/assignment_{number}/.grdr_config.toml' config = grdr_config(config_path) # -- Parse run options ---------------------------------------------------# if context.get_parameter_source('run-command') != ParameterSource.COMMANDLINE and config.run_config['command'] != None: run_command = config.run_config['command'] if context.get_parameter_source('procopt') != ParameterSource.COMMANDLINE and config.run_config['procopt'] != None: procopt = config.run_config['procopt'] if context.get_parameter_source('numproc') != ParameterSource.COMMANDLINE and config.run_config['numproc'] != None: numproc = config.run_config['numproc'] if context.get_parameter_source('run-timeout') != ParameterSource.COMMANDLINE and config.run_config['timeout'] != None: run_timeout = config.run_config['timeout'] if len(config.run_config['_signatures']) == 0: click.echo( f"{ERROR_BOX}: No signature detected in configuration file. A signature list, even if empty (signature = []) us required)") # -- Parse compile options ----------------------------------------------# if context.get_parameter_source('compile-command') != ParameterSource.COMMANDLINE and config.compile_config['command'] != None: compile_command = config.compile_config['command'] if context.get_parameter_source('clean-command') != ParameterSource.COMMANDLINE and config.compile_config['clean-command'] != None: clean_command = config.compile_config['clean-command'] if context.get_parameter_source('compile-timeout') != ParameterSource.COMMANDLINE and config.compile_config['timeout'] != None: compile_timeout = config.compile_config['timeout'] # -------------------------------------------------------------------------# if config.correctness_config["ref-solution"] == None: click.echo( f"{ERROR_BOX}: No ref-solution detected in configuration file.") return reference_solution = np.empty((0)) try: reference_solution = np.loadtxt( config.correctness_config["ref-solution"]) except Exception as e: click.echo( f"{ERROR_BOX}: Failed to load reference solution. Except:{e}") return if config.correctness_config["local-solution"] == None: click.echo( f"{ERROR_BOX}: No local-solution detected in configuration file.") return local_solution_path: str = config.correctness_config["local-solution"] if config.correctness_config["signature"] == None: click.echo( f"{ERROR_BOX}: No signature detected in configuration file.") return if student and student not in students: click.echo( f"{ERROR_BOX}: No student {student} in {os.path.relpath(path)}.") return if student: students = [student] asg_dirs = [os.path.join( path, student, f"assignment_{number}") for student in students] for asg_dir, student in zip(asg_dirs, students): if not os.path.exists(asg_dir) or not os.path.isdir(asg_dir): click.echo( f"{WARNING_BOX}: No {student}/assignment_{number} was found, skipping...") continue local_roots = [os.path.join(asg_dir, dir) for dir in os.listdir(asg_dir) if 'root_' in dir and os.path.isdir(os.path.join(asg_dir, dir))] if len(local_roots) == 0: click.echo( f"{WARNING_BOX}: No roots could be identified for {student}/assignment_{number}") continue click.echo(f"[{student}]:") command_prefix = f'{run_command} {procopt}{numproc} {executable_prefix}' for root in local_roots: os.chdir(root) executables = [file for file in os.listdir( os.getcwd()) if os.path.isfile(file) and os.access(file, os.X_OK)] for executable in executables: os.remove(executable) if os.path.exists(local_solution_path): os.remove(local_solution_path) clean_log = clean_root(root, clean_command) if not clean_log.run_success or not clean_log.cmd_return_code == 0: click.echo( f"{indent_text('[CLEAN]:' + clean_log.oneline(type_as_prefix=False), 4)}") continue build_log = build_root(root, compile_command, timeout=compile_timeout) if not build_log.run_success or not build_log.cmd_return_code == 0: click.echo( f"{indent_text('[BUILD]:' + build_log.oneline(type_as_prefix=False), 4)}") continue executables = [file for file in os.listdir( os.getcwd()) if os.path.isfile(file) and os.access(file, os.X_OK)] local_command = command_prefix + executables[0] + ' ' rn_log = run_root(root, local_command + config.correctness_config['signature'], timeout=run_timeout) if not rn_log.run_success or not rn_log.cmd_return_code == 0: click.echo( f"{indent_text('[RUN]:' + rn_log.oneline(type_as_prefix=False), 4)}") continue run_success = True summary = '' cmd_return_code = 0 cmd_stderr = '' if run_success and not os.path.exists(local_solution_path): run_success = False summary = f'Output file {local_solution_path} was not produced by run' cmd_return_code = 1 local_solution = np.empty((0)) if run_success: try: local_solution = np.loadtxt(local_solution_path) except Exception as e: run_success = False summary = f'Failed to load output file {local_solution_path}. Except: {e}' cmd_return_code = 1 if run_success: if cmd_return_code == 0 and reference_solution.shape != local_solution.shape: cmd_return_code = 1 cmd_stderr = f'Correctness comparison failed due to missmatching data shapes. ref: {reference_solution.shape}, student: {local_solution.shape}' if cmd_return_code == 0 and not np.allclose(local_solution, reference_solution, atol=tol): cmd_return_code = 1 cmd_stderr = f'Correctness comparison failed due to missmatching data. Diff max|ref - student| = { np.max(np.abs(reference_solution- local_solution))}' log = run_log( 'CORRECTNESS', f"PHONEY: compare {config.correctness_config['ref-solution']}{os.path.abspath(local_solution_path)}", os.getcwd(), run_success, summary, cmd_return_code, '', cmd_stderr) if not verbose: click.echo( f"{indent_text(log.oneline(type_as_prefix=False), 4)}") else: click.echo(f"{indent_text(log.as_str(),4)}") if os.path.exists(local_solution_path): os.remove(local_solution_path) for executable in executables: os.remove(executable) clean_log = clean_root(root, clean_command) if not clean_log.run_success or not clean_log.cmd_return_code == 0: click.echo( f"{indent_text('[CLEAN]:' + clean_log.oneline(type_as_prefix=False), 4)}") continue os.chdir(current_wd)