diff --git a/src/auto_grader/cli.py b/src/auto_grader/cli.py index c4f4eb8..d603fb9 100644 --- a/src/auto_grader/cli.py +++ b/src/auto_grader/cli.py @@ -8,6 +8,7 @@ import shlex import tomllib import re import difflib +import numpy as np from unidecode import unidecode SUCCESS_BOX = "[\033[0;92mSUCCESS\033[0m]" @@ -123,7 +124,7 @@ def _extract_prototypes_to_header(filename, functions, header_filename): matches = func_pattern.findall(code) if not matches: - print(f"No functions found in {filename}") + click.echo(f"{WARNING_BOX}:No definitions found for {functions}") return prototypes = [] @@ -407,6 +408,7 @@ def compile(number, path, command, verbose): @click.option('-s', '--solution-path', default='./solutions', type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True)) @click.option('-p', '--path', default='./roots', type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True)) def dissect(number, solution_path, path): + sol_as_path = os.path.join(solution_path, f"assignment_{number}") config_path = os.path.join(sol_as_path, "test_config.toml") @@ -422,40 +424,53 @@ def dissect(number, solution_path, path): with open(config_path, 'rb') as handle: config = tomllib.load(handle) + if not 'test_functions' in config["dissect"]: click.echo( f"{ERROR_BOX}: No function dissect config found for assignment_{number}.", err=True) return test_functions = config['dissect']['test_functions'] - if not 'all_functions' in config["dissect"]: - click.echo( - f"{ERROR_BOX}: No function dissect config found for assignment_{number}.", err=True) - return - all_functions = config['dissect']['all_functions'] + if not 'build_cmd' in config["dissect"]: click.echo( f"{ERROR_BOX}: No build config found for assignment_{number}.", err=True) return static_functions = config['dissect']['static_functions'] + if not 'build_cmd' in config["dissect"]: click.echo( f"{ERROR_BOX}: No build config found for assignment_{number}.", err=True) return build_cmd = config['dissect']['build_cmd'] + if not 'run_cmd' in config["dissect"]: click.echo( f"{ERROR_BOX}: No run config found for assignment_{number}.", err=True) return run_cmd = config['dissect']['run_cmd'] + if not 'solution_files' in config["dissect"]: click.echo( f"{ERROR_BOX}: No solution_files config found for assignment_{number}.", err=True) return solution_files = config['dissect']['solution_files'] + if not 'ref_solution' in config["correctness"]: + click.echo( + f"{ERROR_BOX}: No ref_solution config found for assignment_{number}.", err=True) + return + ref_solution = config['correctness']['ref_solution'] + + if not 'user_solution' in config["correctness"]: + click.echo( + f"{ERROR_BOX}: No user_solution config found for assignment_{number}.", err=True) + return + user_solution = config['correctness']['user_solution'] + students = os.listdir(path) asg_dirs = [os.path.join( path, student, f"assignment_{number}") for student in students] + for asg_dir, student in zip(asg_dirs, students): if not os.path.exists(asg_dir) or not os.path.isdir(asg_dir): click.echo( @@ -469,7 +484,7 @@ def dissect(number, solution_path, path): continue current_wd = os.getcwd() click.echo(f"({student}:assignment_{number}):") - failed = False + flags = [f"-DGRDR_TEST_{func.upper()}" for func in test_functions] for root in local_roots: root_src_paths = [os.path.join(root, 'src', file) @@ -485,8 +500,8 @@ def dissect(number, solution_path, path): _extract_prototypes_to_header( dest, static_functions, os.path.join(sol_as_path, 'src', 'aid.h')) - _unstaticize_file(dest, all_functions) - _weaken_file(dest, all_functions) + _unstaticize_file(dest, test_functions) + _weaken_file(dest, test_functions) _staticize_file(dest, static_functions) result = 0 @@ -497,8 +512,7 @@ def dissect(number, solution_path, path): result = subprocess.run( ['make', 'distclean'], capture_output=True, text=True) result = subprocess.run( - shlex.split(command), capture_output=True, text=True) - + shlex.split(command), capture_output=True, text=True, timeout=10) if result.returncode == 0: click.echo( f"\t[{os.path.basename(root)}]:[BUILD]:[{function}]:{SUCCESS_BOX}") @@ -506,6 +520,42 @@ def dissect(number, solution_path, path): click.echo( f"\t[{os.path.basename(root)}]:[BUILD]:[{function}]:{ERROR_BOX}") click.echo(f"\t\tstderr: {result.stderr[:61]:61s}...") + continue + + ref_sol_path = os.path.join(sol_as_path, ref_solution) + usr_sol_path = os.path.join(sol_as_path, user_solution) + + if os.path.exists(usr_sol_path): + os.remove(usr_sol_path) + + try: + result = subprocess.run( + shlex.split(run_cmd), capture_output=True, text=True, timeout=10) + except subprocess.TimeoutExpired: + result.returncode = 1 + click.echo( + f"\t[{os.path.basename(root)}]:[RUN]:[{function}]:{ERROR_BOX} -- Timeout") + + if result.returncode == 0: + click.echo( + f"\t[{os.path.basename(root)}]:[RUN]:[{function}]:{SUCCESS_BOX}") + else: + click.echo( + f"\t[{os.path.basename(root)}]:[RUN]:[{function}]:{ERROR_BOX}") + click.echo(f"\t\tstderr: {result.stderr[:61]:61s}...") + continue + + ref_sol_data = np.loadtxt(ref_sol_path) + usr_sol_data = np.loadtxt(usr_sol_path) + + if np.allclose(ref_sol_data, usr_sol_data) == 0: + click.echo( + f"\t[{os.path.basename(root)}]:[CORRECT]:[{function}]:{SUCCESS_BOX}") + else: + click.echo( + f"\t[{os.path.basename(root)}]:[CORRECT]:[{function}]:{ERROR_BOX}") + click.echo(f"\t\tstderr: {result.stderr[:61]:61s}...") + continue os.chdir(current_wd)