Coverage for src/ensae_teaching_cs/ml/competitions.py: 60%

1""" 

2@file 

3@brief Compute metrics in for a competition 

4""" 

5import os 

6import sys 

7 

8 

9def main_codalab_wrapper(fct, metric_name, argv, truth_file="truth.txt", submission_file="answer.txt", output_file="scores.txt"): 

10 """ 

11 Adapts the template available at 

12 `evaluate.py <https://github.com/Tivix/competition-examples/blob/master/hello_world/competition/scoring_program/evaluate.py>`_ 

13 """ 

14 input_dir = argv[1] 

15 output_dir = argv[2] 

16 

17 submit_dir = os.path.join(input_dir, 'res') 

18 truth_dir = os.path.join(input_dir, 'ref') 

19 

20 if not os.path.isdir(submit_dir): 

21 raise FileNotFoundError(f"{submit_dir} doesn't exist") 

22 

23 if os.path.isdir(submit_dir) and os.path.isdir(truth_dir): 

24 if not os.path.exists(output_dir): 

25 os.makedirs(output_dir) 

26 

27 private_codalab_wrapper(fct, metric_name, 

28 fold1=truth_dir, f1=truth_file, 

29 fold2=submit_dir, f2=submission_file, 

30 output=os.path.join(output_dir, output_file)) 

31 else: 

32 raise FileNotFoundError( 

33 f"{submit_dir} or {truth_dir} is not a folder") 

34 

35 

36def private_codalab_wrapper(fct, metric_name, fold1, fold2, f1="answer.txt", f2="answer.txt", 

37 output="scores.txt", use_print=False): 

38 """ 

39 Wraps the function following the guidelines 

40 `User_Building a Scoring Program for a Competition 

41 <https://github.com/codalab/codalab-competitions/wiki/User_Building-a-Scoring-Program-for-a-Competition>`_. 

42 It replicates the example available at 

43 `competition-examples/hello_world <https://github.com/Tivix/competition-examples/tree/master/hello_world/competition>`_. 

44 

45 @param fct function to wrap 

46 @param metric_name metric name 

47 @param fold1 folder which contains the data for folder containing the truth 

48 @param fold2 folder which contains the data for folder containing the data 

49 @param f1 filename for the truth 

50 @param f2 filename for the produced answers 

51 @param output produces an output with the expected results 

52 @param use_print display intermediate results 

53 @return metric 

54 """ 

55 f1 = os.path.join(fold1, f1) 

56 f2 = os.path.join(fold2, f2) 

57 if not os.path.exists(f1): 

58 raise FileNotFoundError(f"unable to find '{f1}'") 

59 if not os.path.exists(f2): 

60 raise FileNotFoundError(f"unable to find '{f2}'") 

61 if f1 == f2: 

62 raise ValueError( 

63 f"answers and scores are the same file: '{f1}'") 

64 

65 with open(f1, "r") as f: 

66 lines = f.readlines() 

67 answers = [float(_) for _ in lines if _] 

68 if use_print: 

69 print("Reading answers:", f1, len(answers), "rows") 

70 print("First answers:", answers[:10]) 

71 

72 with open(f2, "r") as f: 

73 lines = f.readlines() 

74 scores = [float(_) for _ in lines if _] 

75 if use_print: 

76 print("Reading scores:", f1, len(scores), "rows") 

77 print("First scores:", scores[:10]) 

78 

79 metric = fct(answers, scores) 

80 res = f"{metric_name}:{metric}" 

81 if use_print: 

82 print("Results=", res) 

83 with open(output, "w") as f: 

84 f.write(res) 

85 if use_print: 

86 print("Wrote", res, "in", output) 

87 return metric 

88 

89 

90def AUC(answers, scores): 

91 """ 

92 Computes the `AUC <https://en.wikipedia.org/wiki/Area_under_the_curve_(pharmacokinetics)>`_. 

93 

94 @param answers expected answers 0 (false), 1 (true) 

95 @param scores score obtained for class 1 

96 @return number 

97 """ 

98 ab = list(zip(answers, scores)) 

99 plus = [s for a, s in ab if a == 1] 

100 moins = [s for a, s in ab if a != 1] 

101 auc = 0 

102 for p in plus: 

103 for m in moins: 

104 if p > m: 

105 auc += 2 

106 elif p == m: 

107 auc += 1 

108 den = len(plus) * len(moins) 

109 if den == 0: 

110 return 1.0 if len(moins) == 0 else 0.0 

111 return auc * 1.0 / (len(plus) * len(moins) * 2) 

112 

113 

114if __name__ == "__main__": 

115 if len(sys.argv) < 3: 

116 raise RuntimeError(f"bad arguments: {sys.argv}") 

117 main_codalab_wrapper(AUC, "AUC", sys.argv)