Coverage for hopwise/evaluator/register.py: 81%

1# @Time : 2021/6/23 

2# @Author : Zihan Lin 

3# @Email : zhlin@ruc.edu.cn 

4 

5# UPDATE 

6# @Time : 2021/8/29 

7# @Author : Zhichao Feng 

8# @email : fzcbupt@gmail.com 

9 

10"""hopwise.evaluator.register 

11################################################ 

12""" 

13 

14import inspect 

15import sys 

16 

17 

18def cluster_info(module_name): 

19 """Collect information of all metrics, including: 

20 

21 - ``metric_need``: Information needed to calculate this metric, the combination of ``rec.items, rec.topk, 

22 rec.meanrank, rec.score, data.num_items, data.num_users, data.count_items, data.count_users, data.label``. 

23 - ``metric_type``: Whether the scores required by metric are grouped by user, range in ``EvaluatorType.RANKING`` 

24 and ``EvaluatorType.VALUE``. 

25 - ``smaller``: Whether the smaller metric value represents better performance, 

26 range in ``True`` and ``False``, default to ``False``. 

27 

28 Note: 

29 For ``metric_type``: in current hopwise, all the "grouped-score" metrics are ranking-based and all the 

30 "non-grouped-score" metrics are value-based. To keep with our paper, we adopted the more formal terms: 

31 ``RANKING`` and ``VALUE``. 

32 

33 Args: 

34 module_name (str): the name of module ``hopwise.evaluator.metrics``. 

35 

36 Returns: 

37 dict: Three dictionaries containing the above information 

38 and a dictionary matching metric names to metric classes. 

39 """ # noqa: E501 

40 smaller_m = [] 

41 m_dict, m_info, m_types = {}, {}, {} 

42 metric_class = inspect.getmembers( 

43 sys.modules[module_name], 

44 lambda x: inspect.isclass(x) and x.__module__ == module_name, 

45 ) 

46 for name, metric_cls in metric_class: 

47 name = name.lower() # noqa: PLW2901 

48 m_dict[name] = metric_cls 

49 if hasattr(metric_cls, "metric_need"): 

50 m_info[name] = metric_cls.metric_need 

51 else: 

52 raise AttributeError(f"Metric '{name}' has no attribute [metric_need].") 

53 if hasattr(metric_cls, "metric_type"): 

54 m_types[name] = metric_cls.metric_type 

55 else: 

56 raise AttributeError(f"Metric '{name}' has no attribute [metric_type].") 

57 if metric_cls.smaller is True: 

58 smaller_m.append(name) 

59 return smaller_m, m_info, m_types, m_dict 

60 

61 

62metric_module_name = "hopwise.evaluator.metrics" 

63smaller_metrics, metric_information, metric_types, metrics_dict = cluster_info(metric_module_name) 

64 

65 

66class Register: 

67 """Register module load the registry according to the metrics in config. 

68 It is a member of DataCollector. 

69 The DataCollector collect the resource that need for Evaluator under the guidance of Register 

70 """ 

71 

72 def __init__(self, config): 

73 self.config = config 

74 self.metrics = [metric.lower() for metric in self.config["metrics"]] 

75 self._build_register() 

76 

77 def _build_register(self): 

78 for metric in self.metrics: 

79 metric_needs = metric_information[metric] 

80 for info in metric_needs: 

81 setattr(self, info, True) 

82 

83 def has_metric(self, metric: str): 

84 if metric.lower() in self.metrics: 

85 return True 

86 else: 

87 return False 

88 

89 def need(self, key: str): 

90 if hasattr(self, key): 

91 return getattr(self, key) 

92 return False 

93 

94 

95class Register_KG(Register): 

96 """Register module load the registry according to the metrics in config. 

97 It is a member of DataCollector. 

98 The DataCollector collect the resource that need for Evaluator under the guidance of Register 

99 """ 

100 

101 def __init__(self, config): 

102 super().__init__(config) 

103 self.metrics = [metric.lower() for metric in self.config["metrics_lp"]] 

104 self._build_register()