Coverage for mlprodict/cli/asv_bench.py: 100%

1""" 

2@file 

3@brief Command line about validation of prediction runtime. 

4""" 

5import json 

6from logging import getLogger 

7from ..asv_benchmark import create_asv_benchmark 

8 

9 

10def asv_bench(location='asvsklonnx', opset_min=-1, opset_max=None, 

11 runtime='scikit-learn,python_compiled', models=None, 

12 skip_models=None, extended_list=True, 

13 dims='1,10,100,1000,10000', 

14 n_features='4,20', dtype=None, 

15 verbose=1, fLOG=print, clean=True, flat=False, 

16 conf_params=None, build=None, add_pyspy=False, 

17 env=None, matrix=None): 

18 """ 

19 Creates an :epkg:`asv` benchmark in a folder 

20 but does not run it. 

21 

22 :param location: location of the benchmark 

23 :param n_features: number of features to try 

24 :param dims: number of observations to try 

25 :param verbose: integer from 0 (None) to 2 (full verbose) 

26 :param opset_min: tries every conversion from this minimum opset, 

27 `-1` to get the current opset defined by module onnx 

28 :param opset_max: tries every conversion up to maximum opset, 

29 `-1` to get the current opset defined by module onnx 

30 :param runtime: runtime to check, *scikit-learn*, *python*, 

31 *python_compiled* compiles the graph structure 

32 and is more efficient when the number of observations is 

33 small, *onnxruntime1* to check :epkg:`onnxruntime`, 

34 *onnxruntime2* to check every ONNX node independently 

35 with onnxruntime, many runtime can be checked at the same time 

36 if the value is a comma separated list 

37 :param models: list of models to test or empty 

38 string to test them all 

39 :param skip_models: models to skip 

40 :param extended_list: extends the list of :epkg:`scikit-learn` converters 

41 with converters implemented in this module 

42 :param dtype: '32' or '64' or None for both, 

43 limits the test to one specific number types 

44 :param fLOG: logging function 

45 :param clean: clean the folder first, otherwise overwrites the content 

46 :param conf_params: to overwrite some of the configuration parameters, 

47 format ``name,value;name2,value2`` 

48 :param flat: one folder for all files or subfolders 

49 :param build: location of the outputs (env, html, results) 

50 :param add_pyspy: add an extra folder with code to profile 

51 each configuration 

52 :param env: default environment or ``same`` to use the current one 

53 :param matrix: specifies versions for a module as a json string, 

54 example: ``{'onnxruntime': ['1.1.1', '1.1.2']}``, 

55 if a package name starts with `'~'`, the package is removed 

56 :return: created files 

57 

58 .. cmdref:: 

59 :title: Automatically creates an asv benchmark 

60 :cmd: -m mlprodict asv_bench --help 

61 :lid: l-cmd-asv-bench 

62 

63 The command creates a benchmark based on asv module. 

64 It does not run it. 

65 

66 Example:: 

67 

68 python -m mlprodict asv_bench --models LogisticRegression,LinearRegression 

69 """ 

70 if not isinstance(models, list): 

71 models = (None if models in (None, "") 

72 else models.strip().split(',')) 

73 if not isinstance(skip_models, list): 

74 skip_models = ({} if skip_models in (None, "") 

75 else skip_models.strip().split(',')) 

76 if opset_max == "": 

77 opset_max = None # pragma: no cover 

78 if isinstance(opset_min, str): 

79 opset_min = int(opset_min) # pragma: no cover 

80 if isinstance(opset_max, str): 

81 opset_max = int(opset_max) # pragma: no cover 

82 if isinstance(verbose, str): 

83 verbose = int(verbose) # pragma: no cover 

84 if isinstance(extended_list, str): 

85 extended_list = extended_list in ( 

86 '1', 'True', 'true') # pragma: no cover 

87 if isinstance(add_pyspy, str): 

88 add_pyspy = add_pyspy in ('1', 'True', 'true') # pragma: no cover 

89 if not isinstance(runtime, list): 

90 runtime = runtime.split(',') 

91 if not isinstance(dims, list): 

92 dims = [int(_) for _ in dims.split(',')] 

93 if matrix is not None: 

94 if matrix in ('None', ''): 

95 matrix = None # pragma: no cover 

96 elif not isinstance(matrix, dict): 

97 matrix = json.loads(matrix) 

98 if not isinstance(n_features, list): 

99 if n_features in (None, ""): 

100 n_features = None # pragma: no cover 

101 else: 

102 n_features = list(map(int, n_features.split(','))) 

103 flat = flat in (True, 'True', 1, '1') 

104 

105 def fct_filter_exp(m, s): 

106 return str(m) not in skip_models 

107 

108 if dtype in ('', None): 

109 fct_filter = fct_filter_exp 

110 elif dtype == '32': # pragma: no cover 

111 def fct_filter_exp2(m, p): 

112 return fct_filter_exp(m, p) and '64' not in p 

113 fct_filter = fct_filter_exp2 

114 elif dtype == '64': 

115 def fct_filter_exp3(m, p): 

116 return fct_filter_exp(m, p) and '64' in p 

117 fct_filter = fct_filter_exp3 

118 else: 

119 raise ValueError( # pragma: no cover 

120 "dtype must be empty, 32, 64 not '{}'.".format(dtype)) 

121 

122 if conf_params is not None: 

123 res = {} 

124 kvs = conf_params.split(';') 

125 for kv in kvs: 

126 spl = kv.split(',') 

127 if len(spl) != 2: 

128 raise ValueError( # pragma: no cover 

129 "Unable to interpret '{}'.".format(kv)) 

130 k, v = spl 

131 res[k] = v 

132 conf_params = res 

133 

134 if verbose <= 1: 

135 logger = getLogger('skl2onnx') 

136 logger.disabled = True 

137 

138 return create_asv_benchmark( 

139 location=location, opset_min=opset_min, opset_max=opset_max, 

140 runtime=runtime, models=models, skip_models=skip_models, 

141 extended_list=extended_list, dims=dims, 

142 n_features=n_features, dtype=dtype, verbose=verbose, 

143 fLOG=fLOG, clean=clean, conf_params=conf_params, 

144 filter_exp=fct_filter, filter_scenario=None, 

145 flat=flat, build=build, add_pyspy=add_pyspy, 

146 env=env, matrix=matrix)