Coverage for mlprodict/onnxrt/validate/validate.py: 98%

1""" 

2@file 

3@brief Validates runtime for many :scikit-learn: operators. 

4The submodule relies on :epkg:`onnxconverter_common`, 

5:epkg:`sklearn-onnx`. 

6""" 

7import pprint 

8from inspect import signature 

9import numpy 

10from numpy.linalg import LinAlgError 

11import sklearn 

12from sklearn import __all__ as sklearn__all__, __version__ as sklearn_version 

13from sklearn.exceptions import ConvergenceWarning 

14from sklearn.utils._testing import ignore_warnings 

15from ... import ( 

16 __version__ as ort_version, 

17 __max_supported_opset__, get_ir_version, 

18 __max_supported_opsets__) 

19from ...onnx_conv import to_onnx, register_converters, register_rewritten_operators 

20from ...tools.model_info import analyze_model, set_random_state 

21from ..onnx_inference import OnnxInference 

22from ...onnx_tools.optim.sklearn_helper import inspect_sklearn_model, set_n_jobs 

23from ...onnx_tools.optim.onnx_helper import onnx_statistics 

24from ...onnx_tools.optim import onnx_optimisations 

25from .validate_problems import find_suitable_problem 

26from .validate_scenarios import _extra_parameters 

27from .validate_difference import measure_relative_difference 

28from .validate_helper import ( 

29 _dispsimple, sklearn_operators, 

30 _measure_time, _shape_exc, dump_into_folder, 

31 default_time_kwargs, RuntimeBadResultsError, 

32 _dictionary2str, _merge_options, _multiply_time_kwargs, 

33 _get_problem_data) 

34from .validate_benchmark import benchmark_fct 

35 

36 

37@ignore_warnings(category=(UserWarning, ConvergenceWarning)) 

38def _dofit_model(dofit, obs, inst, X_train, y_train, X_test, y_test, 

39 Xort_test, init_types, store_models, 

40 debug, verbose, fLOG): 

41 if dofit: 

42 if verbose >= 2 and fLOG is not None: 

43 fLOG("[enumerate_compatible_opset] fit, type: '{}' dtype: {}".format( 

44 type(X_train), getattr(X_train, 'dtype', '-'))) 

45 try: 

46 set_random_state(inst) 

47 if y_train is None: 

48 t4 = _measure_time(lambda: inst.fit(X_train))[1] 

49 else: 

50 t4 = _measure_time( 

51 lambda: inst.fit(X_train, y_train))[1] 

52 except (AttributeError, TypeError, ValueError, 

53 IndexError, NotImplementedError, MemoryError, 

54 LinAlgError, StopIteration) as e: 

55 if debug: 

56 raise # pragma: no cover 

57 obs["_1training_time_exc"] = str(e) 

58 return False 

59 

60 obs["training_time"] = t4 

61 try: 

62 skl_st = inspect_sklearn_model(inst) 

63 except NotImplementedError: 

64 skl_st = {} 

65 obs.update({'skl_' + k: v for k, v in skl_st.items()}) 

66 

67 if store_models: 

68 obs['MODEL'] = inst 

69 obs['X_test'] = X_test 

70 obs['Xort_test'] = Xort_test 

71 obs['init_types'] = init_types 

72 else: 

73 obs["training_time"] = 0. 

74 if store_models: 

75 obs['MODEL'] = inst 

76 obs['init_types'] = init_types 

77 

78 return True 

79 

80 

81def _run_skl_prediction(obs, check_runtime, assume_finite, inst, 

82 method_name, predict_kwargs, X_test, 

83 benchmark, debug, verbose, time_kwargs, 

84 skip_long_test, time_kwargs_fact, fLOG): 

85 if not check_runtime: 

86 return None # pragma: no cover 

87 if verbose >= 2 and fLOG is not None: 

88 fLOG("[enumerate_compatible_opset] check_runtime SKL {}-{}-{}-{}-{}".format( 

89 id(inst), method_name, predict_kwargs, time_kwargs, 

90 time_kwargs_fact)) 

91 with sklearn.config_context(assume_finite=assume_finite): 

92 # compute sklearn prediction 

93 obs['ort_version'] = ort_version 

94 try: 

95 meth = getattr(inst, method_name) 

96 except AttributeError as e: # pragma: no cover 

97 if debug: 

98 raise # pragma: no cover 

99 obs['_2skl_meth_exc'] = str(e) 

100 return e 

101 try: 

102 ypred, t4, ___ = _measure_time( 

103 lambda: meth(X_test, **predict_kwargs)) 

104 obs['lambda-skl'] = (lambda xo: meth(xo, **predict_kwargs), X_test) 

105 except (ValueError, AttributeError, # pragma: no cover 

106 TypeError, MemoryError, IndexError) as e: 

107 if debug: 

108 raise # pragma: no cover 

109 obs['_3prediction_exc'] = str(e) 

110 return e 

111 obs['prediction_time'] = t4 

112 obs['assume_finite'] = assume_finite 

113 if benchmark and 'lambda-skl' in obs: 

114 obs['bench-skl'] = benchmark_fct( 

115 *obs['lambda-skl'], obs=obs, 

116 time_kwargs=_multiply_time_kwargs( 

117 time_kwargs, time_kwargs_fact, inst), 

118 skip_long_test=skip_long_test) 

119 if verbose >= 3 and fLOG is not None: 

120 fLOG("[enumerate_compatible_opset] scikit-learn prediction") 

121 _dispsimple(ypred, fLOG) 

122 if verbose >= 2 and fLOG is not None: 

123 fLOG("[enumerate_compatible_opset] predictions stored") 

124 return ypred 

125 

126 

127def _retrieve_problems_extra(model, verbose, fLOG, extended_list): 

128 """ 

129 Use by @see fn enumerate_compatible_opset. 

130 """ 

131 extras = None 

132 if extended_list: 

133 from ...onnx_conv.validate_scenarios import find_suitable_problem as fsp_extended 

134 problems = fsp_extended(model) 

135 if problems is not None: 

136 from ...onnx_conv.validate_scenarios import build_custom_scenarios as fsp_scenarios 

137 extra_parameters = fsp_scenarios() 

138 

139 if verbose >= 2 and fLOG is not None: 

140 fLOG( 

141 "[enumerate_compatible_opset] found custom for model={}".format(model)) 

142 extras = extra_parameters.get(model, None) 

143 if extras is not None: 

144 fLOG( 

145 "[enumerate_compatible_opset] found custom scenarios={}".format(extras)) 

146 else: 

147 problems = None 

148 

149 if problems is None: 

150 # scikit-learn 

151 extra_parameters = _extra_parameters 

152 try: 

153 problems = find_suitable_problem(model) 

154 except RuntimeError as e: # pragma: no cover 

155 return {'name': model.__name__, 'skl_version': sklearn_version, 

156 '_0problem_exc': e}, extras 

157 extras = extra_parameters.get(model, [('default', {})]) 

158 

159 # checks existence of random_state 

160 sig = signature(model.__init__) 

161 if 'random_state' in sig.parameters: 

162 new_extras = [] 

163 for extra in extras: 

164 if 'random_state' not in extra[1]: 

165 ps = extra[1].copy() 

166 ps['random_state'] = 42 

167 if len(extra) == 2: 

168 extra = (extra[0], ps) 

169 else: 

170 extra = (extra[0], ps) + extra[2:] 

171 new_extras.append(extra) 

172 extras = new_extras 

173 

174 return problems, extras 

175 

176 

177def enumerate_compatible_opset(model, opset_min=-1, opset_max=-1, # pylint: disable=R0914 

178 check_runtime=True, debug=False, 

179 runtime='python', dump_folder=None, 

180 store_models=False, benchmark=False, 

181 assume_finite=True, node_time=False, 

182 fLOG=print, filter_exp=None, 

183 verbose=0, time_kwargs=None, 

184 extended_list=False, dump_all=False, 

185 n_features=None, skip_long_test=True, 

186 filter_scenario=None, time_kwargs_fact=None, 

187 time_limit=4, n_jobs=None): 

188 """ 

189 Lists all compatible opsets for a specific model. 

190 

191 @param model operator class 

192 @param opset_min starts with this opset 

193 @param opset_max ends with this opset (None to use 

194 current onnx opset) 

195 @param check_runtime checks that runtime can consume the 

196 model and compute predictions 

197 @param debug catch exception (True) or not (False) 

198 @param runtime test a specific runtime, by default ``'python'`` 

199 @param dump_folder dump information to replicate in case of mismatch 

200 @param dump_all dump all models not only the one which fail 

201 @param store_models if True, the function 

202 also stores the fitted model and its conversion 

203 into :epkg:`ONNX` 

204 @param benchmark if True, measures the time taken by each function 

205 to predict for different number of rows 

206 @param fLOG logging function 

207 @param filter_exp function which tells if the experiment must be run, 

208 None to run all, takes *model, problem* as an input 

209 @param filter_scenario second function which tells if the experiment must be run, 

210 None to run all, takes *model, problem, scenario, extra, options* 

211 as an input 

212 @param node_time collect time for each node in the :epkg:`ONNX` graph 

213 @param assume_finite See `config_context 

214 <https://scikit-learn.org/stable/modules/generated/ 

215 sklearn.config_context.html>`_, If True, validation for finiteness 

216 will be skipped, saving time, but leading to potential crashes. 

217 If False, validation for finiteness will be performed, avoiding error. 

218 @param verbose verbosity 

219 @param extended_list extends the list to custom converters 

220 and problems 

221 @param time_kwargs to define a more precise way to measure a model 

222 @param n_features modifies the shorts datasets used to train the models 

223 to use exactly this number of features, it can also 

224 be a list to test multiple datasets 

225 @param skip_long_test skips tests for high values of N if they seem too long 

226 @param time_kwargs_fact see :func:`_multiply_time_kwargs <mlprodict.onnxrt.validate.validate_helper._multiply_time_kwargs>` 

227 @param time_limit to stop benchmarking after this amount of time was spent 

228 @param n_jobs *n_jobs* is set to the number of CPU by default unless this 

229 value is changed 

230 @return dictionaries, each row has the following 

231 keys: opset, exception if any, conversion time, 

232 problem chosen to test the conversion... 

233 

234 The function requires :epkg:`sklearn-onnx`. 

235 The outcome can be seen at pages references 

236 by :ref:`l-onnx-availability`. 

237 The parameter *time_kwargs* is a dictionary which defines the 

238 number of times to repeat the same predictions in order 

239 to give more precise figures. The default value (if None) is returned 

240 by the following code: 

241 

242 .. runpython:: 

243 :showcode: 

244 :warningout: DeprecationWarning 

245 

246 from mlprodict.onnxrt.validate.validate_helper import default_time_kwargs 

247 import pprint 

248 pprint.pprint(default_time_kwargs()) 

249 

250 Parameter *time_kwargs_fact* multiples these values for some 

251 specific models. ``'lin'`` multiplies by 10 when the model 

252 is linear. 

253 """ 

254 if opset_min == -1: 

255 opset_min = __max_supported_opset__ # pragma: no cover 

256 if opset_max == -1: 

257 opset_max = __max_supported_opset__ # pragma: no cover 

258 if verbose > 0 and fLOG is not None: 

259 fLOG("[enumerate_compatible_opset] opset in [{}, {}].".format( 

260 opset_min, opset_max)) 

261 if verbose > 1 and fLOG: 

262 fLOG("[enumerate_compatible_opset] validate class '{}'.".format( 

263 model.__name__)) 

264 if verbose > 2: 

265 fLOG(model) 

266 

267 if time_kwargs is None: 

268 time_kwargs = default_time_kwargs() 

269 problems, extras = _retrieve_problems_extra( 

270 model, verbose, fLOG, extended_list) 

271 if isinstance(problems, dict): 

272 yield problems # pragma: no cover 

273 problems = [] # pragma: no cover 

274 

275 if opset_max is None: 

276 opset_max = __max_supported_opset__ # pragma: no cover 

277 opsets = list(range(opset_min, opset_max + 1)) # pragma: no cover 

278 opsets.append(None) # pragma: no cover 

279 else: 

280 opsets = list(range(opset_min, opset_max + 1)) 

281 

282 if extras is None: 

283 problems = [] 

284 yield {'name': model.__name__, 'skl_version': sklearn_version, 

285 '_0problem_exc': 'SKIPPED'} 

286 

287 if not isinstance(n_features, list): 

288 n_features = [n_features] 

289 

290 for prob in problems: 

291 if filter_exp is not None and not filter_exp(model, prob): 

292 continue 

293 for n_feature in n_features: 

294 if verbose >= 2 and fLOG is not None: 

295 fLOG("[enumerate_compatible_opset] problem={} n_feature={}".format( 

296 prob, n_feature)) 

297 

298 (X_train, X_test, y_train, 

299 y_test, Xort_test, 

300 init_types, conv_options, method_name, 

301 output_index, dofit, predict_kwargs) = _get_problem_data(prob, n_feature) 

302 

303 for scenario_extra in extras: 

304 subset_problems = None 

305 optimisations = None 

306 new_conv_options = None 

307 if len(scenario_extra) > 2: 

308 options = scenario_extra[2] 

309 if isinstance(options, dict): 

310 subset_problems = options.get('subset_problems', None) 

311 optimisations = options.get('optim', None) 

312 new_conv_options = options.get('conv_options', None) 

313 else: 

314 subset_problems = options 

315 

316 if subset_problems and isinstance(subset_problems, (list, set)): 

317 if prob not in subset_problems: 

318 # Skips unrelated problem for a specific configuration. 

319 continue 

320 elif subset_problems is not None: 

321 raise RuntimeError( # pragma: no cover 

322 "subset_problems must be a set or a list not {}.".format( 

323 subset_problems)) 

324 

325 try: 

326 scenario, extra = scenario_extra[:2] 

327 except TypeError as e: # pragma: no cover 

328 raise TypeError( 

329 "Unable to interpret 'scenario_extra'\n{}".format( 

330 scenario_extra)) from e 

331 if optimisations is None: 

332 optimisations = [None] 

333 if new_conv_options is None: 

334 new_conv_options = [{}] 

335 

336 if (filter_scenario is not None and 

337 not filter_scenario(model, prob, scenario, 

338 extra, new_conv_options)): 

339 continue 

340 

341 if verbose >= 2 and fLOG is not None: 

342 fLOG("[enumerate_compatible_opset] ##############################") 

343 fLOG("[enumerate_compatible_opset] scenario={} optim={} extra={} dofit={} (problem={})".format( 

344 scenario, optimisations, extra, dofit, prob)) 

345 

346 # training 

347 obs = {'scenario': scenario, 'name': model.__name__, 

348 'skl_version': sklearn_version, 'problem': prob, 

349 'method_name': method_name, 'output_index': output_index, 

350 'fit': dofit, 'conv_options': conv_options, 

351 'idtype': Xort_test.dtype, 'predict_kwargs': predict_kwargs, 

352 'init_types': init_types, 'inst': extra if extra else None, 

353 'n_features': X_train.shape[1] if len(X_train.shape) == 2 else 1} 

354 inst = None 

355 extra = set_n_jobs(model, extra, n_jobs=n_jobs) 

356 try: 

357 inst = model(**extra) 

358 except TypeError as e: # pragma: no cover 

359 if debug: # pragma: no cover 

360 raise 

361 if "__init__() missing" not in str(e): 

362 raise RuntimeError( 

363 "Unable to instantiate model '{}'.\nextra=\n{}".format( 

364 model.__name__, pprint.pformat(extra))) from e 

365 yield obs.copy() 

366 continue 

367 

368 if not _dofit_model(dofit, obs, inst, X_train, y_train, X_test, y_test, 

369 Xort_test, init_types, store_models, 

370 debug, verbose, fLOG): 

371 yield obs.copy() 

372 continue 

373 

374 # statistics about the trained model 

375 skl_infos = analyze_model(inst) 

376 for k, v in skl_infos.items(): 

377 obs['fit_' + k] = v 

378 

379 # runtime 

380 ypred = _run_skl_prediction( 

381 obs, check_runtime, assume_finite, inst, 

382 method_name, predict_kwargs, X_test, 

383 benchmark, debug, verbose, time_kwargs, 

384 skip_long_test, time_kwargs_fact, fLOG) 

385 if isinstance(ypred, Exception): 

386 yield obs.copy() 

387 continue 

388 

389 for run_obs in _call_conv_runtime_opset( 

390 obs=obs.copy(), opsets=opsets, debug=debug, 

391 new_conv_options=new_conv_options, 

392 model=model, prob=prob, scenario=scenario, 

393 extra=extra, extras=extras, conv_options=conv_options, 

394 init_types=init_types, inst=inst, 

395 optimisations=optimisations, verbose=verbose, 

396 benchmark=benchmark, runtime=runtime, 

397 filter_scenario=filter_scenario, 

398 X_test=X_test, y_test=y_test, ypred=ypred, 

399 Xort_test=Xort_test, method_name=method_name, 

400 check_runtime=check_runtime, 

401 output_index=output_index, 

402 kwargs=dict( 

403 dump_all=dump_all, 

404 dump_folder=dump_folder, 

405 node_time=node_time, 

406 skip_long_test=skip_long_test, 

407 store_models=store_models, 

408 time_kwargs=_multiply_time_kwargs( 

409 time_kwargs, time_kwargs_fact, inst) 

410 ), 

411 time_limit=time_limit, 

412 fLOG=fLOG): 

413 yield run_obs 

414 

415 

416def _check_run_benchmark(benchmark, stat_onnx, bench_memo, runtime): 

417 unique = set(stat_onnx.items()) 

418 unique.add(runtime) 

419 run_benchmark = benchmark and all( 

420 map(lambda u: unique != u, bench_memo)) 

421 if run_benchmark: 

422 bench_memo.append(unique) 

423 return run_benchmark 

424 

425 

426def _call_conv_runtime_opset( 

427 obs, opsets, debug, new_conv_options, 

428 model, prob, scenario, extra, extras, conv_options, 

429 init_types, inst, optimisations, verbose, 

430 benchmark, runtime, filter_scenario, 

431 check_runtime, X_test, y_test, ypred, Xort_test, 

432 method_name, output_index, 

433 kwargs, time_limit, fLOG): 

434 # Calls the conversion and runtime for different opets 

435 if None in opsets: 

436 set_opsets = [None] + list(sorted((_ for _ in opsets if _ is not None), 

437 reverse=True)) 

438 else: 

439 set_opsets = list(sorted(opsets, reverse=True)) 

440 bench_memo = [] 

441 

442 for opset in set_opsets: 

443 if verbose >= 2 and fLOG is not None: 

444 fLOG("[enumerate_compatible_opset] opset={} init_types={}".format( 

445 opset, init_types)) 

446 obs_op = obs.copy() 

447 if opset is not None: 

448 obs_op['opset'] = opset 

449 

450 if len(init_types) != 1: 

451 raise NotImplementedError( # pragma: no cover 

452 "Multiple types are is not implemented: " 

453 "{}.".format(init_types)) 

454 

455 if not isinstance(runtime, list): 

456 runtime = [runtime] 

457 

458 obs_op_0c = obs_op.copy() 

459 for aoptions in new_conv_options: 

460 obs_op = obs_op_0c.copy() 

461 all_conv_options = {} if conv_options is None else conv_options.copy() 

462 all_conv_options = _merge_options( 

463 all_conv_options, aoptions) 

464 obs_op['conv_options'] = all_conv_options 

465 

466 if (filter_scenario is not None and 

467 not filter_scenario(model, prob, scenario, 

468 extra, all_conv_options)): 

469 continue 

470 

471 for rt in runtime: 

472 def fct_conv(itt=inst, it=init_types[0][1], ops=opset, 

473 options=all_conv_options): 

474 if isinstance(ops, int): 

475 ops_dict = __max_supported_opsets__.copy() 

476 ops_dict[''] = ops 

477 else: 

478 ops_dict = ops 

479 return to_onnx(itt, it, target_opset=ops_dict, options=options, 

480 rewrite_ops=rt in ('', None, 'python', 

481 'python_compiled')) 

482 

483 if verbose >= 2 and fLOG is not None: 

484 fLOG( 

485 "[enumerate_compatible_opset] conversion to onnx: {}".format(all_conv_options)) 

486 try: 

487 conv, t4 = _measure_time(fct_conv)[:2] 

488 obs_op["convert_time"] = t4 

489 except (RuntimeError, IndexError, AttributeError, TypeError, 

490 ValueError, NameError, NotImplementedError) as e: 

491 if debug: 

492 fLOG(pprint.pformat(obs_op)) # pragma: no cover 

493 raise # pragma: no cover 

494 obs_op["_4convert_exc"] = e 

495 yield obs_op.copy() 

496 continue 

497 

498 if verbose >= 6 and fLOG is not None: 

499 fLOG( # pragma: no cover 

500 "[enumerate_compatible_opset] ONNX:\n{}".format(conv)) 

501 

502 if all_conv_options.get('optim', '') == 'cdist': # pragma: no cover 

503 check_cdist = [_ for _ in str(conv).split('\n') 

504 if 'CDist' in _] 

505 check_scan = [_ for _ in str(conv).split('\n') 

506 if 'Scan' in _] 

507 if len(check_cdist) == 0 and len(check_scan) > 0: 

508 raise RuntimeError( 

509 "Operator CDist was not used in\n{}" 

510 "".format(conv)) 

511 

512 obs_op0 = obs_op.copy() 

513 for optimisation in optimisations: 

514 obs_op = obs_op0.copy() 

515 if optimisation is not None: 

516 if optimisation == 'onnx': 

517 obs_op['optim'] = optimisation 

518 if len(aoptions) != 0: 

519 obs_op['optim'] += '/' + \ 

520 _dictionary2str(aoptions) 

521 conv = onnx_optimisations(conv) 

522 else: 

523 raise ValueError( # pragma: no cover 

524 "Unknown optimisation option '{}' (extra={})" 

525 "".format(optimisation, extras)) 

526 else: 

527 obs_op['optim'] = _dictionary2str(aoptions) 

528 

529 if verbose >= 3 and fLOG is not None: 

530 fLOG("[enumerate_compatible_opset] optim='{}' optimisation={} all_conv_options={}".format( 

531 obs_op['optim'], optimisation, all_conv_options)) 

532 if kwargs['store_models']: 

533 obs_op['ONNX'] = conv 

534 if verbose >= 2 and fLOG is not None: 

535 fLOG( # pragma: no cover 

536 "[enumerate_compatible_opset] onnx nodes: {}".format( 

537 len(conv.graph.node))) 

538 stat_onnx = onnx_statistics(conv) 

539 obs_op.update( 

540 {'onx_' + k: v for k, v in stat_onnx.items()}) 

541 

542 # opset_domain 

543 for op_imp in list(conv.opset_import): 

544 obs_op['domain_opset_%s' % 

545 op_imp.domain] = op_imp.version 

546 

547 run_benchmark = _check_run_benchmark( 

548 benchmark, stat_onnx, bench_memo, rt) 

549 

550 # prediction 

551 if check_runtime: 

552 yield _call_runtime(obs_op=obs_op.copy(), conv=conv, 

553 opset=opset, debug=debug, 

554 runtime=rt, inst=inst, 

555 X_test=X_test, y_test=y_test, 

556 init_types=init_types, 

557 method_name=method_name, 

558 output_index=output_index, 

559 ypred=ypred, Xort_test=Xort_test, 

560 model=model, 

561 dump_folder=kwargs['dump_folder'], 

562 benchmark=run_benchmark, 

563 node_time=kwargs['node_time'], 

564 time_kwargs=kwargs['time_kwargs'], 

565 fLOG=fLOG, verbose=verbose, 

566 store_models=kwargs['store_models'], 

567 dump_all=kwargs['dump_all'], 

568 skip_long_test=kwargs['skip_long_test'], 

569 time_limit=time_limit) 

570 else: 

571 yield obs_op.copy() # pragma: no cover 

572 

573 

574def _call_runtime(obs_op, conv, opset, debug, inst, runtime, 

575 X_test, y_test, init_types, method_name, output_index, 

576 ypred, Xort_test, model, dump_folder, 

577 benchmark, node_time, fLOG, 

578 verbose, store_models, time_kwargs, 

579 dump_all, skip_long_test, time_limit): 

580 """ 

581 Private. 

582 """ 

583 if 'onnxruntime' in runtime: 

584 old = conv.ir_version 

585 conv.ir_version = get_ir_version(opset) 

586 else: 

587 old = None 

588 

589 ser, t5, ___ = _measure_time(lambda: conv.SerializeToString()) 

590 obs_op['tostring_time'] = t5 

591 obs_op['runtime'] = runtime 

592 

593 if old is not None: 

594 conv.ir_version = old 

595 

596 # load 

597 if verbose >= 2 and fLOG is not None: 

598 fLOG("[enumerate_compatible_opset-R] load onnx") 

599 try: 

600 sess, t5, ___ = _measure_time( 

601 lambda: OnnxInference( 

602 ser, runtime=runtime, runtime_options=dict( 

603 log_severity_level=3))) 

604 obs_op['tostring_time'] = t5 

605 except (RuntimeError, ValueError, KeyError, IndexError, TypeError) as e: 

606 if debug: 

607 raise # pragma: no cover 

608 obs_op['_5ort_load_exc'] = e 

609 return obs_op 

610 

611 # compute batch 

612 if store_models: 

613 obs_op['OINF'] = sess 

614 if verbose >= 2 and fLOG is not None: 

615 fLOG("[enumerate_compatible_opset-R] compute batch with runtime " 

616 "'{}'".format(runtime)) 

617 

618 def fct_batch(se=sess, xo=Xort_test, it=init_types): # pylint: disable=W0102 

619 return se.run({it[0][0]: xo}, 

620 verbose=max(verbose - 1, 1) if debug else 0, fLOG=fLOG) 

621 

622 try: 

623 opred, t5, ___ = _measure_time(fct_batch) 

624 obs_op['ort_run_time_batch'] = t5 

625 obs_op['lambda-batch'] = (lambda xo: sess.run( 

626 {init_types[0][0]: xo}, node_time=node_time), Xort_test) 

627 except (RuntimeError, TypeError, ValueError, KeyError, IndexError) as e: 

628 if debug: 

629 raise RuntimeError("Issue with {}.".format( 

630 obs_op)) from e # pragma: no cover 

631 obs_op['_6ort_run_batch_exc'] = e 

632 if (benchmark or node_time) and 'lambda-batch' in obs_op: 

633 try: 

634 benres = benchmark_fct(*obs_op['lambda-batch'], obs=obs_op, 

635 node_time=node_time, time_kwargs=time_kwargs, 

636 skip_long_test=skip_long_test, 

637 time_limit=time_limit) 

638 obs_op['bench-batch'] = benres 

639 except (RuntimeError, TypeError, ValueError) as e: # pragma: no cover 

640 if debug: 

641 raise e # pragma: no cover 

642 obs_op['_6ort_run_batch_exc'] = e 

643 obs_op['_6ort_run_batch_bench_exc'] = e 

644 

645 # difference 

646 debug_exc = [] 

647 if verbose >= 2 and fLOG is not None: 

648 fLOG("[enumerate_compatible_opset-R] differences") 

649 if '_6ort_run_batch_exc' not in obs_op: 

650 if isinstance(opred, dict): 

651 ch = [(k, v) for k, v in opred.items()] 

652 opred = [_[1] for _ in ch] 

653 

654 if output_index != 'all': 

655 try: 

656 opred = opred[output_index] 

657 except IndexError as e: # pragma: no cover 

658 if debug: 

659 raise IndexError( 

660 "Issue with output_index={}/{}".format( 

661 output_index, len(opred))) from e 

662 obs_op['_8max_rel_diff_batch_exc'] = ( 

663 "Unable to fetch output {}/{} for model '{}'" 

664 "".format(output_index, len(opred), 

665 model.__name__)) 

666 opred = None 

667 

668 if opred is not None: 

669 if store_models: 

670 obs_op['skl_outputs'] = ypred 

671 obs_op['ort_outputs'] = opred 

672 if verbose >= 3 and fLOG is not None: 

673 fLOG("[_call_runtime] runtime prediction") 

674 _dispsimple(opred, fLOG) 

675 

676 if (method_name == "decision_function" and hasattr(opred, 'shape') and 

677 hasattr(ypred, 'shape') and len(opred.shape) == 2 and 

678 opred.shape[1] == 2 and len(ypred.shape) == 1): 

679 # decision_function, for binary classification, 

680 # raw score is a distance 

681 try: 

682 max_rel_diff = measure_relative_difference( 

683 ypred, opred[:, 1]) 

684 except AttributeError: # pragma: no cover 

685 max_rel_diff = numpy.nan 

686 else: 

687 try: 

688 max_rel_diff = measure_relative_difference( 

689 ypred, opred) 

690 except AttributeError: # pragma: no cover 

691 max_rel_diff = numpy.nan 

692 

693 if max_rel_diff >= 1e9 and debug: # pragma: no cover 

694 _shape = lambda o: o.shape if hasattr( 

695 o, 'shape') else 'no shape' 

696 raise RuntimeError( 

697 "Big difference (opset={}, runtime='{}' p='{}' s='{}')" 

698 ":\n-------\n{}-{}\n{}\n--------\n{}-{}\n{}".format( 

699 opset, runtime, obs_op['problem'], obs_op['scenario'], 

700 type(ypred), _shape(ypred), ypred, 

701 type(opred), _shape(opred), opred)) 

702 

703 if numpy.isnan(max_rel_diff): 

704 obs_op['_8max_rel_diff_batch_exc'] = ( # pragma: no cover 

705 "Unable to compute differences between" 

706 " {}-{}\n{}\n--------\n{}".format( 

707 _shape_exc( 

708 ypred), _shape_exc(opred), 

709 ypred, opred)) 

710 if debug: # pragma: no cover 

711 debug_exc.append(RuntimeError( 

712 obs_op['_8max_rel_diff_batch_exc'])) 

713 else: 

714 obs_op['max_rel_diff_batch'] = max_rel_diff 

715 if dump_folder and max_rel_diff > 1e-5: 

716 dump_into_folder(dump_folder, kind='batch', obs_op=obs_op, 

717 X_test=X_test, y_test=y_test, Xort_test=Xort_test) 

718 if debug and max_rel_diff >= 0.1: # pragma: no cover 

719 raise RuntimeError("Two big differences {}\n{}\n{}\n{}".format( 

720 max_rel_diff, inst, conv, pprint.pformat(obs_op))) 

721 

722 if debug and len(debug_exc) == 2: 

723 raise debug_exc[0] # pragma: no cover 

724 if debug and verbose >= 2: # pragma: no cover 

725 if verbose >= 3: 

726 fLOG(pprint.pformat(obs_op)) 

727 else: 

728 obs_op_log = {k: v for k, 

729 v in obs_op.items() if 'lambda-' not in k} 

730 fLOG(pprint.pformat(obs_op_log)) 

731 if verbose >= 2 and fLOG is not None: 

732 fLOG("[enumerate_compatible_opset-R] next...") 

733 if dump_all: 

734 dump = dump_into_folder(dump_folder, kind='batch', obs_op=obs_op, 

735 X_test=X_test, y_test=y_test, Xort_test=Xort_test, 

736 is_error=len(debug_exc) > 1, 

737 onnx_bytes=conv.SerializeToString(), 

738 skl_model=inst, ypred=ypred) 

739 obs_op['dumped'] = dump 

740 return obs_op 

741 

742 

743def _enumerate_validated_operator_opsets_ops(extended_list, models, skip_models): 

744 ops = [_ for _ in sklearn_operators(extended=extended_list)] 

745 

746 if models is not None: 

747 if not all(map(lambda m: isinstance(m, str), models)): 

748 raise ValueError( # pragma: no cover 

749 "models must be a set of strings.") 

750 ops_ = [_ for _ in ops if _['name'] in models] 

751 if len(ops) == 0: 

752 raise ValueError( # pragma: no cover 

753 "Parameter models is wrong: {}\n{}".format( 

754 models, ops[0])) 

755 ops = ops_ 

756 if skip_models is not None: 

757 ops = [m for m in ops if m['name'] not in skip_models] 

758 return ops 

759 

760 

761def _enumerate_validated_operator_opsets_version(runtime): 

762 from numpy import __version__ as numpy_version # delayed 

763 from onnx import __version__ as onnx_version # delayed 

764 from scipy import __version__ as scipy_version # delayed 

765 from skl2onnx import __version__ as skl2onnx_version # delayed 

766 from onnxruntime import __version__ as onnxrt_version # delayed 

767 add_versions = {'v_numpy': numpy_version, 'v_onnx': onnx_version,