Coverage for mlprodict/onnxrt/validate/validate.py: 97%

1""" 

2@file 

3@brief Validates runtime for many :scikit-learn: operators. 

4The submodule relies on :epkg:`onnxconverter_common`, 

5:epkg:`sklearn-onnx`. 

6""" 

7import pprint 

8from inspect import signature 

9import numpy 

10from numpy.linalg import LinAlgError 

11import sklearn 

12from sklearn import __all__ as sklearn__all__, __version__ as sklearn_version 

13from sklearn.exceptions import ConvergenceWarning 

14from sklearn.utils._testing import ignore_warnings 

15from ... import ( 

16 __version__ as ort_version, 

17 __max_supported_opset__, get_ir_version, 

18 __max_supported_opsets__) 

19from ...onnx_conv import ( 

20 to_onnx, register_converters, register_rewritten_operators, 

21 register_new_operators) 

22from ...tools.model_info import analyze_model, set_random_state 

23from ..onnx_inference import OnnxInference 

24from ...onnx_tools.optim.sklearn_helper import inspect_sklearn_model, set_n_jobs 

25from ...onnx_tools.optim.onnx_helper import onnx_statistics 

26from ...onnx_tools.optim import onnx_optimisations 

27from .validate_problems import find_suitable_problem 

28from .validate_scenarios import _extra_parameters 

29from .validate_difference import measure_relative_difference 

30from .validate_helper import ( 

31 _dispsimple, sklearn_operators, 

32 _measure_time, _shape_exc, dump_into_folder, 

33 default_time_kwargs, RuntimeBadResultsError, 

34 _dictionary2str, _merge_options, _multiply_time_kwargs, 

35 _get_problem_data) 

36from .validate_benchmark import benchmark_fct 

37 

38 

39@ignore_warnings(category=(UserWarning, ConvergenceWarning)) 

40def _dofit_model(dofit, obs, inst, X_train, y_train, X_test, y_test, 

41 Xort_test, init_types, store_models, 

42 debug, verbose, fLOG): 

43 if dofit: 

44 if verbose >= 2 and fLOG is not None: 

45 fLOG("[enumerate_compatible_opset] fit, type: '{}' dtype: {}".format( 

46 type(X_train), getattr(X_train, 'dtype', '-'))) 

47 try: 

48 set_random_state(inst) 

49 if y_train is None: 

50 t4 = _measure_time(lambda: inst.fit(X_train))[1] 

51 else: 

52 t4 = _measure_time( 

53 lambda: inst.fit(X_train, y_train))[1] 

54 except (AttributeError, TypeError, ValueError, 

55 IndexError, NotImplementedError, MemoryError, 

56 LinAlgError, StopIteration) as e: 

57 if debug: 

58 raise # pragma: no cover 

59 obs["_1training_time_exc"] = str(e) 

60 return False 

61 

62 obs["training_time"] = t4 

63 try: 

64 skl_st = inspect_sklearn_model(inst) 

65 except NotImplementedError: 

66 skl_st = {} 

67 obs.update({'skl_' + k: v for k, v in skl_st.items()}) 

68 

69 if store_models: 

70 obs['MODEL'] = inst 

71 obs['X_test'] = X_test 

72 obs['Xort_test'] = Xort_test 

73 obs['init_types'] = init_types 

74 else: 

75 obs["training_time"] = 0. 

76 if store_models: 

77 obs['MODEL'] = inst 

78 obs['init_types'] = init_types 

79 

80 return True 

81 

82 

83def _run_skl_prediction(obs, check_runtime, assume_finite, inst, 

84 method_name, predict_kwargs, X_test, 

85 benchmark, debug, verbose, time_kwargs, 

86 skip_long_test, time_kwargs_fact, fLOG): 

87 if not check_runtime: 

88 return None # pragma: no cover 

89 if verbose >= 2 and fLOG is not None: 

90 fLOG("[enumerate_compatible_opset] check_runtime SKL {}-{}-{}-{}-{}".format( 

91 id(inst), method_name, predict_kwargs, time_kwargs, 

92 time_kwargs_fact)) 

93 with sklearn.config_context(assume_finite=assume_finite): 

94 # compute sklearn prediction 

95 obs['ort_version'] = ort_version 

96 try: 

97 meth = getattr(inst, method_name) 

98 except AttributeError as e: # pragma: no cover 

99 if debug: 

100 raise # pragma: no cover 

101 obs['_2skl_meth_exc'] = str(e) 

102 return e 

103 try: 

104 ypred, t4, ___ = _measure_time( 

105 lambda: meth(X_test, **predict_kwargs)) 

106 obs['lambda-skl'] = (lambda xo: meth(xo, **predict_kwargs), X_test) 

107 except (ValueError, AttributeError, # pragma: no cover 

108 TypeError, MemoryError, IndexError) as e: 

109 if debug: 

110 raise # pragma: no cover 

111 obs['_3prediction_exc'] = str(e) 

112 return e 

113 obs['prediction_time'] = t4 

114 obs['assume_finite'] = assume_finite 

115 if benchmark and 'lambda-skl' in obs: 

116 obs['bench-skl'] = benchmark_fct( 

117 *obs['lambda-skl'], obs=obs, 

118 time_kwargs=_multiply_time_kwargs( 

119 time_kwargs, time_kwargs_fact, inst), 

120 skip_long_test=skip_long_test) 

121 if verbose >= 3 and fLOG is not None: 

122 fLOG("[enumerate_compatible_opset] scikit-learn prediction") 

123 _dispsimple(ypred, fLOG) 

124 if verbose >= 2 and fLOG is not None: 

125 fLOG("[enumerate_compatible_opset] predictions stored") 

126 return ypred 

127 

128 

129def _retrieve_problems_extra(model, verbose, fLOG, extended_list): 

130 """ 

131 Use by @see fn enumerate_compatible_opset. 

132 """ 

133 extras = None 

134 if extended_list: 

135 from ...onnx_conv.validate_scenarios import find_suitable_problem as fsp_extended 

136 problems = fsp_extended(model) 

137 if problems is not None: 

138 from ...onnx_conv.validate_scenarios import build_custom_scenarios as fsp_scenarios 

139 extra_parameters = fsp_scenarios() 

140 

141 if verbose >= 2 and fLOG is not None: 

142 fLOG( 

143 f"[enumerate_compatible_opset] found custom for model={model}") 

144 extras = extra_parameters.get(model, None) 

145 if extras is not None: 

146 fLOG( 

147 f"[enumerate_compatible_opset] found custom scenarios={extras}") 

148 else: 

149 problems = None 

150 

151 if problems is None: 

152 # scikit-learn 

153 extra_parameters = _extra_parameters 

154 try: 

155 problems = find_suitable_problem(model) 

156 except RuntimeError as e: # pragma: no cover 

157 return {'name': model.__name__, 'skl_version': sklearn_version, 

158 '_0problem_exc': e}, extras 

159 extras = extra_parameters.get(model, [('default', {})]) 

160 

161 # checks existence of random_state 

162 sig = signature(model.__init__) 

163 if 'random_state' in sig.parameters: 

164 new_extras = [] 

165 for extra in extras: 

166 if 'random_state' not in extra[1]: 

167 ps = extra[1].copy() 

168 ps['random_state'] = 42 

169 if len(extra) == 2: 

170 extra = (extra[0], ps) 

171 else: 

172 extra = (extra[0], ps) + extra[2:] 

173 new_extras.append(extra) 

174 extras = new_extras 

175 

176 return problems, extras 

177 

178 

179def enumerate_compatible_opset(model, opset_min=-1, opset_max=-1, # pylint: disable=R0914 

180 check_runtime=True, debug=False, 

181 runtime='python', dump_folder=None, 

182 store_models=False, benchmark=False, 

183 assume_finite=True, node_time=False, 

184 fLOG=print, filter_exp=None, 

185 verbose=0, time_kwargs=None, 

186 extended_list=False, dump_all=False, 

187 n_features=None, skip_long_test=True, 

188 filter_scenario=None, time_kwargs_fact=None, 

189 time_limit=4, n_jobs=None): 

190 """ 

191 Lists all compatible opsets for a specific model. 

192 

193 @param model operator class 

194 @param opset_min starts with this opset 

195 @param opset_max ends with this opset (None to use 

196 current onnx opset) 

197 @param check_runtime checks that runtime can consume the 

198 model and compute predictions 

199 @param debug catch exception (True) or not (False) 

200 @param runtime test a specific runtime, by default ``'python'`` 

201 @param dump_folder dump information to replicate in case of mismatch 

202 @param dump_all dump all models not only the one which fail 

203 @param store_models if True, the function 

204 also stores the fitted model and its conversion 

205 into :epkg:`ONNX` 

206 @param benchmark if True, measures the time taken by each function 

207 to predict for different number of rows 

208 @param fLOG logging function 

209 @param filter_exp function which tells if the experiment must be run, 

210 None to run all, takes *model, problem* as an input 

211 @param filter_scenario second function which tells if the experiment must be run, 

212 None to run all, takes *model, problem, scenario, extra, options* 

213 as an input 

214 @param node_time collect time for each node in the :epkg:`ONNX` graph 

215 @param assume_finite See `config_context 

216 <https://scikit-learn.org/stable/modules/generated/ 

217 sklearn.config_context.html>`_, If True, validation for finiteness 

218 will be skipped, saving time, but leading to potential crashes. 

219 If False, validation for finiteness will be performed, avoiding error. 

220 @param verbose verbosity 

221 @param extended_list extends the list to custom converters 

222 and problems 

223 @param time_kwargs to define a more precise way to measure a model 

224 @param n_features modifies the shorts datasets used to train the models 

225 to use exactly this number of features, it can also 

226 be a list to test multiple datasets 

227 @param skip_long_test skips tests for high values of N if they seem too long 

228 @param time_kwargs_fact see :func:`_multiply_time_kwargs <mlprodict.onnxrt.validate.validate_helper._multiply_time_kwargs>` 

229 @param time_limit to stop benchmarking after this amount of time was spent 

230 @param n_jobs *n_jobs* is set to the number of CPU by default unless this 

231 value is changed 

232 @return dictionaries, each row has the following 

233 keys: opset, exception if any, conversion time, 

234 problem chosen to test the conversion... 

235 

236 The function requires :epkg:`sklearn-onnx`. 

237 The outcome can be seen at pages references 

238 by :ref:`l-onnx-availability`. 

239 The parameter *time_kwargs* is a dictionary which defines the 

240 number of times to repeat the same predictions in order 

241 to give more precise figures. The default value (if None) is returned 

242 by the following code: 

243 

244 .. runpython:: 

245 :showcode: 

246 :warningout: DeprecationWarning 

247 

248 from mlprodict.onnxrt.validate.validate_helper import default_time_kwargs 

249 import pprint 

250 pprint.pprint(default_time_kwargs()) 

251 

252 Parameter *time_kwargs_fact* multiples these values for some 

253 specific models. ``'lin'`` multiplies by 10 when the model 

254 is linear. 

255 """ 

256 if opset_min == -1: 

257 opset_min = __max_supported_opset__ # pragma: no cover 

258 if opset_max == -1: 

259 opset_max = __max_supported_opset__ # pragma: no cover 

260 if verbose > 0 and fLOG is not None: 

261 fLOG( 

262 f"[enumerate_compatible_opset] opset in [{opset_min}, {opset_max}].") 

263 if verbose > 1 and fLOG: 

264 fLOG( 

265 f"[enumerate_compatible_opset] validate class '{model.__name__}'.") 

266 if verbose > 2: 

267 fLOG(model) 

268 

269 if time_kwargs is None: 

270 time_kwargs = default_time_kwargs() 

271 problems, extras = _retrieve_problems_extra( 

272 model, verbose, fLOG, extended_list) 

273 if isinstance(problems, dict): 

274 yield problems # pragma: no cover 

275 problems = [] # pragma: no cover 

276 

277 if opset_max is None: 

278 opset_max = __max_supported_opset__ # pragma: no cover 

279 opsets = list(range(opset_min, opset_max + 1)) # pragma: no cover 

280 opsets.append(None) # pragma: no cover 

281 else: 

282 opsets = list(range(opset_min, opset_max + 1)) 

283 

284 if extras is None: 

285 problems = [] 

286 yield {'name': model.__name__, 'skl_version': sklearn_version, 

287 '_0problem_exc': 'SKIPPED'} 

288 

289 if not isinstance(n_features, list): 

290 n_features = [n_features] 

291 

292 for prob in problems: 

293 if filter_exp is not None and not filter_exp(model, prob): 

294 continue 

295 for n_feature in n_features: 

296 if verbose >= 2 and fLOG is not None: 

297 fLOG("[enumerate_compatible_opset] problem={} n_feature={}".format( 

298 prob, n_feature)) 

299 

300 (X_train, X_test, y_train, 

301 y_test, Xort_test, 

302 init_types, conv_options, method_name, 

303 output_index, dofit, predict_kwargs) = _get_problem_data(prob, n_feature) 

304 

305 for scenario_extra in extras: 

306 subset_problems = None 

307 optimisations = None 

308 new_conv_options = None 

309 if len(scenario_extra) > 2: 

310 options = scenario_extra[2] 

311 if isinstance(options, dict): 

312 subset_problems = options.get('subset_problems', None) 

313 optimisations = options.get('optim', None) 

314 new_conv_options = options.get('conv_options', None) 

315 else: 

316 subset_problems = options 

317 

318 if subset_problems and isinstance(subset_problems, (list, set)): 

319 if prob not in subset_problems: 

320 # Skips unrelated problem for a specific configuration. 

321 continue 

322 elif subset_problems is not None: 

323 raise RuntimeError( # pragma: no cover 

324 "subset_problems must be a set or a list not {}.".format( 

325 subset_problems)) 

326 

327 try: 

328 scenario, extra = scenario_extra[:2] 

329 except TypeError as e: # pragma: no cover 

330 raise TypeError( 

331 "Unable to interpret 'scenario_extra'\n{}".format( 

332 scenario_extra)) from e 

333 if optimisations is None: 

334 optimisations = [None] 

335 if new_conv_options is None: 

336 new_conv_options = [{}] 

337 

338 if (filter_scenario is not None and 

339 not filter_scenario(model, prob, scenario, 

340 extra, new_conv_options)): 

341 continue 

342 

343 if verbose >= 2 and fLOG is not None: 

344 fLOG("[enumerate_compatible_opset] ##############################") 

345 fLOG("[enumerate_compatible_opset] scenario={} optim={} extra={} dofit={} (problem={})".format( 

346 scenario, optimisations, extra, dofit, prob)) 

347 

348 # training 

349 obs = {'scenario': scenario, 'name': model.__name__, 

350 'skl_version': sklearn_version, 'problem': prob, 

351 'method_name': method_name, 'output_index': output_index, 

352 'fit': dofit, 'conv_options': conv_options, 

353 'idtype': Xort_test.dtype, 'predict_kwargs': predict_kwargs, 

354 'init_types': init_types, 'inst': extra if extra else None, 

355 'n_features': X_train.shape[1] if len(X_train.shape) == 2 else 1} 

356 inst = None 

357 extra = set_n_jobs(model, extra, n_jobs=n_jobs) 

358 try: 

359 inst = model(**extra) 

360 except TypeError as e: # pragma: no cover 

361 if debug: # pragma: no cover 

362 raise 

363 if "__init__() missing" not in str(e): 

364 raise RuntimeError( 

365 "Unable to instantiate model '{}'.\nextra=\n{}".format( 

366 model.__name__, pprint.pformat(extra))) from e 

367 yield obs.copy() 

368 continue 

369 

370 if not _dofit_model(dofit, obs, inst, X_train, y_train, X_test, y_test, 

371 Xort_test, init_types, store_models, 

372 debug, verbose, fLOG): 

373 yield obs.copy() 

374 continue 

375 

376 # statistics about the trained model 

377 skl_infos = analyze_model(inst) 

378 for k, v in skl_infos.items(): 

379 obs['fit_' + k] = v 

380 

381 # runtime 

382 ypred = _run_skl_prediction( 

383 obs, check_runtime, assume_finite, inst, 

384 method_name, predict_kwargs, X_test, 

385 benchmark, debug, verbose, time_kwargs, 

386 skip_long_test, time_kwargs_fact, fLOG) 

387 if isinstance(ypred, Exception): 

388 yield obs.copy() 

389 continue 

390 

391 for run_obs in _call_conv_runtime_opset( 

392 obs=obs.copy(), opsets=opsets, debug=debug, 

393 new_conv_options=new_conv_options, 

394 model=model, prob=prob, scenario=scenario, 

395 extra=extra, extras=extras, conv_options=conv_options, 

396 init_types=init_types, inst=inst, 

397 optimisations=optimisations, verbose=verbose, 

398 benchmark=benchmark, runtime=runtime, 

399 filter_scenario=filter_scenario, 

400 X_test=X_test, y_test=y_test, ypred=ypred, 

401 Xort_test=Xort_test, method_name=method_name, 

402 check_runtime=check_runtime, 

403 output_index=output_index, 

404 kwargs=dict( 

405 dump_all=dump_all, 

406 dump_folder=dump_folder, 

407 node_time=node_time, 

408 skip_long_test=skip_long_test, 

409 store_models=store_models, 

410 time_kwargs=_multiply_time_kwargs( 

411 time_kwargs, time_kwargs_fact, inst) 

412 ), 

413 time_limit=time_limit, 

414 fLOG=fLOG): 

415 yield run_obs 

416 

417 

418def _check_run_benchmark(benchmark, stat_onnx, bench_memo, runtime): 

419 unique = set(stat_onnx.items()) 

420 unique.add(runtime) 

421 run_benchmark = benchmark and all( 

422 map(lambda u: unique != u, bench_memo)) 

423 if run_benchmark: 

424 bench_memo.append(unique) 

425 return run_benchmark 

426 

427 

428def _call_conv_runtime_opset( 

429 obs, opsets, debug, new_conv_options, 

430 model, prob, scenario, extra, extras, conv_options, 

431 init_types, inst, optimisations, verbose, 

432 benchmark, runtime, filter_scenario, 

433 check_runtime, X_test, y_test, ypred, Xort_test, 

434 method_name, output_index, 

435 kwargs, time_limit, fLOG): 

436 # Calls the conversion and runtime for different opets 

437 if None in opsets: 

438 set_opsets = [None] + list(sorted((_ for _ in opsets if _ is not None), 

439 reverse=True)) 

440 else: 

441 set_opsets = list(sorted(opsets, reverse=True)) 

442 bench_memo = [] 

443 

444 for opset in set_opsets: 

445 if verbose >= 2 and fLOG is not None: 

446 fLOG( 

447 f"[enumerate_compatible_opset] opset={opset} init_types={init_types}") 

448 obs_op = obs.copy() 

449 if opset is not None: 

450 obs_op['opset'] = opset 

451 

452 if len(init_types) != 1: 

453 raise NotImplementedError( # pragma: no cover 

454 f"Multiple types are is not implemented: {init_types}.") 

455 

456 if not isinstance(runtime, list): 

457 runtime = [runtime] 

458 

459 obs_op_0c = obs_op.copy() 

460 for aoptions in new_conv_options: 

461 obs_op = obs_op_0c.copy() 

462 all_conv_options = {} if conv_options is None else conv_options.copy() 

463 all_conv_options = _merge_options( 

464 all_conv_options, aoptions) 

465 obs_op['conv_options'] = all_conv_options 

466 

467 if (filter_scenario is not None and 

468 not filter_scenario(model, prob, scenario, 

469 extra, all_conv_options)): 

470 continue 

471 

472 for rt in runtime: 

473 def fct_conv(itt=inst, it=init_types[0][1], ops=opset, 

474 options=all_conv_options): 

475 if isinstance(ops, int): 

476 ops_dict = __max_supported_opsets__.copy() 

477 ops_dict[''] = ops 

478 else: 

479 ops_dict = ops 

480 return to_onnx(itt, it, target_opset=ops_dict, options=options, 

481 rewrite_ops=rt in ('', None, 'python', 

482 'python_compiled')) 

483 

484 if verbose >= 2 and fLOG is not None: 

485 fLOG( 

486 f"[enumerate_compatible_opset] conversion to onnx: {all_conv_options}") 

487 try: 

488 conv, t4 = _measure_time(fct_conv)[:2] 

489 obs_op["convert_time"] = t4 

490 except (RuntimeError, IndexError, AttributeError, TypeError, 

491 ValueError, NameError, NotImplementedError) as e: 

492 if debug: 

493 fLOG(pprint.pformat(obs_op)) # pragma: no cover 

494 raise # pragma: no cover 

495 obs_op["_4convert_exc"] = e 

496 yield obs_op.copy() 

497 continue 

498 

499 if verbose >= 6 and fLOG is not None: 

500 fLOG( # pragma: no cover 

501 f"[enumerate_compatible_opset] ONNX:\n{conv}") 

502 

503 if all_conv_options.get('optim', '') == 'cdist': # pragma: no cover 

504 check_cdist = [_ for _ in str(conv).split('\n') 

505 if 'CDist' in _] 

506 check_scan = [_ for _ in str(conv).split('\n') 

507 if 'Scan' in _] 

508 if len(check_cdist) == 0 and len(check_scan) > 0: 

509 raise RuntimeError( 

510 f"Operator CDist was not used in\n{conv}") 

511 

512 obs_op0 = obs_op.copy() 

513 for optimisation in optimisations: 

514 obs_op = obs_op0.copy() 

515 if optimisation is not None: 

516 if optimisation == 'onnx': 

517 obs_op['optim'] = optimisation 

518 if len(aoptions) != 0: 

519 obs_op['optim'] += '/' + \ 

520 _dictionary2str(aoptions) 

521 conv = onnx_optimisations(conv) 

522 else: 

523 raise ValueError( # pragma: no cover 

524 "Unknown optimisation option '{}' (extra={})" 

525 "".format(optimisation, extras)) 

526 else: 

527 obs_op['optim'] = _dictionary2str(aoptions) 

528 

529 if verbose >= 3 and fLOG is not None: 

530 fLOG("[enumerate_compatible_opset] optim='{}' optimisation={} all_conv_options={}".format( 

531 obs_op['optim'], optimisation, all_conv_options)) 

532 if kwargs['store_models']: 

533 obs_op['ONNX'] = conv 

534 if verbose >= 2 and fLOG is not None: 

535 fLOG( # pragma: no cover 

536 "[enumerate_compatible_opset] onnx nodes: {}".format( 

537 len(conv.graph.node))) 

538 stat_onnx = onnx_statistics(conv) 

539 obs_op.update( 

540 {'onx_' + k: v for k, v in stat_onnx.items()}) 

541 

542 # opset_domain 

543 for op_imp in list(conv.opset_import): 

544 obs_op[f'domain_opset_{op_imp.domain}'] = op_imp.version 

545 

546 run_benchmark = _check_run_benchmark( 

547 benchmark, stat_onnx, bench_memo, rt) 

548 

549 # prediction 

550 if check_runtime: 

551 yield _call_runtime(obs_op=obs_op.copy(), conv=conv, 

552 opset=opset, debug=debug, 

553 runtime=rt, inst=inst, 

554 X_test=X_test, y_test=y_test, 

555 init_types=init_types, 

556 method_name=method_name, 

557 output_index=output_index, 

558 ypred=ypred, Xort_test=Xort_test, 

559 model=model, 

560 dump_folder=kwargs['dump_folder'], 

561 benchmark=run_benchmark, 

562 node_time=kwargs['node_time'], 

563 time_kwargs=kwargs['time_kwargs'], 

564 fLOG=fLOG, verbose=verbose, 

565 store_models=kwargs['store_models'], 

566 dump_all=kwargs['dump_all'], 

567 skip_long_test=kwargs['skip_long_test'], 

568 time_limit=time_limit) 

569 else: 

570 yield obs_op.copy() # pragma: no cover 

571 

572 

573def _call_runtime(obs_op, conv, opset, debug, inst, runtime, 

574 X_test, y_test, init_types, method_name, output_index, 

575 ypred, Xort_test, model, dump_folder, 

576 benchmark, node_time, fLOG, 

577 verbose, store_models, time_kwargs, 

578 dump_all, skip_long_test, time_limit): 

579 """ 

580 Private. 

581 """ 

582 if 'onnxruntime' in runtime: 

583 old = conv.ir_version 

584 conv.ir_version = get_ir_version(opset) 

585 else: 

586 old = None 

587 

588 ser, t5, ___ = _measure_time(lambda: conv.SerializeToString()) 

589 obs_op['tostring_time'] = t5 

590 obs_op['runtime'] = runtime 

591 

592 if old is not None: 

593 conv.ir_version = old 

594 

595 # load 

596 if verbose >= 2 and fLOG is not None: 

597 fLOG("[enumerate_compatible_opset-R] load onnx") 

598 try: 

599 sess, t5, ___ = _measure_time( 

600 lambda: OnnxInference( 

601 ser, runtime=runtime, runtime_options=dict( 

602 log_severity_level=3))) 

603 obs_op['tostring_time'] = t5 

604 except (RuntimeError, ValueError, KeyError, IndexError, TypeError) as e: 

605 if debug: 

606 raise # pragma: no cover 

607 obs_op['_5ort_load_exc'] = e 

608 return obs_op 

609 

610 # compute batch 

611 if store_models: 

612 obs_op['OINF'] = sess 

613 if verbose >= 2 and fLOG is not None: 

614 fLOG( 

615 f"[enumerate_compatible_opset-R] compute batch with runtime '{runtime}'") 

616 

617 def fct_batch(se=sess, xo=Xort_test, it=init_types): # pylint: disable=W0102 

618 return se.run({it[0][0]: xo}, 

619 verbose=max(verbose - 1, 1) if debug else 0, fLOG=fLOG) 

620 

621 try: 

622 opred, t5, ___ = _measure_time(fct_batch) 

623 obs_op['ort_run_time_batch'] = t5 

624 obs_op['lambda-batch'] = (lambda xo: sess.run( 

625 {init_types[0][0]: xo}, node_time=node_time), Xort_test) 

626 except (RuntimeError, TypeError, ValueError, KeyError, IndexError) as e: 

627 if debug: 

628 raise RuntimeError( 

629 f"Issue with {obs_op}.") from e # pragma: no cover 

630 obs_op['_6ort_run_batch_exc'] = e 

631 if (benchmark or node_time) and 'lambda-batch' in obs_op: 

632 try: 

633 benres = benchmark_fct(*obs_op['lambda-batch'], obs=obs_op, 

634 node_time=node_time, time_kwargs=time_kwargs, 

635 skip_long_test=skip_long_test, 

636 time_limit=time_limit) 

637 obs_op['bench-batch'] = benres 

638 except (RuntimeError, TypeError, ValueError) as e: # pragma: no cover 

639 if debug: 

640 raise e # pragma: no cover 

641 obs_op['_6ort_run_batch_exc'] = e 

642 obs_op['_6ort_run_batch_bench_exc'] = e 

643 

644 # difference 

645 debug_exc = [] 

646 if verbose >= 2 and fLOG is not None: 

647 fLOG("[enumerate_compatible_opset-R] differences") 

648 if '_6ort_run_batch_exc' not in obs_op: 

649 if isinstance(opred, dict): 

650 ch = [(k, v) for k, v in opred.items()] 

651 opred = [_[1] for _ in ch] 

652 

653 if output_index != 'all': 

654 try: 

655 opred = opred[output_index] 

656 except IndexError as e: # pragma: no cover 

657 if debug: 

658 raise IndexError( 

659 f"Issue with output_index={output_index}/{len(opred)}") from e 

660 obs_op['_8max_rel_diff_batch_exc'] = ( 

661 "Unable to fetch output {}/{} for model '{}'" 

662 "".format(output_index, len(opred), 

663 model.__name__)) 

664 opred = None 

665 

666 if opred is not None: 

667 if store_models: 

668 obs_op['skl_outputs'] = ypred 

669 obs_op['ort_outputs'] = opred 

670 if verbose >= 3 and fLOG is not None: 

671 fLOG("[_call_runtime] runtime prediction") 

672 _dispsimple(opred, fLOG) 

673 

674 if (method_name == "decision_function" and hasattr(opred, 'shape') and 

675 hasattr(ypred, 'shape') and len(opred.shape) == 2 and 

676 opred.shape[1] == 2 and len(ypred.shape) == 1): 

677 # decision_function, for binary classification, 

678 # raw score is a distance 

679 try: 

680 max_rel_diff = measure_relative_difference( 

681 ypred, opred[:, 1]) 

682 except AttributeError: # pragma: no cover 

683 max_rel_diff = numpy.nan 

684 else: 

685 try: 

686 max_rel_diff = measure_relative_difference( 

687 ypred, opred) 

688 except AttributeError: # pragma: no cover 

689 max_rel_diff = numpy.nan 

690 

691 if max_rel_diff >= 1e9 and debug: # pragma: no cover 

692 _shape = lambda o: o.shape if hasattr( 

693 o, 'shape') else 'no shape' 

694 raise RuntimeError( 

695 "Big difference (opset={}, runtime='{}' p='{}' s='{}')" 

696 ":\n-------\n{}-{}\n{}\n--------\n{}-{}\n{}".format( 

697 opset, runtime, obs_op['problem'], obs_op['scenario'], 

698 type(ypred), _shape(ypred), ypred, 

699 type(opred), _shape(opred), opred)) 

700 

701 if numpy.isnan(max_rel_diff): 

702 obs_op['_8max_rel_diff_batch_exc'] = ( # pragma: no cover 

703 "Unable to compute differences between" 

704 " {}-{}\n{}\n--------\n{}".format( 

705 _shape_exc( 

706 ypred), _shape_exc(opred), 

707 ypred, opred)) 

708 if debug: # pragma: no cover 

709 debug_exc.append(RuntimeError( 

710 obs_op['_8max_rel_diff_batch_exc'])) 

711 else: 

712 obs_op['max_rel_diff_batch'] = max_rel_diff 

713 if dump_folder and max_rel_diff > 1e-5: 

714 dump_into_folder(dump_folder, kind='batch', obs_op=obs_op, 

715 X_test=X_test, y_test=y_test, Xort_test=Xort_test) 

716 if debug and max_rel_diff >= 0.1: # pragma: no cover 

717 raise RuntimeError("Two big differences {}\n{}\n{}\n{}".format( 

718 max_rel_diff, inst, conv, pprint.pformat(obs_op))) 

719 

720 if debug and len(debug_exc) == 2: 

721 raise debug_exc[0] # pragma: no cover 

722 if debug and verbose >= 2: # pragma: no cover 

723 if verbose >= 3: 

724 fLOG(pprint.pformat(obs_op)) 

725 else: 

726 obs_op_log = {k: v for k, 

727 v in obs_op.items() if 'lambda-' not in k} 

728 fLOG(pprint.pformat(obs_op_log)) 

729 if verbose >= 2 and fLOG is not None: 

730 fLOG("[enumerate_compatible_opset-R] next...") 

731 if dump_all: 

732 dump = dump_into_folder(dump_folder, kind='batch', obs_op=obs_op, 

733 X_test=X_test, y_test=y_test, Xort_test=Xort_test, 

734 is_error=len(debug_exc) > 1, 

735 onnx_bytes=conv.SerializeToString(), 

736 skl_model=inst, ypred=ypred) 

737 obs_op['dumped'] = dump 

738 return obs_op 

739 

740 

741def _enumerate_validated_operator_opsets_ops(extended_list, models, skip_models): 

742 ops = [_ for _ in sklearn_operators(extended=extended_list)] 

743 

744 if models is not None: 

745 if not all(map(lambda m: isinstance(m, str), models)): 

746 raise ValueError( # pragma: no cover 

747 "models must be a set of strings.") 

748 ops_ = [_ for _ in ops if _['name'] in models] 

749 if len(ops) == 0: 

750 raise ValueError( # pragma: no cover 

751 f"Parameter models is wrong: {models}\n{ops[0]}") 

752 ops = ops_ 

753 if skip_models is not None: 

754 ops = [m for m in ops if m['name'] not in skip_models] 

755 return ops 

756 

757 

758def _enumerate_validated_operator_opsets_version(runtime): 

759 from numpy import __version__ as numpy_version # delayed 

760 from onnx import __version__ as onnx_version # delayed 

761 from scipy import __version__ as scipy_version # delayed 

762 from skl2onnx import __version__ as skl2onnx_version # delayed 

763 from onnxruntime import __version__ as onnxrt_version # delayed 

764 add_versions = {'v_numpy': numpy_version, 'v_onnx': onnx_version, 

765 'v_scipy': scipy_version, 'v_skl2onnx': skl2onnx_version, 

766 'v_sklearn': sklearn_version, 'v_onnxruntime': ort_version} 

767 if "onnxruntime" in runtime: 

768 add_versions['v_onnxruntime'] = onnxrt_version 

769 return add_versions 

770 

771 

772def enumerate_validated_operator_opsets(verbose=0, opset_min=-1, opset_max=-1, 

773 check_runtime=True, debug=False, runtime='python',