Coverage for src/km3dq_common/config_library.py: 62%

220 statements  

« prev     ^ index     » next       coverage.py v7.10.7, created at 2025-10-27 12:26 +0000

1#! /usr/bin/env python 

2############################################################################### 

3# Definition of QAQC variable, defects, data-quality tags... 

4# 

5# Developer: Benjamin Trocme (benjamin.trocme at apc.in2p3.fr) - 2023 

6 

7import tomli 

8import sys 

9import re 

10import urllib.request 

11import atexit 

12 

13if sys.version_info < (3, 9): 

14 import importlib_resources 

15 

16 try: 

17 from importlib_resources import as_file 

18 except ImportError: 

19 from importlib_resources.trees import as_file 

20else: 

21 import importlib.resources as importlib_resources 

22 from importlib.resources import as_file 

23 

24try: 

25 from contextlib import ExitStack 

26except ImportError: 

27 from contextlib2 import ExitStack 

28 

29 

30############################################################################### 

31def read_configuration(filename, source, raise_missing=True): 

32 """ 

33 """ 

34 

35 if source == "sftp": 

36 with urllib.request.urlopen( 

37 f"https://sftp.km3net.de/data/km3dq_lw_db/Common/{filename}.toml" 

38 ) as s_f: 

39 return tomli.loads(s_f.read().decode("utf-8")) 

40 else: 

41 dir_absolute_path = "." # Facility never used / debugged 

42 ref = importlib_resources.files("km3dq_common") / f"{dir_absolute_path}/{filename}.toml" 

43 file_manager = ExitStack() 

44 atexit.register(file_manager.close) 

45 file_path = file_manager.enter_context(as_file(ref)) 

46 if raise_missing and not file_path.exists(): 

47 raise RuntimeError(f"Unknown or missing file: {filename}") 

48 return tomli.load(file_path) 

49 

50 

51############################################################################### 

52def get_detx_caract(det, source="sftp"): 

53 """ 

54 

55 Return the number of DOMs (incl base module so far) and PMts 

56 

57 Formerly, it was defined in km3dq_perf_aux but now use the name of the detector 

58 

59 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/), 

60 either the directory absolute-path. 

61 """ 

62 

63 r = re.search(r"D.[AO]RC.(\d+)", det) 

64 return dict( 

65 active_dus=int(r.group(1)), 

66 doms=int(r.group(1)) * 18, 

67 pmts=int(r.group(1)) * 18 * 31, 

68 ) 

69 

70 

71############################################################################### 

72def configure_var_name(v_prop, plot_type="basic_plots"): 

73 """Description of JQAQC variables""" 

74 

75 # QAQC variables 

76 v_prop["description"] = { 

77 "run": "Run", 

78 "livetime_s": "Livetime (s)", 

79 "nb_of_meta": "Nb of metadata", 

80 "timestampdiff": "Missing time-slices", 

81 "timestampdiff_r": ("Missing time-slices" "/Run duration"), 

82 "triggerrate": "Trigger rate(Hz)", 

83 "HRV": "High rate veto bit", 

84 "DAQ": "Fraction of data without UDP packet-loss", 

85 "WR": "White rabbit status", 

86 "FIFO": "FIFO almost full", 

87 "PMTs": "Number of active PMTs", 

88 "pmts_norm": "Fraction of active PMTs", 

89 "MEAN_Rate_Hz": "Mean rate (Hz)", 

90 "RMS_Rate_Hz": "RMS rate (Hz)", 

91 "hrv_fifo_failures": "Number of hrv/fifo failures", 

92 "duplic_timeslices": "Number of duplicated timeslices", 

93 "out_sync": "Out-of-sync", 

94 "out_usync": "Micro out-of-sync", 

95 "JDAQJTProc": "Triggered != Retriggered", 

96 "Acoustics": "Number of acoustic events", 

97 "Acoustics_per_mn": "Number of acoustic events per minute", 

98 "zero_AHRS": ("Number of DOMs without valid " "AHRS data"), 

99 "ahrs_per_mn": ("Number of AHRS event per minute " "in > 80% of DOMs"), 

100 "JDAQEvent": "Total", 

101 "JTrigger3DShower": "3D shower trigger", 

102 "JTriggerMXShower": "MX shower trigger", 

103 "JTrigger3DMuon": "3D Muon trigger ", 

104 "HV_check": "High-voltage inconsistencies", 

105 "DB_check": "Wrong position calibrations" 

106 } 

107 

108 if plot_type == "performance_plots": 

109 v_prop["description"]["DAQ"] = "UDP packet-loss" 

110 

111 

112############################################################################### 

113def configure_dataquality_tag(tag, source="sftp"): 

114 """ 

115 TOMl configuration of the data quality tags 

116 === Arguments === 

117 - tag : tag name - [string] - Ex: "default", "calibration"... 

118 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/), 

119 either the directory absolute-path. 

120 === Output === 

121 - Dictionnary with all tag characteristics for all detectors 

122 """ 

123 

124 #################################################### 

125 def single_property_process(tag, tag_parent, toml_input): 

126 output = {} 

127 

128 for i_tag in (tag_parent, tag): 

129 if i_tag in toml_input.keys(): 

130 output = toml_input[i_tag] 

131 

132 return output 

133 

134 #################################################### 

135 def all_site_process(tag, tag_parent, toml_input, sites): 

136 output = {"ARCA": "", "ORCA": ""} 

137 

138 for i_tag in (tag_parent, tag): 

139 if i_tag == "": # No tag_parent 

140 continue 

141 for i_site in sites: 

142 if i_site in toml_input[i_tag].keys(): 

143 output[i_site] = toml_input[i_tag][i_site] 

144 

145 return output 

146 

147 #################################################### 

148 def all_dets_process(tag, tag_parent, toml_input, det_list): 

149 # det_list: detector list 

150 output = {} 

151 

152 for i_tag in (tag_parent, tag): 

153 if i_tag == "": # No tag_parent 

154 continue 

155 for i_site in ["ARCA", "ORCA"]: 

156 for i_det in det_list[i_site]: 

157 if i_tag in toml_input.keys(): 

158 if i_det in toml_input[i_tag]: # Detector explicitly defined 

159 output[i_det] = toml_input[i_tag][i_det] 

160 elif i_site in toml_input[i_tag]: # Use the site definition 

161 output[i_det] = toml_input[i_tag][i_site] 

162 

163 return output 

164 

165 #################################################### 

166 # First retrieve the general caracteristics: 

167 # status, description, dataset name, grl... 

168 try: 

169 toml = read_configuration("dataquality_tag", source) 

170 

171 dq_t = {} 

172 # The path directory is the name of the data-quality tag 

173 dq_t["dir_path"] = tag 

174 # Retrieve the general descriptions 

175 dq_t["name"] = toml["short_descr"][tag] 

176 dq_t["descr"] = toml["descr"][tag] 

177 dq_t["status"] = toml["status"][tag] 

178 # Dataset name 

179 dq_t["dataset"] = toml["dataset"][tag] 

180 dq_t["site"] = toml["site"][tag] 

181 # GRL criteria 

182 dq_t["grl"] = toml["grl"][tag] 

183 # Defect tag 

184 dq_t["def_tag"] = toml["def_tag"][tag] 

185 

186 except KeyError: 

187 print("Problem in dataquality_tag.toml file") 

188 sys.exit() 

189 

190 # Retrieve the dataset description from the dataset.toml file 

191 try: 

192 toml_ds = read_configuration("dataset", source) 

193 

194 dq_t["dataset_descr"] = toml_ds["descr"][dq_t["dataset"]] 

195 dq_t["dataset_parent"] = toml_ds["parent"][dq_t["dataset"]] 

196 # Name of processing type / versions 

197 dq_t["proc"] = single_property_process( 

198 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["proc"] 

199 ) 

200 except KeyError: 

201 print("Problem in dataset configuration - dataset step") 

202 sys.exit() 

203 try: 

204 # List of detectors per site - Single string 

205 dq_t["det"] = all_site_process( 

206 dq_t["dataset"], 

207 dq_t["dataset_parent"], 

208 toml_ds["site_all_dets"], 

209 dq_t["site"].split(" ") 

210 ) 

211 for i_site in dq_t["det"]: 

212 if dq_t["det"][i_site] != "": 

213 dq_t["det"][i_site] = dq_t["det"][i_site].split(" ") 

214 else: 

215 dq_t["det"][i_site] = [] 

216 except KeyError: 

217 print("Problem in dataset configuration - detector step") 

218 sys.exit() 

219 try: 

220 # Run type - Detector dependant -> specific processing 

221 # Empty for v16 and v17 as did not contribute to the naming of file 

222 dq_t["run_type"] = all_dets_process( 

223 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["run_type"], dq_t["det"] 

224 ) 

225 # QAQC version - Detector dependant -> specific processing 

226 # ) 

227 dq_t["qaqc_version"] = all_dets_process( 

228 dq_t["dataset"], 

229 dq_t["dataset_parent"], 

230 toml_ds["qaqc_version"], 

231 dq_t["det"] 

232 ) 

233 # Compile the run type and qaqc version to get the filename 

234 dq_t["qaqc_name"] = {} 

235 for i_det in dq_t["run_type"]: 

236 dq_t["qaqc_name"][i_det] = ( 

237 f"{dq_t['qaqc_version'][i_det]}_{dq_t['run_type'][i_det]}" 

238 ) 

239 except KeyError: 

240 print("Problem in dataset configuration - type/version step") 

241 sys.exit() 

242 try: 

243 # Run range - Detector dependant -> specific processing 

244 dq_t["run_range"] = all_dets_process( 

245 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["run_range"], dq_t["det"] 

246 ) 

247 except KeyError: 

248 print("Problem in dataset configuration - run-range step") 

249 sys.exit() 

250 

251 # Retrieve the processing type/versions from the processing.toml file 

252 # Detector dependant -> specific processing defined in dataset 

253 try: 

254 toml_proc = read_configuration("processing", source) 

255 

256 dq_t["proc_descr"] = toml_proc["descr"][dq_t["proc"]] 

257 dq_t["proc_type"] = all_dets_process( 

258 dq_t["proc"], "", toml_proc["proc_type"], dq_t["det"] 

259 ) 

260 dq_t["proc_version"] = all_dets_process( 

261 dq_t["proc"], "", toml_proc["proc_version"], dq_t["det"] 

262 ) 

263 except KeyError: 

264 print("Problem in processing configuration - run-range step") 

265 sys.exit() 

266 

267 # Retrieve the good run list definition (name and definition) 

268 # The threshold retrieval is completed in the script configure_var_thresholds 

269 try: 

270 toml_grl = read_configuration("grl", source) 

271 

272 dq_t["grl_descr"] = toml_grl["descr"][dq_t["grl"]] 

273 grl_tag_parent = toml_grl["parent"][dq_t["grl"]] 

274 if grl_tag_parent == "": 

275 dq_t["grl_name"] = toml_grl["grl_name"][dq_t["grl"]] 

276 else: 

277 dq_t["grl_name"] = toml_grl["grl_name"][grl_tag_parent] 

278 for i_name in dq_t["grl_name"].keys(): 

279 if i_name in toml_grl["grl_name"][dq_t["grl"]].keys(): 

280 dq_t["grl_name"][i_name] = toml_grl["grl_name"][dq_t["grl"]][i_name] 

281 except KeyError: 

282 print("Problem in grl configuration") 

283 sys.exit() 

284 

285 return dq_t 

286 

287 

288############################################################################### 

289def configure_defect(source="sftp"): 

290 """ 

291 TOMl configuration of the defects 

292 

293 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/), 

294 either the directory absolute-path. 

295 """ 

296 toml = read_configuration("defect", source) 

297 

298 return { 

299 "descr": toml["descr"], 

300 "type": list(toml["descr"].keys()), 

301 "bit": toml["bit"] 

302 } 

303 

304 

305############################################################################### 

306def configure_def_var_name(v_prop): 

307 """Description of defect variables""" 

308 

309 # QAQC variables 

310 v_prop["description"]["def_daq"] = "DAQ defects" 

311 v_prop["description"]["def_operation"] = "Operation defects" 

312 v_prop["description"]["def_data_corruption"] = "Data corruption defects" 

313 v_prop["description"]["def_calibration"] = "Calibration defects" 

314 v_prop["description"]["def_data_processing"] = "Data processing defects" 

315 v_prop["description"]["def_analysis"] = "Analysis defects" 

316 v_prop["description"]["def_high_level_analysis"] = "Neutrino analysis defects" 

317 v_prop["description"]["def_signoff"] = "Missing signoff" 

318 

319 

320############################################################################### 

321def configure_fact(source="sftp"): 

322 """ 

323 TOMl configuration of the facts 

324 

325 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/), 

326 either the directory absolute-path. 

327 """ 

328 toml = read_configuration("fact", source) 

329 

330 return {"type": toml["type"]} 

331 

332 

333############################################################################### 

334def configure_det_fact(source="sftp"): 

335 """ 

336 TOMl configuration of the detector facts 

337 

338 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/), 

339 either the directory absolute-path. 

340 """ 

341 toml = read_configuration("det_fact", source) 

342 

343 return { 

344 "type": toml["type"], 

345 "status": toml["status"], 

346 } 

347 

348 

349############################################################################### 

350def configure_var_unit(v_prop): 

351 """Description of variables""" 

352 v_prop["unit"] = { 

353 "livetime": "s", 

354 "timestampdiff": "s", 

355 "triggerrate": "Hz", 

356 "meanrate": "Hz", 

357 "rmsrate": "Hz", 

358 } 

359 

360 

361############################################################################### 

362def configure_var_bit(v_prop): 

363 """ 

364 Bit of variables contributing to the veto//Qscore degradation 

365 used to store in the TTree 

366 """ 

367 v_prop["bit"] = { 

368 "livetime_s": 0, 

369 "timestampdiff": 1, 

370 "timestampdiff_r": 2, 

371 "triggerrate": 3, 

372 "HRV": 4, 

373 "DAQ": 5, 

374 "WR": 6, 

375 "FIFO": 7, 

376 "PMTs": 8, 

377 "pmts_norm": 9, 

378 "MEAN_Rate_Hz": 10, 

379 "RMS_Rate_Hz": 11, 

380 "out_sync": 12, 

381 "out_usync": 13, 

382 "JDAQJTProc": 14, 

383 "Acoustics": 15, 

384 "zero_AHRS": 16, 

385 "ahrs_per_mn": 17, 

386 "nb_of_meta": 18, 

387 "hrv_fifo_failures": 19, 

388 "duplic_timeslices": 20, 

389 "HV_check": 21, 

390 "def_daq": 23, 

391 "def_operation": 24, 

392 "def_data_corruption": 25, 

393 "def_calibration": 26, 

394 "def_data_processing": 27, 

395 "def_analysis": 28, 

396 "def_high_level_analysis": 29, 

397 "def_signoff": 30, 

398 } 

399 

400 

401############################################################################### 

402def configure_var_thresholds(v_prop, det, dqt, source="sftp"): 

403 """ 

404 Configure various variables for the analysis of the TTree derived from 

405 the JQAQC file 

406 TOMl configuration 

407 

408 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/), 

409 either the directory absolute-path. 

410 """ 

411 toml_grl = read_configuration("grl", source) 

412 toml_perf_aux = read_configuration("km3dq_perf_aux", source) 

413 

414 grl_tag_parent = toml_grl["parent"][dqt["grl"]] 

415 

416 if grl_tag_parent == "": 

417 v_prop["veto_thresholds"] = dict(toml_grl["veto_thresholds"][dqt["grl"]]) 

418 v_prop["qsco_thresholds"] = dict(toml_grl["qsco_thresholds"][dqt["grl"]]) 

419 v_prop["good_thresholds"] = dict(toml_grl["good_thresholds"][dqt["grl"]]) 

420 v_prop["poor_thresholds"] = dict(toml_grl["poor_thresholds"][dqt["grl"]]) 

421 v_prop["bad_thresholds"] = dict(toml_grl["bad_thresholds"][dqt["grl"]]) 

422 v_prop["discard_thresholds"] = dict(toml_grl["discard_thresholds"][dqt["grl"]]) 

423 else: 

424 v_prop["veto_thresholds"] = dict(toml_grl["veto_thresholds"][grl_tag_parent]) 

425 v_prop["qsco_thresholds"] = dict(toml_grl["qsco_thresholds"][grl_tag_parent]) 

426 v_prop["good_thresholds"] = dict(toml_grl["good_thresholds"][grl_tag_parent]) 

427 v_prop["poor_thresholds"] = dict(toml_grl["poor_thresholds"][grl_tag_parent]) 

428 v_prop["bad_thresholds"] = dict(toml_grl["bad_thresholds"][grl_tag_parent]) 

429 v_prop["discard_thresholds"] = dict( 

430 toml_grl["discard_thresholds"][grl_tag_parent] 

431 ) 

432 

433 for i_type in ( 

434 "veto_thresholds", 

435 "qsco_thresholds", 

436 "good_thresholds", 

437 "poor_thresholds", 

438 "bad_thresholds", 

439 "discard_thresholds", 

440 ): 

441 if dqt["grl"] in toml_grl[i_type].keys(): 

442 for i_modif in toml_grl[i_type][dqt["grl"]]: 

443 # If empty list, remove the thresholds 

444 # defined in quasi_online dqt['grl'] 

445 if len(toml_grl[i_type][dqt["grl"]][i_modif]) == 0: 

446 v_prop[i_type].pop(i_modif) 

447 continue 

448 # Otherwise update them 

449 v_prop[i_type][i_modif] = toml_grl[i_type][dqt["grl"]][i_modif] 

450 

451 # DUMMY detector used only by the get_grl_html_descr function 

452 # used by km3dq_perf/generatewww.py - No impact on perf 

453 if det == "DUMMY": 

454 return 

455 

456 # Detector dependant treatment 

457 try: 

458 # Display options - Default value 

459 v_prop["basic_plots_options"] = dict( 

460 toml_perf_aux["basic_plots_options"]["default"] 

461 ) 

462 v_prop["basic_plots_y_range_display"] = dict( 

463 toml_perf_aux["basic_plots_y_range_display"]["default"] 

464 ) 

465 # Default value overriden by detector specific values 

466 if "ORCA" in det: # Increased range for bioluminescence 

467 v_prop["basic_plots_y_range_display"]["HRV"] = [-0.1, 0.8] 

468 

469 try: 

470 for i_var, i_var_range in toml_perf_aux["basic_plots_y_range_display"][det].items(): 

471 v_prop["basic_plots_y_range_display"][i_var] = i_var_range 

472 except KeyError: 

473 pass 

474 

475 # PMTS treatment to be removed at some point?? 

476 v_prop["basic_plots_y_range_display"]["PMTs"][1] = ( 

477 get_detx_caract(det)["pmts"] * 1.2 

478 ) 

479 except KeyError: 

480 print("config_library.py / configure_var_thresholds: " 

481 "Problem in setting y-range display") 

482 

483 

484############################################################################### 

485def get_grl_html_descr(dq_tag_0): 

486 """ 

487 Returns the grl description in an html for the summary webpage 

488 The grl may depend on the detector due to missing variables. This is 

489 by-passed by passing a dummy value to the configure_var_thresholds 

490 function. In this way, no variable is discarded 

491 """ 

492 

493 var_prop = {} 

494 configure_var_thresholds(var_prop, "DUMMY", dq_tag_0) 

495 configure_var_unit(var_prop) 

496 html_descr = {} 

497 for i_key in var_prop.keys(): 

498 # One assumes that the QQC variables arrive before the defects 

499 # in the configuration file 

500 html_descr[i_key] = "<b>QAQC variables</b><br>" 

501 defect_found = False 

502 for i_var in var_prop[i_key].keys(): 

503 # qaqc variable 

504 if any( 

505 ( 

506 isinstance(var_prop[i_key][i_var][0], int), 

507 isinstance(var_prop[i_key][i_var][0], float), 

508 ) 

509 ): 

510 if var_prop[i_key][i_var][0] == var_prop[i_key][i_var][1]: 

511 thresh = f"{var_prop[i_key][i_var][0]}" 

512 if i_var == "veto": 

513 thresh = f"{var_prop[i_key][i_var][0] == 1}" 

514 else: 

515 thresh = ( 

516 f"{var_prop[i_key][i_var][0]} &#8212 " 

517 f"{var_prop[i_key][i_var][1]}" 

518 ) 

519 if i_var in var_prop["unit"].keys(): 

520 thresh += f" {var_prop['unit'][i_var]}" 

521 

522 html_descr[i_key] += f"{i_var}: {thresh}" 

523 # defect 

524 elif isinstance(var_prop[i_key][i_var][0], str): 

525 if defect_found is False: 

526 html_descr[i_key] += "<hr><b>Defects</b><br>" 

527 defect_found = True 

528 html_descr[i_key] += ( 

529 f"{i_var.replace('def_', '')}: " f"{var_prop[i_key][i_var][0]}" 

530 ) 

531 for j in range(1, len(var_prop[i_key][i_var])): 

532 html_descr[i_key] += f", {var_prop[i_key][i_var][j]}" 

533 

534 html_descr[i_key] += "<br>" 

535 

536 return html_descr