Coverage for src/km3dq_common/config_library.py: 62%

211 statements  

« prev     ^ index     » next       coverage.py v7.10.7, created at 2025-10-16 11:32 +0000

1#! /usr/bin/env python 

2############################################################################### 

3# Definition of QAQC variable, defects, data-quality tags... 

4# 

5# Developer: Benjamin Trocme (benjamin.trocme at apc.in2p3.fr) - 2023 

6 

7import tomli 

8import sys 

9import urllib.request 

10import atexit 

11 

12if sys.version_info < (3, 9): 

13 import importlib_resources 

14 

15 try: 

16 from importlib_resources import as_file 

17 except ImportError: 

18 from importlib_resources.trees import as_file 

19else: 

20 import importlib.resources as importlib_resources 

21 from importlib.resources import as_file 

22 

23try: 

24 from contextlib import ExitStack 

25except ImportError: 

26 from contextlib2 import ExitStack 

27 

28 

29############################################################################### 

30def read_configuration(filename, source, raise_missing=True): 

31 """ 

32 """ 

33 

34 if source == "sftp": 

35 with urllib.request.urlopen( 

36 f"https://sftp.km3net.de/data/km3dq_lw_db/Common/{filename}.toml" 

37 ) as s_f: 

38 return tomli.loads(s_f.read().decode("utf-8")) 

39 else: 

40 ref = importlib_resources.files("km3dq_common") / f"{dir_absolute_path}/{filename}.toml" 

41 file_manager = ExitStack() 

42 atexit.register(file_manager.close) 

43 file_path = file_manager.enter_context(as_file(ref)) 

44 if raise_missing and not file_path.exists(): 

45 raise RuntimeError(f"Unknown or missing file: {filename}") 

46 return tomli.load(file_path) 

47 

48 

49############################################################################### 

50def get_detx_caract(det, source="sftp"): 

51 """ 

52 

53 Return the number of DOMs (incl base module so far) and PMts 

54 

55 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/), 

56 either the directory absolute-path. 

57 """ 

58 

59 toml = read_configuration("km3dq_perf_aux", source) 

60 

61 return toml["hardware_charact"][det] 

62 

63 

64############################################################################### 

65def configure_var_name(v_prop): 

66 """Description of JQAQC variables""" 

67 

68 # QAQC variables 

69 v_prop["description"] = { 

70 "run": "Run", 

71 "livetime_s": "Livetime (s)", 

72 "nb_of_meta": "Nb of metadata", 

73 "timestampdiff": "Missing time-slices", 

74 "timestampdiff_r": ("Missing time-slices" "/Run duration"), 

75 "triggerrate": "Trigger rate(Hz)", 

76 "HRV": "High rate veto bit", 

77 "DAQ": "Fraction of data without UDP packet-loss", 

78 "WR": "White rabbit status", 

79 "FIFO": "FIFO almost full", 

80 "PMTs": "Number of active PMTs", 

81 "pmts_norm": "Fraction of active PMTs", 

82 "MEAN_Rate_Hz": "Mean rate (Hz)", 

83 "RMS_Rate_Hz": "RMS rate (Hz)", 

84 "hrv_fifo_failures": "Number of hrv/fifo failures", 

85 "duplic_timeslices": "Number of duplicated timeslices", 

86 "out_sync": "Out-of-sync", 

87 "out_usync": "Micro out-of-sync", 

88 "JDAQJTProc": "Triggered != Retriggered", 

89 "Acoustics": "Number of acoustic events", 

90 "Acoustics_per_mn": "Number of acoustic events per minute", 

91 "zero_AHRS": ("Number of DOMs without valid " "AHRS data"), 

92 "ahrs_per_mn": ("Number of AHRS event per minute " "in > 80% of DOMs"), 

93 "JDAQEvent": "Total", 

94 "JTrigger3DShower": "3D shower trigger", 

95 "JTriggerMXShower": "MX shower trigger", 

96 "JTrigger3DMuon": "3D Muon trigger ", 

97 "HV_check": "High-voltage inconsistencies", 

98 } 

99 

100 

101############################################################################### 

102def configure_dataquality_tag(tag, source="sftp"): 

103 """ 

104 TOMl configuration of the data quality tags 

105 === Arguments === 

106 - tag : tag name - [string] - Ex: "default", "calibration"... 

107 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/), 

108 either the directory absolute-path. 

109 === Output === 

110 - Dictionnary with all tag characteristics for all detectors 

111 """ 

112 

113 #################################################### 

114 def single_property_process(tag, tag_parent, toml_input): 

115 output = {} 

116 

117 for i_tag in (tag_parent, tag): 

118 if i_tag in toml_input.keys(): 

119 output = toml_input[i_tag] 

120 

121 return output 

122 

123 #################################################### 

124 def all_site_process(tag, tag_parent, toml_input, sites): 

125 output = {"ARCA": "", "ORCA": ""} 

126 

127 for i_tag in (tag_parent, tag): 

128 if i_tag == "": # No tag_parent 

129 continue 

130 for i_site in sites: 

131 if i_site in toml_input[i_tag].keys(): 

132 output[i_site] = toml_input[i_tag][i_site] 

133 

134 return output 

135 

136 #################################################### 

137 def all_dets_process(tag, tag_parent, toml_input, det_list): 

138 # det_list: detector list 

139 output = {} 

140 

141 for i_tag in (tag_parent, tag): 

142 if i_tag == "": # No tag_parent 

143 continue 

144 for i_site in ["ARCA", "ORCA"]: 

145 for i_det in det_list[i_site]: 

146 if i_tag in toml_input.keys(): 

147 if i_det in toml_input[i_tag]: # Detector explicitly defined 

148 output[i_det] = toml_input[i_tag][i_det] 

149 elif i_site in toml_input[i_tag]: # Use the site definition 

150 output[i_det] = toml_input[i_tag][i_site] 

151 

152 return output 

153 

154 #################################################### 

155 # First retrieve the general caracteristics: 

156 # status, description, dataset name, grl... 

157 try: 

158 toml = read_configuration("dataquality_tag", source) 

159 

160 dq_t = {} 

161 # The path directory is the name of the data-quality tag 

162 dq_t["dir_path"] = tag 

163 # Retrieve the general descriptions 

164 dq_t["name"] = toml["short_descr"][tag] 

165 dq_t["descr"] = toml["descr"][tag] 

166 dq_t["status"] = toml["status"][tag] 

167 # Dataset name 

168 dq_t["dataset"] = toml["dataset"][tag] 

169 dq_t["site"] = toml["site"][tag] 

170 # GRL criteria 

171 dq_t["grl"] = toml["grl"][tag] 

172 # Defect tag 

173 dq_t["def_tag"] = toml["def_tag"][tag] 

174 

175 except: 

176 print("Problem in dataquality_tag.toml file") 

177 sys.exit() 

178 

179 # Retrieve the dataset description from the dataset.toml file 

180 try: 

181 toml_ds = read_configuration("dataset", source) 

182 

183 dq_t["dataset_descr"] = toml_ds["descr"][dq_t["dataset"]] 

184 dq_t["dataset_parent"] = toml_ds["parent"][dq_t["dataset"]] 

185 # Name of processing type / versions 

186 dq_t["proc"] = single_property_process( 

187 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["proc"] 

188 ) 

189 except KeyError: 

190 print("Problem in dataset configuration - dataset step") 

191 sys.exit() 

192 try: 

193 # List of detectors per site - Single string 

194 dq_t["det"] = all_site_process( 

195 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["site_all_dets"], dq_t["site"].split(" ") 

196 ) 

197 for i_site in dq_t["det"]: 

198 if dq_t["det"][i_site] != "": 

199 dq_t["det"][i_site] = dq_t["det"][i_site].split(" ") 

200 else: 

201 dq_t["det"][i_site] = [] 

202 except KeyError: 

203 print("Problem in dataset configuration - detector step") 

204 sys.exit() 

205 try: 

206 # Run type - Detector dependant -> specific processing 

207 # Empty for v16 and v17 as did not contribute to the naming of file 

208 dq_t["run_type"] = all_dets_process( 

209 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["run_type"], dq_t["det"] 

210 ) 

211 # QAQC version - Detector dependant -> specific processing 

212 # ) 

213 dq_t["qaqc_version"] = all_dets_process( 

214 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["qaqc_version"], dq_t["det"] 

215 ) 

216 # Compile the run type and qaqc version to get the filename 

217 dq_t["qaqc_name"] = {} 

218 for i_det in dq_t["run_type"]: 

219 dq_t["qaqc_name"][i_det] = f"{dq_t['qaqc_version'][i_det]}_{dq_t['run_type'][i_det]}" 

220 except KeyError: 

221 print("Problem in dataset configuration - type/version step") 

222 sys.exit() 

223 try: 

224 # Run range - Detector dependant -> specific processing 

225 dq_t["run_range"] = all_dets_process( 

226 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["run_range"], dq_t["det"] 

227 ) 

228 except: 

229 print("Problem in dataset configuration - run-range step") 

230 sys.exit() 

231 

232 # Retrieve the processing type/versions from the processing.toml file 

233 # Detector dependant -> specific processing defined in dataset 

234 try: 

235 toml_proc = read_configuration("processing", source) 

236 

237 dq_t["proc_descr"] = toml_proc["descr"][dq_t["proc"]] 

238 dq_t["proc_type"] = all_dets_process( 

239 dq_t["proc"], "", toml_proc["proc_type"], dq_t["det"] 

240 ) 

241 dq_t["proc_version"] = all_dets_process( 

242 dq_t["proc"], "", toml_proc["proc_version"], dq_t["det"] 

243 ) 

244 except: 

245 sys.exit() 

246 

247 # Retrieve the good run list definition (name and definition) 

248 # The threshold retrieval is completed in the script configure_var_thresholds 

249 try: 

250 toml_grl = read_configuration("grl", source) 

251 

252 dq_t["grl_descr"] = toml_grl["descr"][dq_t["grl"]] 

253 grl_tag_parent = toml_grl["parent"][dq_t["grl"]] 

254 if grl_tag_parent == "": 

255 dq_t["grl_name"] = toml_grl["grl_name"][dq_t["grl"]] 

256 else: 

257 dq_t["grl_name"] = toml_grl["grl_name"][grl_tag_parent] 

258 for i_name in dq_t["grl_name"].keys(): 

259 if i_name in toml_grl["grl_name"][dq_t["grl"]].keys(): 

260 dq_t["grl_name"][i_name] = toml_grl["grl_name"][dq_t["grl"]][i_name] 

261 

262 except: 

263 print("Problem in grl configuration") 

264 sys.exit() 

265 

266 return dq_t 

267 

268 

269############################################################################### 

270def configure_defect(source="sftp"): 

271 """ 

272 TOMl configuration of the defects 

273  

274 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/), 

275 either the directory absolute-path. 

276 """ 

277 toml = read_configuration("defect", source) 

278 

279 return { 

280 "descr": toml["descr"], 

281 "type": list(toml["descr"].keys()), 

282 "bit": toml["bit"] 

283 } 

284 

285 

286############################################################################### 

287def configure_def_var_name(v_prop): 

288 """Description of defect variables""" 

289 

290 # QAQC variables 

291 v_prop["description"]["def_daq"] = "DAQ defects" 

292 v_prop["description"]["def_operation"] = "Operation defects" 

293 v_prop["description"]["def_data_corruption"] = "Data corruption defects" 

294 v_prop["description"]["def_calibration"] = "Calibration defects" 

295 v_prop["description"]["def_data_processing"] = "Data processing defects" 

296 v_prop["description"]["def_analysis"] = "Analysis defects" 

297 v_prop["description"]["def_high_level_analysis"] = "Neutrino analysis defects" 

298 v_prop["description"]["def_signoff"] = "Missing signoff" 

299 

300 

301############################################################################### 

302def configure_fact(source="sftp"): 

303 """ 

304 TOMl configuration of the facts 

305  

306 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/), 

307 either the directory absolute-path. 

308 """ 

309 toml = read_configuration("fact", source) 

310 

311 return {"type": toml["type"]} 

312 

313 

314############################################################################### 

315def configure_det_fact(source="sftp"): 

316 """ 

317 TOMl configuration of the detector facts 

318 

319 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/), 

320 either the directory absolute-path. 

321 """ 

322 toml = read_configuration("det_fact", source) 

323 

324 return { 

325 "type": toml["type"], 

326 "status": toml["status"], 

327 } 

328 

329 

330############################################################################### 

331def configure_var_unit(v_prop): 

332 """Description of variables""" 

333 v_prop["unit"] = { 

334 "livetime": "s", 

335 "timestampdiff": "s", 

336 "triggerrate": "Hz", 

337 "meanrate": "Hz", 

338 "rmsrate": "Hz", 

339 } 

340 

341 

342############################################################################### 

343def configure_var_bit(v_prop): 

344 """ 

345 Bit of variables contributing to the veto//Qscore degradation 

346 used to store in the TTree 

347 """ 

348 v_prop["bit"] = { 

349 "livetime_s": 0, 

350 "timestampdiff": 1, 

351 "timestampdiff_r": 2, 

352 "triggerrate": 3, 

353 "HRV": 4, 

354 "DAQ": 5, 

355 "WR": 6, 

356 "FIFO": 7, 

357 "PMTs": 8, 

358 "pmts_norm": 9, 

359 "MEAN_Rate_Hz": 10, 

360 "RMS_Rate_Hz": 11, 

361 "out_sync": 12, 

362 "out_usync": 13, 

363 "JDAQJTProc": 14, 

364 "Acoustics": 15, 

365 "zero_AHRS": 16, 

366 "ahrs_per_mn": 17, 

367 "nb_of_meta": 18, 

368 "hrv_fifo_failures": 19, 

369 "duplic_timeslices": 20, 

370 "HV_check": 21, 

371 "def_daq": 23, 

372 "def_operation": 24, 

373 "def_data_corruption": 25, 

374 "def_calibration": 26, 

375 "def_data_processing": 27, 

376 "def_analysis": 28, 

377 "def_high_level_analysis": 29, 

378 "def_signoff": 30, 

379 } 

380 

381 

382############################################################################### 

383def configure_var_thresholds(v_prop, det, dqt, source="sftp"): 

384 """ 

385 Configure various variables for the analysis of the TTree derived from 

386 the JQAQC file 

387 TOMl configuration 

388 

389 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/), 

390 either the directory absolute-path. 

391 """ 

392 toml_grl = read_configuration("grl", source) 

393 toml_perf_aux = read_configuration("km3dq_perf_aux", source) 

394 

395 grl_tag_parent = toml_grl["parent"][dqt["grl"]] 

396 

397 if grl_tag_parent == "": 

398 v_prop["veto_thresholds"] = dict(toml_grl["veto_thresholds"][dqt["grl"]]) 

399 v_prop["qsco_thresholds"] = dict(toml_grl["qsco_thresholds"][dqt["grl"]]) 

400 v_prop["good_thresholds"] = dict(toml_grl["good_thresholds"][dqt["grl"]]) 

401 v_prop["poor_thresholds"] = dict(toml_grl["poor_thresholds"][dqt["grl"]]) 

402 v_prop["bad_thresholds"] = dict(toml_grl["bad_thresholds"][dqt["grl"]]) 

403 v_prop["discard_thresholds"] = dict(toml_grl["discard_thresholds"][dqt["grl"]]) 

404 else: 

405 v_prop["veto_thresholds"] = dict(toml_grl["veto_thresholds"][grl_tag_parent]) 

406 v_prop["qsco_thresholds"] = dict(toml_grl["qsco_thresholds"][grl_tag_parent]) 

407 v_prop["good_thresholds"] = dict(toml_grl["good_thresholds"][grl_tag_parent]) 

408 v_prop["poor_thresholds"] = dict(toml_grl["poor_thresholds"][grl_tag_parent]) 

409 v_prop["bad_thresholds"] = dict(toml_grl["bad_thresholds"][grl_tag_parent]) 

410 v_prop["discard_thresholds"] = dict( 

411 toml_grl["discard_thresholds"][grl_tag_parent] 

412 ) 

413 

414 for i_type in ( 

415 "veto_thresholds", 

416 "qsco_thresholds", 

417 "good_thresholds", 

418 "poor_thresholds", 

419 "bad_thresholds", 

420 "discard_thresholds", 

421 ): 

422 if dqt["grl"] in toml_grl[i_type].keys(): 

423 for i_modif in toml_grl[i_type][dqt["grl"]]: 

424 # If empty list, remove the thresholds defined in quasi_online dqt['grl'] 

425 if len(toml_grl[i_type][dqt["grl"]][i_modif]) == 0: 

426 v_prop[i_type].pop(i_modif) 

427 continue 

428 # Otherwise update them 

429 v_prop[i_type][i_modif] = toml_grl[i_type][dqt["grl"]][i_modif] 

430 

431 # DUMMY detector used only by the get_grl_html_descr function 

432 # used by km3dq_perf/generatewww.py - No impact on perf 

433 if det == "DUMMY": 

434 return 

435 

436 # Detector dependant treatment 

437 try: 

438 # Display options - Not DQ tag dependant (as of today but could be changed) 

439 v_prop["basic_plots_options"] = dict( 

440 toml_perf_aux["basic_plots_options"]["quasi_online"] 

441 ) 

442 v_prop["basic_plots_y_range_display"] = dict( 

443 toml_perf_aux["basic_plots_y_range_display"]["quasi_online"] 

444 ) 

445 v_prop["basic_plots_y_range_display"]["PMTs"][1] = ( 

446 get_detx_caract(det)["pmts"] * 1.2 

447 ) 

448 if "ORCA" in det: # Increased range for bioluminescence 

449 v_prop["basic_plots_y_range_display"]["HRV"] = [-0.1, 0.8] 

450 except KeyError: 

451 print(f"config_library.py / configure_var_thresholds: Missing DQ-tag information for the detector {det}") 

452 print(f"QAQC version: {dqt['qaqc_version'].keys()}") 

453 

454############################################################################### 

455def get_grl_html_descr(dq_tag_0): 

456 """ 

457 Returns the grl description in an html for the summary webpage 

458 The grl may depend on the detector due to missing variables. This is 

459 by-passed by passing a dummy value to the configure_var_thresholds 

460 function. In this way, no variable is discarded 

461 """ 

462 

463 var_prop = {} 

464 configure_var_thresholds(var_prop, "DUMMY", dq_tag_0) 

465 configure_var_unit(var_prop) 

466 html_descr = {} 

467 for i_key in var_prop.keys(): 

468 # One assumes that the QQC variables arrive before the defects 

469 # in the configuration file 

470 html_descr[i_key] = "<b>QAQC variables</b><br>" 

471 defect_found = False 

472 for i_var in var_prop[i_key].keys(): 

473 # qaqc variable 

474 if any( 

475 ( 

476 isinstance(var_prop[i_key][i_var][0], int), 

477 isinstance(var_prop[i_key][i_var][0], float), 

478 ) 

479 ): 

480 if var_prop[i_key][i_var][0] == var_prop[i_key][i_var][1]: 

481 thresh = f"{var_prop[i_key][i_var][0]}" 

482 if i_var == "veto": 

483 thresh = f"{var_prop[i_key][i_var][0] == 1}" 

484 else: 

485 thresh = ( 

486 f"{var_prop[i_key][i_var][0]} &#8212 " 

487 f"{var_prop[i_key][i_var][1]}" 

488 ) 

489 if i_var in var_prop["unit"].keys(): 

490 thresh += f" {var_prop['unit'][i_var]}" 

491 

492 html_descr[i_key] += f"{i_var}: {thresh}" 

493 # defect 

494 elif isinstance(var_prop[i_key][i_var][0], str): 

495 if defect_found is False: 

496 html_descr[i_key] += "<hr><b>Defects</b><br>" 

497 defect_found = True 

498 html_descr[i_key] += ( 

499 f"{i_var.replace('def_', '')}: " f"{var_prop[i_key][i_var][0]}" 

500 ) 

501 for j in range(1, len(var_prop[i_key][i_var])): 

502 html_descr[i_key] += f", {var_prop[i_key][i_var][j]}" 

503 

504 html_descr[i_key] += "<br>" 

505 

506 return html_descr