Coverage for src/km3dq_common/config_library.py: 64%

199 statements  

« prev     ^ index     » next       coverage.py v7.9.1, created at 2025-06-16 14:13 +0000

1#! /usr/bin/env python 

2############################################################################### 

3# Definition of QAQC variable, defects, data-quality tags... 

4# 

5# Developer: Benjamin Trocme (benjamin.trocme at apc.in2p3.fr) - 2023 

6 

7import tomli 

8import sys 

9import urllib.request 

10import atexit 

11 

12if sys.version_info < (3, 9): 

13 import importlib_resources 

14 

15 try: 

16 from importlib_resources import as_file 

17 except ImportError: 

18 from importlib_resources.trees import as_file 

19else: 

20 import importlib.resources as importlib_resources 

21 from importlib.resources import as_file 

22 

23try: 

24 from contextlib import ExitStack 

25except ImportError: 

26 from contextlib2 import ExitStack 

27 

28 

29############################################################################### 

30def read_configuration(filename, source, raise_missing=True): 

31 """ 

32 """ 

33 

34 if source == "sftp": 

35 with urllib.request.urlopen( 

36 f"https://sftp.km3net.de/data/km3dq_lw_db/Common/{filename}.toml" 

37 ) as s_f: 

38 return tomli.loads(s_f.read().decode("utf-8")) 

39 else: 

40 ref = importlib_resources.files("km3dq_common") / f"{dir_absolute_path}/{filename}.toml" 

41 file_manager = ExitStack() 

42 atexit.register(file_manager.close) 

43 file_path = file_manager.enter_context(as_file(ref)) 

44 if raise_missing and not file_path.exists(): 

45 raise RuntimeError(f"Unknown or missing file: {filename}") 

46 return tomli.load(file_path) 

47 

48 

49############################################################################### 

50def get_detx_caract(det, source="sftp"): 

51 """ 

52 

53 Return the number of DOMs (incl base module so far) and PMts 

54 

55 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/), 

56 either the directory absolute-path. 

57 """ 

58 

59 toml = read_configuration("km3dq_perf_aux", source) 

60 

61 return toml["hardware_charact"][det] 

62 

63 

64############################################################################### 

65def configure_var_name(v_prop): 

66 """Description of JQAQC variables""" 

67 

68 # QAQC variables 

69 v_prop["description"] = { 

70 "run": "Run", 

71 "livetime_s": "Livetime (s)", 

72 "nb_of_meta": "Nb of metadata", 

73 "timestampdiff": "Missing time-slices", 

74 "timestampdiff_r": ("Missing time-slices" "/Run duration"), 

75 "triggerrate": "Trigger rate(Hz)", 

76 "HRV": "High rate veto bit", 

77 "DAQ": "Fraction of data without UDP packet-loss", 

78 "WR": "White rabbit status", 

79 "FIFO": "FIFO almost full", 

80 "PMTs": "Number of active PMTs", 

81 "pmts_norm": "Fraction of active PMTs", 

82 "MEAN_Rate_Hz": "Mean rate (Hz)", 

83 "RMS_Rate_Hz": "RMS rate (Hz)", 

84 "hrv_fifo_failures": "Number of hrv/fifo failures", 

85 "duplic_timeslices": "Number of duplicated timeslices", 

86 "out_sync": "Out-of-sync", 

87 "out_usync": "Micro out-of-sync", 

88 "JDAQJTProc": "Triggered != Retriggered", 

89 "Acoustics": "Number of acoustic events", 

90 "Acoustics_per_mn": "Number of acoustic events per minute", 

91 "zero_AHRS": ("Number of DOMs without valid " "AHRS data"), 

92 "ahrs_per_mn": ("Number of AHRS event per minute " "in > 80% of DOMs"), 

93 "JDAQEvent": "Total", 

94 "JTrigger3DShower": "3D shower trigger", 

95 "JTriggerMXShower": "MX shower trigger", 

96 "JTrigger3DMuon": "3D Muon trigger ", 

97 "hv_inconsist": "High-voltage inconsistencies", 

98 } 

99 

100 

101############################################################################### 

102def configure_dataquality_tag(tag, source="sftp"): 

103 """ 

104 TOMl configuration of the data quality tags 

105 === Arguments === 

106 - tag : tag name - [string] - Ex: "default", "calibration"... 

107 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/), 

108 either the directory absolute-path. 

109 === Output === 

110 - Dictionnary with all tag characteristics for all detectors 

111 """ 

112 

113 #################################################### 

114 def single_property_process(tag, tag_parent, toml_input): 

115 output = {} 

116 

117 for i_tag in (tag_parent, tag): 

118 if i_tag in toml_input.keys(): 

119 output = toml_input[i_tag] 

120 

121 return output 

122 

123 #################################################### 

124 def all_site_process(tag, tag_parent, toml_input, sites): 

125 output = {"ARCA": "", "ORCA": ""} 

126 

127 for i_tag in (tag_parent, tag): 

128 if i_tag == "": # No tag_parent 

129 continue 

130 for i_site in sites: 

131 if i_site in toml_input[i_tag].keys(): 

132 output[i_site] = toml_input[i_tag][i_site] 

133 

134 return output 

135 

136 #################################################### 

137 def all_dets_process(tag, tag_parent, toml_input, det_list): 

138 # det_list: detector list 

139 output = {} 

140 

141 for i_tag in (tag_parent, tag): 

142 if i_tag == "": # No tag_parent 

143 continue 

144 for i_site in ["ARCA", "ORCA"]: 

145 for i_det in det_list[i_site]: 

146 if i_tag in toml_input.keys(): 

147 if i_det in toml_input[i_tag]: # Detector explicitly defined 

148 output[i_det] = toml_input[i_tag][i_det] 

149 elif i_site in toml_input[i_tag]: # Use the site definition 

150 output[i_det] = toml_input[i_tag][i_site] 

151 

152 return output 

153 

154 #################################################### 

155 # First retrieve the general caracteristics: 

156 # status, description, dataset name, grl... 

157 try: 

158 toml = read_configuration("dataquality_tag", source) 

159 

160 dq_t = {} 

161 # The path directory is the name of the data-quality tag 

162 dq_t["dir_path"] = tag 

163 # Retrieve the general descriptions 

164 dq_t["name"] = toml["short_descr"][tag] 

165 dq_t["descr"] = toml["descr"][tag] 

166 dq_t["status"] = toml["status"][tag] 

167 # Dataset name 

168 dq_t["dataset"] = toml["dataset"][tag] 

169 dq_t["site"] = toml["site"][tag] 

170 # GRL criteria 

171 dq_t["grl"] = toml["grl"][tag] 

172 # Defect tag 

173 dq_t["def_tag"] = toml["def_tag"][tag] 

174 

175 except: 

176 print("Problem in dataquality_tag.toml file") 

177 sys.exit() 

178 

179 # Retrieve the dataset description from the dataset.toml file 

180 try: 

181 toml_ds = read_configuration("dataset", source) 

182 

183 dq_t["dataset_descr"] = toml_ds["descr"][dq_t["dataset"]] 

184 dq_t["dataset_parent"] = toml_ds["parent"][dq_t["dataset"]] 

185 # Name of processing type / versions 

186 dq_t["proc"] = single_property_process( 

187 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["proc"] 

188 ) 

189 # List of detectors per site - Single string 

190 dq_t["det"] = all_site_process( 

191 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["site_all_dets"], dq_t["site"].split(" ") 

192 ) 

193 for i_site in dq_t["det"]: 

194 if dq_t["det"][i_site] != "": 

195 dq_t["det"][i_site] = dq_t["det"][i_site].split(" ") 

196 else: 

197 dq_t["det"][i_site] = [] 

198 # Run type - Detector dependant -> specific processing 

199 # Empty for v16 and v17 as did not contribute to the naming of file 

200 dq_t["run_type"] = all_dets_process( 

201 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["run_type"], dq_t["det"] 

202 ) 

203 # QAQC version - Detector dependant -> specific processing 

204 # ) 

205 dq_t["qaqc_version"] = all_dets_process( 

206 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["qaqc_version"], dq_t["det"] 

207 ) 

208 # Compile the run type and qaqc version to get the filename 

209 dq_t["qaqc_name"] = {} 

210 for i_det in dq_t["run_type"]: 

211 dq_t["qaqc_name"][i_det] = f"{dq_t['qaqc_version'][i_det]}_{dq_t['run_type'][i_det]}" 

212 

213 # Run range - Detector dependant -> specific processing 

214 dq_t["run_range"] = all_dets_process( 

215 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["run_range"], dq_t["det"] 

216 ) 

217 

218 except: 

219 print("Problem in dataset configuration") 

220 sys.exit() 

221 

222 # Retrieve the processing type/versions from the processing.toml file 

223 # Detector dependant -> specific processing defined in dataset 

224 try: 

225 toml_proc = read_configuration("processing", source) 

226 

227 dq_t["proc_descr"] = toml_proc["descr"][dq_t["proc"]] 

228 dq_t["proc_type"] = all_dets_process( 

229 dq_t["proc"], "", toml_proc["proc_type"], dq_t["det"] 

230 ) 

231 dq_t["proc_version"] = all_dets_process( 

232 dq_t["proc"], "", toml_proc["proc_version"], dq_t["det"] 

233 ) 

234 except: 

235 sys.exit() 

236 

237 # Retrieve the good run list definition (name and definition) 

238 # The threshold retrieval is completed in the script configure_var_thresholds 

239 try: 

240 toml_grl = read_configuration("grl", source) 

241 

242 dq_t["grl_descr"] = toml_grl["descr"][dq_t["grl"]] 

243 grl_tag_parent = toml_grl["parent"][dq_t["grl"]] 

244 if grl_tag_parent == "": 

245 dq_t["grl_name"] = toml_grl["grl_name"][dq_t["grl"]] 

246 else: 

247 dq_t["grl_name"] = toml_grl["grl_name"][grl_tag_parent] 

248 for i_name in dq_t["grl_name"].keys(): 

249 if i_name in toml_grl["grl_name"][dq_t["grl"]].keys(): 

250 dq_t["grl_name"][i_name] = toml_grl["grl_name"][dq_t["grl"]][i_name] 

251 

252 except: 

253 print("Problem in grl configuration") 

254 sys.exit() 

255 

256 return dq_t 

257 

258 

259############################################################################### 

260def configure_defect(source="sftp"): 

261 """ 

262 TOMl configuration of the defects 

263  

264 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/), 

265 either the directory absolute-path. 

266 """ 

267 toml = read_configuration("defect", source) 

268 

269 return { 

270 "descr": toml["descr"], 

271 "type": list(toml["descr"].keys()), 

272 "bit": toml["bit"] 

273 } 

274 

275 

276############################################################################### 

277def configure_def_var_name(v_prop): 

278 """Description of defect variables""" 

279 

280 # QAQC variables 

281 v_prop["description"]["def_daq"] = "DAQ defects" 

282 v_prop["description"]["def_operation"] = "Operation defects" 

283 v_prop["description"]["def_data_corruption"] = "Data corruption defects" 

284 v_prop["description"]["def_calibration"] = "Calibration defects" 

285 v_prop["description"]["def_data_processing"] = "Data processing defects" 

286 v_prop["description"]["def_analysis"] = "Analysis defects" 

287 v_prop["description"]["def_high_level_analysis"] = "Neutrino analysis defects" 

288 v_prop["description"]["def_signoff"] = "Missing signoff" 

289 

290 

291############################################################################### 

292def configure_fact(source="sftp"): 

293 """ 

294 TOMl configuration of the facts 

295  

296 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/), 

297 either the directory absolute-path. 

298 """ 

299 toml = read_configuration("fact", source) 

300 

301 return {"type": toml["type"]} 

302 

303 

304############################################################################### 

305def configure_det_fact(source="sftp"): 

306 """ 

307 TOMl configuration of the detector facts 

308 

309 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/), 

310 either the directory absolute-path. 

311 """ 

312 toml = read_configuration("det_fact", source) 

313 

314 return { 

315 "type": toml["type"], 

316 "status": toml["status"], 

317 } 

318 

319 

320############################################################################### 

321def configure_var_unit(v_prop): 

322 """Description of variables""" 

323 v_prop["unit"] = { 

324 "livetime": "s", 

325 "timestampdiff": "s", 

326 "triggerrate": "Hz", 

327 "meanrate": "Hz", 

328 "rmsrate": "Hz", 

329 } 

330 

331 

332############################################################################### 

333def configure_var_bit(v_prop): 

334 """ 

335 Bit of variables contributing to the veto//Qscore degradation 

336 used to store in the TTree 

337 """ 

338 v_prop["bit"] = { 

339 "livetime_s": 0, 

340 "timestampdiff": 1, 

341 "timestampdiff_r": 2, 

342 "triggerrate": 3, 

343 "HRV": 4, 

344 "DAQ": 5, 

345 "WR": 6, 

346 "FIFO": 7, 

347 "PMTs": 8, 

348 "pmts_norm": 9, 

349 "MEAN_Rate_Hz": 10, 

350 "RMS_Rate_Hz": 11, 

351 "out_sync": 12, 

352 "out_usync": 13, 

353 "JDAQJTProc": 14, 

354 "Acoustics": 15, 

355 "zero_AHRS": 16, 

356 "ahrs_per_mn": 17, 

357 "nb_of_meta": 18, 

358 "hrv_fifo_failures": 19, 

359 "duplic_timeslices": 20, 

360 "def_daq": 23, 

361 "def_operation": 24, 

362 "def_data_corruption": 25, 

363 "def_calibration": 26, 

364 "def_data_processing": 27, 

365 "def_analysis": 28, 

366 "def_high_level_analysis": 29, 

367 "def_signoff": 30, 

368 } 

369 

370 

371############################################################################### 

372def configure_var_thresholds(v_prop, det, dqt, source="sftp"): 

373 """ 

374 Configure various variables for the analysis of the TTree derived from 

375 the JQAQC file 

376 TOMl configuration 

377 

378 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/), 

379 either the directory absolute-path. 

380 """ 

381 toml_grl = read_configuration("grl", source) 

382 toml_perf_aux = read_configuration("km3dq_perf_aux", source) 

383 

384 grl_tag_parent = toml_grl["parent"][dqt["grl"]] 

385 

386 if grl_tag_parent == "": 

387 v_prop["veto_thresholds"] = dict(toml_grl["veto_thresholds"][dqt["grl"]]) 

388 v_prop["qsco_thresholds"] = dict(toml_grl["qsco_thresholds"][dqt["grl"]]) 

389 v_prop["good_thresholds"] = dict(toml_grl["good_thresholds"][dqt["grl"]]) 

390 v_prop["poor_thresholds"] = dict(toml_grl["poor_thresholds"][dqt["grl"]]) 

391 v_prop["bad_thresholds"] = dict(toml_grl["bad_thresholds"][dqt["grl"]]) 

392 v_prop["discard_thresholds"] = dict(toml_grl["discard_thresholds"][dqt["grl"]]) 

393 else: 

394 v_prop["veto_thresholds"] = dict(toml_grl["veto_thresholds"][grl_tag_parent]) 

395 v_prop["qsco_thresholds"] = dict(toml_grl["qsco_thresholds"][grl_tag_parent]) 

396 v_prop["good_thresholds"] = dict(toml_grl["good_thresholds"][grl_tag_parent]) 

397 v_prop["poor_thresholds"] = dict(toml_grl["poor_thresholds"][grl_tag_parent]) 

398 v_prop["bad_thresholds"] = dict(toml_grl["bad_thresholds"][grl_tag_parent]) 

399 v_prop["discard_thresholds"] = dict( 

400 toml_grl["discard_thresholds"][grl_tag_parent] 

401 ) 

402 

403 for i_type in ( 

404 "veto_thresholds", 

405 "qsco_thresholds", 

406 "good_thresholds", 

407 "poor_thresholds", 

408 "bad_thresholds", 

409 "discard_thresholds", 

410 ): 

411 if dqt["grl"] in toml_grl[i_type].keys(): 

412 for i_modif in toml_grl[i_type][dqt["grl"]]: 

413 # If empty list, remove the thresholds defined in quasi_online dqt['grl'] 

414 if len(toml_grl[i_type][dqt["grl"]][i_modif]) == 0: 

415 v_prop[i_type].pop(i_modif) 

416 continue 

417 # Otherwise update them 

418 v_prop[i_type][i_modif] = toml_grl[i_type][dqt["grl"]][i_modif] 

419 

420 # DUMMY detector used only by the get_grl_html_descr function 

421 # used by km3dq_perf/generatewww.py - No impact on perf 

422 if det == "DUMMY": 

423 return 

424 

425 # Detector dependant treatment 

426 try: 

427 # Display options - Not DQ tag dependant (as of today but could be changed) 

428 v_prop["basic_plots_options"] = dict( 

429 toml_perf_aux["basic_plots_options"]["quasi_online"] 

430 ) 

431 v_prop["basic_plots_y_range_display"] = dict( 

432 toml_perf_aux["basic_plots_y_range_display"]["quasi_online"] 

433 ) 

434 v_prop["basic_plots_y_range_display"]["PMTs"][1] = ( 

435 get_detx_caract(det)["pmts"] * 1.2 

436 ) 

437 if "ORCA" in det: # Increased range for bioluminescence 

438 v_prop["basic_plots_y_range_display"]["HRV"] = [-0.1, 0.8] 

439 except KeyError: 

440 print(f"config_library.py: Missing DQ-tag information for the detector {det}") 

441 print(f"QAQC version: {dqt['qaqc_version'].keys()}") 

442 

443############################################################################### 

444def get_grl_html_descr(dq_tag_0): 

445 """ 

446 Returns the grl description in an html for the summary webpage 

447 The grl may depend on the detector due to missing variables. This is 

448 by-passed by passing a dummy value to the configure_var_thresholds 

449 function. In this way, no variable is discarded 

450 """ 

451 

452 var_prop = {} 

453 configure_var_thresholds(var_prop, "DUMMY", dq_tag_0) 

454 configure_var_unit(var_prop) 

455 html_descr = {} 

456 for i_key in var_prop.keys(): 

457 # One assumes that the QQC variables arrive before the defects 

458 # in the configuration file 

459 html_descr[i_key] = "<b>QAQC variables</b><br>" 

460 defect_found = False 

461 for i_var in var_prop[i_key].keys(): 

462 # qaqc variable 

463 if any( 

464 ( 

465 isinstance(var_prop[i_key][i_var][0], int), 

466 isinstance(var_prop[i_key][i_var][0], float), 

467 ) 

468 ): 

469 if var_prop[i_key][i_var][0] == var_prop[i_key][i_var][1]: 

470 thresh = f"{var_prop[i_key][i_var][0]}" 

471 if i_var == "veto": 

472 thresh = f"{var_prop[i_key][i_var][0] == 1}" 

473 else: 

474 thresh = ( 

475 f"{var_prop[i_key][i_var][0]} &#8212 " 

476 f"{var_prop[i_key][i_var][1]}" 

477 ) 

478 if i_var in var_prop["unit"].keys(): 

479 thresh += f" {var_prop['unit'][i_var]}" 

480 

481 html_descr[i_key] += f"{i_var}: {thresh}" 

482 # defect 

483 elif isinstance(var_prop[i_key][i_var][0], str): 

484 if defect_found is False: 

485 html_descr[i_key] += "<hr><b>Defects</b><br>" 

486 defect_found = True 

487 html_descr[i_key] += ( 

488 f"{i_var.replace('def_', '')}: " f"{var_prop[i_key][i_var][0]}" 

489 ) 

490 for j in range(1, len(var_prop[i_key][i_var])): 

491 html_descr[i_key] += f", {var_prop[i_key][i_var][j]}" 

492 

493 html_descr[i_key] += "<br>" 

494 

495 return html_descr