Coverage for src/km3dq_common/config_library.py: 62%
211 statements
« prev ^ index » next coverage.py v7.10.7, created at 2025-10-16 11:32 +0000
1#! /usr/bin/env python
2###############################################################################
3# Definition of QAQC variable, defects, data-quality tags...
4#
5# Developer: Benjamin Trocme (benjamin.trocme at apc.in2p3.fr) - 2023
7import tomli
8import sys
9import urllib.request
10import atexit
12if sys.version_info < (3, 9):
13 import importlib_resources
15 try:
16 from importlib_resources import as_file
17 except ImportError:
18 from importlib_resources.trees import as_file
19else:
20 import importlib.resources as importlib_resources
21 from importlib.resources import as_file
23try:
24 from contextlib import ExitStack
25except ImportError:
26 from contextlib2 import ExitStack
29###############################################################################
30def read_configuration(filename, source, raise_missing=True):
31 """
32 """
34 if source == "sftp":
35 with urllib.request.urlopen(
36 f"https://sftp.km3net.de/data/km3dq_lw_db/Common/{filename}.toml"
37 ) as s_f:
38 return tomli.loads(s_f.read().decode("utf-8"))
39 else:
40 ref = importlib_resources.files("km3dq_common") / f"{dir_absolute_path}/{filename}.toml"
41 file_manager = ExitStack()
42 atexit.register(file_manager.close)
43 file_path = file_manager.enter_context(as_file(ref))
44 if raise_missing and not file_path.exists():
45 raise RuntimeError(f"Unknown or missing file: {filename}")
46 return tomli.load(file_path)
49###############################################################################
50def get_detx_caract(det, source="sftp"):
51 """
53 Return the number of DOMs (incl base module so far) and PMts
55 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/),
56 either the directory absolute-path.
57 """
59 toml = read_configuration("km3dq_perf_aux", source)
61 return toml["hardware_charact"][det]
64###############################################################################
65def configure_var_name(v_prop):
66 """Description of JQAQC variables"""
68 # QAQC variables
69 v_prop["description"] = {
70 "run": "Run",
71 "livetime_s": "Livetime (s)",
72 "nb_of_meta": "Nb of metadata",
73 "timestampdiff": "Missing time-slices",
74 "timestampdiff_r": ("Missing time-slices" "/Run duration"),
75 "triggerrate": "Trigger rate(Hz)",
76 "HRV": "High rate veto bit",
77 "DAQ": "Fraction of data without UDP packet-loss",
78 "WR": "White rabbit status",
79 "FIFO": "FIFO almost full",
80 "PMTs": "Number of active PMTs",
81 "pmts_norm": "Fraction of active PMTs",
82 "MEAN_Rate_Hz": "Mean rate (Hz)",
83 "RMS_Rate_Hz": "RMS rate (Hz)",
84 "hrv_fifo_failures": "Number of hrv/fifo failures",
85 "duplic_timeslices": "Number of duplicated timeslices",
86 "out_sync": "Out-of-sync",
87 "out_usync": "Micro out-of-sync",
88 "JDAQJTProc": "Triggered != Retriggered",
89 "Acoustics": "Number of acoustic events",
90 "Acoustics_per_mn": "Number of acoustic events per minute",
91 "zero_AHRS": ("Number of DOMs without valid " "AHRS data"),
92 "ahrs_per_mn": ("Number of AHRS event per minute " "in > 80% of DOMs"),
93 "JDAQEvent": "Total",
94 "JTrigger3DShower": "3D shower trigger",
95 "JTriggerMXShower": "MX shower trigger",
96 "JTrigger3DMuon": "3D Muon trigger ",
97 "HV_check": "High-voltage inconsistencies",
98 }
101###############################################################################
102def configure_dataquality_tag(tag, source="sftp"):
103 """
104 TOMl configuration of the data quality tags
105 === Arguments ===
106 - tag : tag name - [string] - Ex: "default", "calibration"...
107 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/),
108 either the directory absolute-path.
109 === Output ===
110 - Dictionnary with all tag characteristics for all detectors
111 """
113 ####################################################
114 def single_property_process(tag, tag_parent, toml_input):
115 output = {}
117 for i_tag in (tag_parent, tag):
118 if i_tag in toml_input.keys():
119 output = toml_input[i_tag]
121 return output
123 ####################################################
124 def all_site_process(tag, tag_parent, toml_input, sites):
125 output = {"ARCA": "", "ORCA": ""}
127 for i_tag in (tag_parent, tag):
128 if i_tag == "": # No tag_parent
129 continue
130 for i_site in sites:
131 if i_site in toml_input[i_tag].keys():
132 output[i_site] = toml_input[i_tag][i_site]
134 return output
136 ####################################################
137 def all_dets_process(tag, tag_parent, toml_input, det_list):
138 # det_list: detector list
139 output = {}
141 for i_tag in (tag_parent, tag):
142 if i_tag == "": # No tag_parent
143 continue
144 for i_site in ["ARCA", "ORCA"]:
145 for i_det in det_list[i_site]:
146 if i_tag in toml_input.keys():
147 if i_det in toml_input[i_tag]: # Detector explicitly defined
148 output[i_det] = toml_input[i_tag][i_det]
149 elif i_site in toml_input[i_tag]: # Use the site definition
150 output[i_det] = toml_input[i_tag][i_site]
152 return output
154 ####################################################
155 # First retrieve the general caracteristics:
156 # status, description, dataset name, grl...
157 try:
158 toml = read_configuration("dataquality_tag", source)
160 dq_t = {}
161 # The path directory is the name of the data-quality tag
162 dq_t["dir_path"] = tag
163 # Retrieve the general descriptions
164 dq_t["name"] = toml["short_descr"][tag]
165 dq_t["descr"] = toml["descr"][tag]
166 dq_t["status"] = toml["status"][tag]
167 # Dataset name
168 dq_t["dataset"] = toml["dataset"][tag]
169 dq_t["site"] = toml["site"][tag]
170 # GRL criteria
171 dq_t["grl"] = toml["grl"][tag]
172 # Defect tag
173 dq_t["def_tag"] = toml["def_tag"][tag]
175 except:
176 print("Problem in dataquality_tag.toml file")
177 sys.exit()
179 # Retrieve the dataset description from the dataset.toml file
180 try:
181 toml_ds = read_configuration("dataset", source)
183 dq_t["dataset_descr"] = toml_ds["descr"][dq_t["dataset"]]
184 dq_t["dataset_parent"] = toml_ds["parent"][dq_t["dataset"]]
185 # Name of processing type / versions
186 dq_t["proc"] = single_property_process(
187 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["proc"]
188 )
189 except KeyError:
190 print("Problem in dataset configuration - dataset step")
191 sys.exit()
192 try:
193 # List of detectors per site - Single string
194 dq_t["det"] = all_site_process(
195 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["site_all_dets"], dq_t["site"].split(" ")
196 )
197 for i_site in dq_t["det"]:
198 if dq_t["det"][i_site] != "":
199 dq_t["det"][i_site] = dq_t["det"][i_site].split(" ")
200 else:
201 dq_t["det"][i_site] = []
202 except KeyError:
203 print("Problem in dataset configuration - detector step")
204 sys.exit()
205 try:
206 # Run type - Detector dependant -> specific processing
207 # Empty for v16 and v17 as did not contribute to the naming of file
208 dq_t["run_type"] = all_dets_process(
209 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["run_type"], dq_t["det"]
210 )
211 # QAQC version - Detector dependant -> specific processing
212 # )
213 dq_t["qaqc_version"] = all_dets_process(
214 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["qaqc_version"], dq_t["det"]
215 )
216 # Compile the run type and qaqc version to get the filename
217 dq_t["qaqc_name"] = {}
218 for i_det in dq_t["run_type"]:
219 dq_t["qaqc_name"][i_det] = f"{dq_t['qaqc_version'][i_det]}_{dq_t['run_type'][i_det]}"
220 except KeyError:
221 print("Problem in dataset configuration - type/version step")
222 sys.exit()
223 try:
224 # Run range - Detector dependant -> specific processing
225 dq_t["run_range"] = all_dets_process(
226 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["run_range"], dq_t["det"]
227 )
228 except:
229 print("Problem in dataset configuration - run-range step")
230 sys.exit()
232 # Retrieve the processing type/versions from the processing.toml file
233 # Detector dependant -> specific processing defined in dataset
234 try:
235 toml_proc = read_configuration("processing", source)
237 dq_t["proc_descr"] = toml_proc["descr"][dq_t["proc"]]
238 dq_t["proc_type"] = all_dets_process(
239 dq_t["proc"], "", toml_proc["proc_type"], dq_t["det"]
240 )
241 dq_t["proc_version"] = all_dets_process(
242 dq_t["proc"], "", toml_proc["proc_version"], dq_t["det"]
243 )
244 except:
245 sys.exit()
247 # Retrieve the good run list definition (name and definition)
248 # The threshold retrieval is completed in the script configure_var_thresholds
249 try:
250 toml_grl = read_configuration("grl", source)
252 dq_t["grl_descr"] = toml_grl["descr"][dq_t["grl"]]
253 grl_tag_parent = toml_grl["parent"][dq_t["grl"]]
254 if grl_tag_parent == "":
255 dq_t["grl_name"] = toml_grl["grl_name"][dq_t["grl"]]
256 else:
257 dq_t["grl_name"] = toml_grl["grl_name"][grl_tag_parent]
258 for i_name in dq_t["grl_name"].keys():
259 if i_name in toml_grl["grl_name"][dq_t["grl"]].keys():
260 dq_t["grl_name"][i_name] = toml_grl["grl_name"][dq_t["grl"]][i_name]
262 except:
263 print("Problem in grl configuration")
264 sys.exit()
266 return dq_t
269###############################################################################
270def configure_defect(source="sftp"):
271 """
272 TOMl configuration of the defects
274 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/),
275 either the directory absolute-path.
276 """
277 toml = read_configuration("defect", source)
279 return {
280 "descr": toml["descr"],
281 "type": list(toml["descr"].keys()),
282 "bit": toml["bit"]
283 }
286###############################################################################
287def configure_def_var_name(v_prop):
288 """Description of defect variables"""
290 # QAQC variables
291 v_prop["description"]["def_daq"] = "DAQ defects"
292 v_prop["description"]["def_operation"] = "Operation defects"
293 v_prop["description"]["def_data_corruption"] = "Data corruption defects"
294 v_prop["description"]["def_calibration"] = "Calibration defects"
295 v_prop["description"]["def_data_processing"] = "Data processing defects"
296 v_prop["description"]["def_analysis"] = "Analysis defects"
297 v_prop["description"]["def_high_level_analysis"] = "Neutrino analysis defects"
298 v_prop["description"]["def_signoff"] = "Missing signoff"
301###############################################################################
302def configure_fact(source="sftp"):
303 """
304 TOMl configuration of the facts
306 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/),
307 either the directory absolute-path.
308 """
309 toml = read_configuration("fact", source)
311 return {"type": toml["type"]}
314###############################################################################
315def configure_det_fact(source="sftp"):
316 """
317 TOMl configuration of the detector facts
319 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/),
320 either the directory absolute-path.
321 """
322 toml = read_configuration("det_fact", source)
324 return {
325 "type": toml["type"],
326 "status": toml["status"],
327 }
330###############################################################################
331def configure_var_unit(v_prop):
332 """Description of variables"""
333 v_prop["unit"] = {
334 "livetime": "s",
335 "timestampdiff": "s",
336 "triggerrate": "Hz",
337 "meanrate": "Hz",
338 "rmsrate": "Hz",
339 }
342###############################################################################
343def configure_var_bit(v_prop):
344 """
345 Bit of variables contributing to the veto//Qscore degradation
346 used to store in the TTree
347 """
348 v_prop["bit"] = {
349 "livetime_s": 0,
350 "timestampdiff": 1,
351 "timestampdiff_r": 2,
352 "triggerrate": 3,
353 "HRV": 4,
354 "DAQ": 5,
355 "WR": 6,
356 "FIFO": 7,
357 "PMTs": 8,
358 "pmts_norm": 9,
359 "MEAN_Rate_Hz": 10,
360 "RMS_Rate_Hz": 11,
361 "out_sync": 12,
362 "out_usync": 13,
363 "JDAQJTProc": 14,
364 "Acoustics": 15,
365 "zero_AHRS": 16,
366 "ahrs_per_mn": 17,
367 "nb_of_meta": 18,
368 "hrv_fifo_failures": 19,
369 "duplic_timeslices": 20,
370 "HV_check": 21,
371 "def_daq": 23,
372 "def_operation": 24,
373 "def_data_corruption": 25,
374 "def_calibration": 26,
375 "def_data_processing": 27,
376 "def_analysis": 28,
377 "def_high_level_analysis": 29,
378 "def_signoff": 30,
379 }
382###############################################################################
383def configure_var_thresholds(v_prop, det, dqt, source="sftp"):
384 """
385 Configure various variables for the analysis of the TTree derived from
386 the JQAQC file
387 TOMl configuration
389 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/),
390 either the directory absolute-path.
391 """
392 toml_grl = read_configuration("grl", source)
393 toml_perf_aux = read_configuration("km3dq_perf_aux", source)
395 grl_tag_parent = toml_grl["parent"][dqt["grl"]]
397 if grl_tag_parent == "":
398 v_prop["veto_thresholds"] = dict(toml_grl["veto_thresholds"][dqt["grl"]])
399 v_prop["qsco_thresholds"] = dict(toml_grl["qsco_thresholds"][dqt["grl"]])
400 v_prop["good_thresholds"] = dict(toml_grl["good_thresholds"][dqt["grl"]])
401 v_prop["poor_thresholds"] = dict(toml_grl["poor_thresholds"][dqt["grl"]])
402 v_prop["bad_thresholds"] = dict(toml_grl["bad_thresholds"][dqt["grl"]])
403 v_prop["discard_thresholds"] = dict(toml_grl["discard_thresholds"][dqt["grl"]])
404 else:
405 v_prop["veto_thresholds"] = dict(toml_grl["veto_thresholds"][grl_tag_parent])
406 v_prop["qsco_thresholds"] = dict(toml_grl["qsco_thresholds"][grl_tag_parent])
407 v_prop["good_thresholds"] = dict(toml_grl["good_thresholds"][grl_tag_parent])
408 v_prop["poor_thresholds"] = dict(toml_grl["poor_thresholds"][grl_tag_parent])
409 v_prop["bad_thresholds"] = dict(toml_grl["bad_thresholds"][grl_tag_parent])
410 v_prop["discard_thresholds"] = dict(
411 toml_grl["discard_thresholds"][grl_tag_parent]
412 )
414 for i_type in (
415 "veto_thresholds",
416 "qsco_thresholds",
417 "good_thresholds",
418 "poor_thresholds",
419 "bad_thresholds",
420 "discard_thresholds",
421 ):
422 if dqt["grl"] in toml_grl[i_type].keys():
423 for i_modif in toml_grl[i_type][dqt["grl"]]:
424 # If empty list, remove the thresholds defined in quasi_online dqt['grl']
425 if len(toml_grl[i_type][dqt["grl"]][i_modif]) == 0:
426 v_prop[i_type].pop(i_modif)
427 continue
428 # Otherwise update them
429 v_prop[i_type][i_modif] = toml_grl[i_type][dqt["grl"]][i_modif]
431 # DUMMY detector used only by the get_grl_html_descr function
432 # used by km3dq_perf/generatewww.py - No impact on perf
433 if det == "DUMMY":
434 return
436 # Detector dependant treatment
437 try:
438 # Display options - Not DQ tag dependant (as of today but could be changed)
439 v_prop["basic_plots_options"] = dict(
440 toml_perf_aux["basic_plots_options"]["quasi_online"]
441 )
442 v_prop["basic_plots_y_range_display"] = dict(
443 toml_perf_aux["basic_plots_y_range_display"]["quasi_online"]
444 )
445 v_prop["basic_plots_y_range_display"]["PMTs"][1] = (
446 get_detx_caract(det)["pmts"] * 1.2
447 )
448 if "ORCA" in det: # Increased range for bioluminescence
449 v_prop["basic_plots_y_range_display"]["HRV"] = [-0.1, 0.8]
450 except KeyError:
451 print(f"config_library.py / configure_var_thresholds: Missing DQ-tag information for the detector {det}")
452 print(f"QAQC version: {dqt['qaqc_version'].keys()}")
454###############################################################################
455def get_grl_html_descr(dq_tag_0):
456 """
457 Returns the grl description in an html for the summary webpage
458 The grl may depend on the detector due to missing variables. This is
459 by-passed by passing a dummy value to the configure_var_thresholds
460 function. In this way, no variable is discarded
461 """
463 var_prop = {}
464 configure_var_thresholds(var_prop, "DUMMY", dq_tag_0)
465 configure_var_unit(var_prop)
466 html_descr = {}
467 for i_key in var_prop.keys():
468 # One assumes that the QQC variables arrive before the defects
469 # in the configuration file
470 html_descr[i_key] = "<b>QAQC variables</b><br>"
471 defect_found = False
472 for i_var in var_prop[i_key].keys():
473 # qaqc variable
474 if any(
475 (
476 isinstance(var_prop[i_key][i_var][0], int),
477 isinstance(var_prop[i_key][i_var][0], float),
478 )
479 ):
480 if var_prop[i_key][i_var][0] == var_prop[i_key][i_var][1]:
481 thresh = f"{var_prop[i_key][i_var][0]}"
482 if i_var == "veto":
483 thresh = f"{var_prop[i_key][i_var][0] == 1}"
484 else:
485 thresh = (
486 f"{var_prop[i_key][i_var][0]} — "
487 f"{var_prop[i_key][i_var][1]}"
488 )
489 if i_var in var_prop["unit"].keys():
490 thresh += f" {var_prop['unit'][i_var]}"
492 html_descr[i_key] += f"{i_var}: {thresh}"
493 # defect
494 elif isinstance(var_prop[i_key][i_var][0], str):
495 if defect_found is False:
496 html_descr[i_key] += "<hr><b>Defects</b><br>"
497 defect_found = True
498 html_descr[i_key] += (
499 f"{i_var.replace('def_', '')}: " f"{var_prop[i_key][i_var][0]}"
500 )
501 for j in range(1, len(var_prop[i_key][i_var])):
502 html_descr[i_key] += f", {var_prop[i_key][i_var][j]}"
504 html_descr[i_key] += "<br>"
506 return html_descr