Coverage for src/km3dq_common/config_library.py: 64%
199 statements
« prev ^ index » next coverage.py v7.9.2, created at 2025-07-07 10:42 +0000
1#! /usr/bin/env python
2###############################################################################
3# Definition of QAQC variable, defects, data-quality tags...
4#
5# Developer: Benjamin Trocme (benjamin.trocme at apc.in2p3.fr) - 2023
7import tomli
8import sys
9import urllib.request
10import atexit
12if sys.version_info < (3, 9):
13 import importlib_resources
15 try:
16 from importlib_resources import as_file
17 except ImportError:
18 from importlib_resources.trees import as_file
19else:
20 import importlib.resources as importlib_resources
21 from importlib.resources import as_file
23try:
24 from contextlib import ExitStack
25except ImportError:
26 from contextlib2 import ExitStack
29###############################################################################
30def read_configuration(filename, source, raise_missing=True):
31 """
32 """
34 if source == "sftp":
35 with urllib.request.urlopen(
36 f"https://sftp.km3net.de/data/km3dq_lw_db/Common/{filename}.toml"
37 ) as s_f:
38 return tomli.loads(s_f.read().decode("utf-8"))
39 else:
40 ref = importlib_resources.files("km3dq_common") / f"{dir_absolute_path}/{filename}.toml"
41 file_manager = ExitStack()
42 atexit.register(file_manager.close)
43 file_path = file_manager.enter_context(as_file(ref))
44 if raise_missing and not file_path.exists():
45 raise RuntimeError(f"Unknown or missing file: {filename}")
46 return tomli.load(file_path)
49###############################################################################
50def get_detx_caract(det, source="sftp"):
51 """
53 Return the number of DOMs (incl base module so far) and PMts
55 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/),
56 either the directory absolute-path.
57 """
59 toml = read_configuration("km3dq_perf_aux", source)
61 return toml["hardware_charact"][det]
64###############################################################################
65def configure_var_name(v_prop):
66 """Description of JQAQC variables"""
68 # QAQC variables
69 v_prop["description"] = {
70 "run": "Run",
71 "livetime_s": "Livetime (s)",
72 "nb_of_meta": "Nb of metadata",
73 "timestampdiff": "Missing time-slices",
74 "timestampdiff_r": ("Missing time-slices" "/Run duration"),
75 "triggerrate": "Trigger rate(Hz)",
76 "HRV": "High rate veto bit",
77 "DAQ": "Fraction of data without UDP packet-loss",
78 "WR": "White rabbit status",
79 "FIFO": "FIFO almost full",
80 "PMTs": "Number of active PMTs",
81 "pmts_norm": "Fraction of active PMTs",
82 "MEAN_Rate_Hz": "Mean rate (Hz)",
83 "RMS_Rate_Hz": "RMS rate (Hz)",
84 "hrv_fifo_failures": "Number of hrv/fifo failures",
85 "duplic_timeslices": "Number of duplicated timeslices",
86 "out_sync": "Out-of-sync",
87 "out_usync": "Micro out-of-sync",
88 "JDAQJTProc": "Triggered != Retriggered",
89 "Acoustics": "Number of acoustic events",
90 "Acoustics_per_mn": "Number of acoustic events per minute",
91 "zero_AHRS": ("Number of DOMs without valid " "AHRS data"),
92 "ahrs_per_mn": ("Number of AHRS event per minute " "in > 80% of DOMs"),
93 "JDAQEvent": "Total",
94 "JTrigger3DShower": "3D shower trigger",
95 "JTriggerMXShower": "MX shower trigger",
96 "JTrigger3DMuon": "3D Muon trigger ",
97 "HV_check": "High-voltage inconsistencies",
98 }
101###############################################################################
102def configure_dataquality_tag(tag, source="sftp"):
103 """
104 TOMl configuration of the data quality tags
105 === Arguments ===
106 - tag : tag name - [string] - Ex: "default", "calibration"...
107 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/),
108 either the directory absolute-path.
109 === Output ===
110 - Dictionnary with all tag characteristics for all detectors
111 """
113 ####################################################
114 def single_property_process(tag, tag_parent, toml_input):
115 output = {}
117 for i_tag in (tag_parent, tag):
118 if i_tag in toml_input.keys():
119 output = toml_input[i_tag]
121 return output
123 ####################################################
124 def all_site_process(tag, tag_parent, toml_input, sites):
125 output = {"ARCA": "", "ORCA": ""}
127 for i_tag in (tag_parent, tag):
128 if i_tag == "": # No tag_parent
129 continue
130 for i_site in sites:
131 if i_site in toml_input[i_tag].keys():
132 output[i_site] = toml_input[i_tag][i_site]
134 return output
136 ####################################################
137 def all_dets_process(tag, tag_parent, toml_input, det_list):
138 # det_list: detector list
139 output = {}
141 for i_tag in (tag_parent, tag):
142 if i_tag == "": # No tag_parent
143 continue
144 for i_site in ["ARCA", "ORCA"]:
145 for i_det in det_list[i_site]:
146 if i_tag in toml_input.keys():
147 if i_det in toml_input[i_tag]: # Detector explicitly defined
148 output[i_det] = toml_input[i_tag][i_det]
149 elif i_site in toml_input[i_tag]: # Use the site definition
150 output[i_det] = toml_input[i_tag][i_site]
152 return output
154 ####################################################
155 # First retrieve the general caracteristics:
156 # status, description, dataset name, grl...
157 try:
158 toml = read_configuration("dataquality_tag", source)
160 dq_t = {}
161 # The path directory is the name of the data-quality tag
162 dq_t["dir_path"] = tag
163 # Retrieve the general descriptions
164 dq_t["name"] = toml["short_descr"][tag]
165 dq_t["descr"] = toml["descr"][tag]
166 dq_t["status"] = toml["status"][tag]
167 # Dataset name
168 dq_t["dataset"] = toml["dataset"][tag]
169 dq_t["site"] = toml["site"][tag]
170 # GRL criteria
171 dq_t["grl"] = toml["grl"][tag]
172 # Defect tag
173 dq_t["def_tag"] = toml["def_tag"][tag]
175 except:
176 print("Problem in dataquality_tag.toml file")
177 sys.exit()
179 # Retrieve the dataset description from the dataset.toml file
180 try:
181 toml_ds = read_configuration("dataset", source)
183 dq_t["dataset_descr"] = toml_ds["descr"][dq_t["dataset"]]
184 dq_t["dataset_parent"] = toml_ds["parent"][dq_t["dataset"]]
185 # Name of processing type / versions
186 dq_t["proc"] = single_property_process(
187 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["proc"]
188 )
189 # List of detectors per site - Single string
190 dq_t["det"] = all_site_process(
191 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["site_all_dets"], dq_t["site"].split(" ")
192 )
193 for i_site in dq_t["det"]:
194 if dq_t["det"][i_site] != "":
195 dq_t["det"][i_site] = dq_t["det"][i_site].split(" ")
196 else:
197 dq_t["det"][i_site] = []
198 # Run type - Detector dependant -> specific processing
199 # Empty for v16 and v17 as did not contribute to the naming of file
200 dq_t["run_type"] = all_dets_process(
201 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["run_type"], dq_t["det"]
202 )
203 # QAQC version - Detector dependant -> specific processing
204 # )
205 dq_t["qaqc_version"] = all_dets_process(
206 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["qaqc_version"], dq_t["det"]
207 )
208 # Compile the run type and qaqc version to get the filename
209 dq_t["qaqc_name"] = {}
210 for i_det in dq_t["run_type"]:
211 dq_t["qaqc_name"][i_det] = f"{dq_t['qaqc_version'][i_det]}_{dq_t['run_type'][i_det]}"
213 # Run range - Detector dependant -> specific processing
214 dq_t["run_range"] = all_dets_process(
215 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["run_range"], dq_t["det"]
216 )
218 except:
219 print("Problem in dataset configuration")
220 sys.exit()
222 # Retrieve the processing type/versions from the processing.toml file
223 # Detector dependant -> specific processing defined in dataset
224 try:
225 toml_proc = read_configuration("processing", source)
227 dq_t["proc_descr"] = toml_proc["descr"][dq_t["proc"]]
228 dq_t["proc_type"] = all_dets_process(
229 dq_t["proc"], "", toml_proc["proc_type"], dq_t["det"]
230 )
231 dq_t["proc_version"] = all_dets_process(
232 dq_t["proc"], "", toml_proc["proc_version"], dq_t["det"]
233 )
234 except:
235 sys.exit()
237 # Retrieve the good run list definition (name and definition)
238 # The threshold retrieval is completed in the script configure_var_thresholds
239 try:
240 toml_grl = read_configuration("grl", source)
242 dq_t["grl_descr"] = toml_grl["descr"][dq_t["grl"]]
243 grl_tag_parent = toml_grl["parent"][dq_t["grl"]]
244 if grl_tag_parent == "":
245 dq_t["grl_name"] = toml_grl["grl_name"][dq_t["grl"]]
246 else:
247 dq_t["grl_name"] = toml_grl["grl_name"][grl_tag_parent]
248 for i_name in dq_t["grl_name"].keys():
249 if i_name in toml_grl["grl_name"][dq_t["grl"]].keys():
250 dq_t["grl_name"][i_name] = toml_grl["grl_name"][dq_t["grl"]][i_name]
252 except:
253 print("Problem in grl configuration")
254 sys.exit()
256 return dq_t
259###############################################################################
260def configure_defect(source="sftp"):
261 """
262 TOMl configuration of the defects
264 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/),
265 either the directory absolute-path.
266 """
267 toml = read_configuration("defect", source)
269 return {
270 "descr": toml["descr"],
271 "type": list(toml["descr"].keys()),
272 "bit": toml["bit"]
273 }
276###############################################################################
277def configure_def_var_name(v_prop):
278 """Description of defect variables"""
280 # QAQC variables
281 v_prop["description"]["def_daq"] = "DAQ defects"
282 v_prop["description"]["def_operation"] = "Operation defects"
283 v_prop["description"]["def_data_corruption"] = "Data corruption defects"
284 v_prop["description"]["def_calibration"] = "Calibration defects"
285 v_prop["description"]["def_data_processing"] = "Data processing defects"
286 v_prop["description"]["def_analysis"] = "Analysis defects"
287 v_prop["description"]["def_high_level_analysis"] = "Neutrino analysis defects"
288 v_prop["description"]["def_signoff"] = "Missing signoff"
291###############################################################################
292def configure_fact(source="sftp"):
293 """
294 TOMl configuration of the facts
296 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/),
297 either the directory absolute-path.
298 """
299 toml = read_configuration("fact", source)
301 return {"type": toml["type"]}
304###############################################################################
305def configure_det_fact(source="sftp"):
306 """
307 TOMl configuration of the detector facts
309 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/),
310 either the directory absolute-path.
311 """
312 toml = read_configuration("det_fact", source)
314 return {
315 "type": toml["type"],
316 "status": toml["status"],
317 }
320###############################################################################
321def configure_var_unit(v_prop):
322 """Description of variables"""
323 v_prop["unit"] = {
324 "livetime": "s",
325 "timestampdiff": "s",
326 "triggerrate": "Hz",
327 "meanrate": "Hz",
328 "rmsrate": "Hz",
329 }
332###############################################################################
333def configure_var_bit(v_prop):
334 """
335 Bit of variables contributing to the veto//Qscore degradation
336 used to store in the TTree
337 """
338 v_prop["bit"] = {
339 "livetime_s": 0,
340 "timestampdiff": 1,
341 "timestampdiff_r": 2,
342 "triggerrate": 3,
343 "HRV": 4,
344 "DAQ": 5,
345 "WR": 6,
346 "FIFO": 7,
347 "PMTs": 8,
348 "pmts_norm": 9,
349 "MEAN_Rate_Hz": 10,
350 "RMS_Rate_Hz": 11,
351 "out_sync": 12,
352 "out_usync": 13,
353 "JDAQJTProc": 14,
354 "Acoustics": 15,
355 "zero_AHRS": 16,
356 "ahrs_per_mn": 17,
357 "nb_of_meta": 18,
358 "hrv_fifo_failures": 19,
359 "duplic_timeslices": 20,
360 "HV_check": 21,
361 "def_daq": 23,
362 "def_operation": 24,
363 "def_data_corruption": 25,
364 "def_calibration": 26,
365 "def_data_processing": 27,
366 "def_analysis": 28,
367 "def_high_level_analysis": 29,
368 "def_signoff": 30,
369 }
372###############################################################################
373def configure_var_thresholds(v_prop, det, dqt, source="sftp"):
374 """
375 Configure various variables for the analysis of the TTree derived from
376 the JQAQC file
377 TOMl configuration
379 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/),
380 either the directory absolute-path.
381 """
382 toml_grl = read_configuration("grl", source)
383 toml_perf_aux = read_configuration("km3dq_perf_aux", source)
385 grl_tag_parent = toml_grl["parent"][dqt["grl"]]
387 if grl_tag_parent == "":
388 v_prop["veto_thresholds"] = dict(toml_grl["veto_thresholds"][dqt["grl"]])
389 v_prop["qsco_thresholds"] = dict(toml_grl["qsco_thresholds"][dqt["grl"]])
390 v_prop["good_thresholds"] = dict(toml_grl["good_thresholds"][dqt["grl"]])
391 v_prop["poor_thresholds"] = dict(toml_grl["poor_thresholds"][dqt["grl"]])
392 v_prop["bad_thresholds"] = dict(toml_grl["bad_thresholds"][dqt["grl"]])
393 v_prop["discard_thresholds"] = dict(toml_grl["discard_thresholds"][dqt["grl"]])
394 else:
395 v_prop["veto_thresholds"] = dict(toml_grl["veto_thresholds"][grl_tag_parent])
396 v_prop["qsco_thresholds"] = dict(toml_grl["qsco_thresholds"][grl_tag_parent])
397 v_prop["good_thresholds"] = dict(toml_grl["good_thresholds"][grl_tag_parent])
398 v_prop["poor_thresholds"] = dict(toml_grl["poor_thresholds"][grl_tag_parent])
399 v_prop["bad_thresholds"] = dict(toml_grl["bad_thresholds"][grl_tag_parent])
400 v_prop["discard_thresholds"] = dict(
401 toml_grl["discard_thresholds"][grl_tag_parent]
402 )
404 for i_type in (
405 "veto_thresholds",
406 "qsco_thresholds",
407 "good_thresholds",
408 "poor_thresholds",
409 "bad_thresholds",
410 "discard_thresholds",
411 ):
412 if dqt["grl"] in toml_grl[i_type].keys():
413 for i_modif in toml_grl[i_type][dqt["grl"]]:
414 # If empty list, remove the thresholds defined in quasi_online dqt['grl']
415 if len(toml_grl[i_type][dqt["grl"]][i_modif]) == 0:
416 v_prop[i_type].pop(i_modif)
417 continue
418 # Otherwise update them
419 v_prop[i_type][i_modif] = toml_grl[i_type][dqt["grl"]][i_modif]
421 # DUMMY detector used only by the get_grl_html_descr function
422 # used by km3dq_perf/generatewww.py - No impact on perf
423 if det == "DUMMY":
424 return
426 # Detector dependant treatment
427 try:
428 # Display options - Not DQ tag dependant (as of today but could be changed)
429 v_prop["basic_plots_options"] = dict(
430 toml_perf_aux["basic_plots_options"]["quasi_online"]
431 )
432 v_prop["basic_plots_y_range_display"] = dict(
433 toml_perf_aux["basic_plots_y_range_display"]["quasi_online"]
434 )
435 v_prop["basic_plots_y_range_display"]["PMTs"][1] = (
436 get_detx_caract(det)["pmts"] * 1.2
437 )
438 if "ORCA" in det: # Increased range for bioluminescence
439 v_prop["basic_plots_y_range_display"]["HRV"] = [-0.1, 0.8]
440 except KeyError:
441 print(f"config_library.py: Missing DQ-tag information for the detector {det}")
442 print(f"QAQC version: {dqt['qaqc_version'].keys()}")
444###############################################################################
445def get_grl_html_descr(dq_tag_0):
446 """
447 Returns the grl description in an html for the summary webpage
448 The grl may depend on the detector due to missing variables. This is
449 by-passed by passing a dummy value to the configure_var_thresholds
450 function. In this way, no variable is discarded
451 """
453 var_prop = {}
454 configure_var_thresholds(var_prop, "DUMMY", dq_tag_0)
455 configure_var_unit(var_prop)
456 html_descr = {}
457 for i_key in var_prop.keys():
458 # One assumes that the QQC variables arrive before the defects
459 # in the configuration file
460 html_descr[i_key] = "<b>QAQC variables</b><br>"
461 defect_found = False
462 for i_var in var_prop[i_key].keys():
463 # qaqc variable
464 if any(
465 (
466 isinstance(var_prop[i_key][i_var][0], int),
467 isinstance(var_prop[i_key][i_var][0], float),
468 )
469 ):
470 if var_prop[i_key][i_var][0] == var_prop[i_key][i_var][1]:
471 thresh = f"{var_prop[i_key][i_var][0]}"
472 if i_var == "veto":
473 thresh = f"{var_prop[i_key][i_var][0] == 1}"
474 else:
475 thresh = (
476 f"{var_prop[i_key][i_var][0]} — "
477 f"{var_prop[i_key][i_var][1]}"
478 )
479 if i_var in var_prop["unit"].keys():
480 thresh += f" {var_prop['unit'][i_var]}"
482 html_descr[i_key] += f"{i_var}: {thresh}"
483 # defect
484 elif isinstance(var_prop[i_key][i_var][0], str):
485 if defect_found is False:
486 html_descr[i_key] += "<hr><b>Defects</b><br>"
487 defect_found = True
488 html_descr[i_key] += (
489 f"{i_var.replace('def_', '')}: " f"{var_prop[i_key][i_var][0]}"
490 )
491 for j in range(1, len(var_prop[i_key][i_var])):
492 html_descr[i_key] += f", {var_prop[i_key][i_var][j]}"
494 html_descr[i_key] += "<br>"
496 return html_descr