Coverage for src/km3dq_common/config_library.py: 62%
220 statements
« prev ^ index » next coverage.py v7.10.7, created at 2025-10-27 12:26 +0000
1#! /usr/bin/env python
2###############################################################################
3# Definition of QAQC variable, defects, data-quality tags...
4#
5# Developer: Benjamin Trocme (benjamin.trocme at apc.in2p3.fr) - 2023
7import tomli
8import sys
9import re
10import urllib.request
11import atexit
13if sys.version_info < (3, 9):
14 import importlib_resources
16 try:
17 from importlib_resources import as_file
18 except ImportError:
19 from importlib_resources.trees import as_file
20else:
21 import importlib.resources as importlib_resources
22 from importlib.resources import as_file
24try:
25 from contextlib import ExitStack
26except ImportError:
27 from contextlib2 import ExitStack
30###############################################################################
31def read_configuration(filename, source, raise_missing=True):
32 """
33 """
35 if source == "sftp":
36 with urllib.request.urlopen(
37 f"https://sftp.km3net.de/data/km3dq_lw_db/Common/{filename}.toml"
38 ) as s_f:
39 return tomli.loads(s_f.read().decode("utf-8"))
40 else:
41 dir_absolute_path = "." # Facility never used / debugged
42 ref = importlib_resources.files("km3dq_common") / f"{dir_absolute_path}/{filename}.toml"
43 file_manager = ExitStack()
44 atexit.register(file_manager.close)
45 file_path = file_manager.enter_context(as_file(ref))
46 if raise_missing and not file_path.exists():
47 raise RuntimeError(f"Unknown or missing file: {filename}")
48 return tomli.load(file_path)
51###############################################################################
52def get_detx_caract(det, source="sftp"):
53 """
55 Return the number of DOMs (incl base module so far) and PMts
57 Formerly, it was defined in km3dq_perf_aux but now use the name of the detector
59 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/),
60 either the directory absolute-path.
61 """
63 r = re.search(r"D.[AO]RC.(\d+)", det)
64 return dict(
65 active_dus=int(r.group(1)),
66 doms=int(r.group(1)) * 18,
67 pmts=int(r.group(1)) * 18 * 31,
68 )
71###############################################################################
72def configure_var_name(v_prop, plot_type="basic_plots"):
73 """Description of JQAQC variables"""
75 # QAQC variables
76 v_prop["description"] = {
77 "run": "Run",
78 "livetime_s": "Livetime (s)",
79 "nb_of_meta": "Nb of metadata",
80 "timestampdiff": "Missing time-slices",
81 "timestampdiff_r": ("Missing time-slices" "/Run duration"),
82 "triggerrate": "Trigger rate(Hz)",
83 "HRV": "High rate veto bit",
84 "DAQ": "Fraction of data without UDP packet-loss",
85 "WR": "White rabbit status",
86 "FIFO": "FIFO almost full",
87 "PMTs": "Number of active PMTs",
88 "pmts_norm": "Fraction of active PMTs",
89 "MEAN_Rate_Hz": "Mean rate (Hz)",
90 "RMS_Rate_Hz": "RMS rate (Hz)",
91 "hrv_fifo_failures": "Number of hrv/fifo failures",
92 "duplic_timeslices": "Number of duplicated timeslices",
93 "out_sync": "Out-of-sync",
94 "out_usync": "Micro out-of-sync",
95 "JDAQJTProc": "Triggered != Retriggered",
96 "Acoustics": "Number of acoustic events",
97 "Acoustics_per_mn": "Number of acoustic events per minute",
98 "zero_AHRS": ("Number of DOMs without valid " "AHRS data"),
99 "ahrs_per_mn": ("Number of AHRS event per minute " "in > 80% of DOMs"),
100 "JDAQEvent": "Total",
101 "JTrigger3DShower": "3D shower trigger",
102 "JTriggerMXShower": "MX shower trigger",
103 "JTrigger3DMuon": "3D Muon trigger ",
104 "HV_check": "High-voltage inconsistencies",
105 "DB_check": "Wrong position calibrations"
106 }
108 if plot_type == "performance_plots":
109 v_prop["description"]["DAQ"] = "UDP packet-loss"
112###############################################################################
113def configure_dataquality_tag(tag, source="sftp"):
114 """
115 TOMl configuration of the data quality tags
116 === Arguments ===
117 - tag : tag name - [string] - Ex: "default", "calibration"...
118 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/),
119 either the directory absolute-path.
120 === Output ===
121 - Dictionnary with all tag characteristics for all detectors
122 """
124 ####################################################
125 def single_property_process(tag, tag_parent, toml_input):
126 output = {}
128 for i_tag in (tag_parent, tag):
129 if i_tag in toml_input.keys():
130 output = toml_input[i_tag]
132 return output
134 ####################################################
135 def all_site_process(tag, tag_parent, toml_input, sites):
136 output = {"ARCA": "", "ORCA": ""}
138 for i_tag in (tag_parent, tag):
139 if i_tag == "": # No tag_parent
140 continue
141 for i_site in sites:
142 if i_site in toml_input[i_tag].keys():
143 output[i_site] = toml_input[i_tag][i_site]
145 return output
147 ####################################################
148 def all_dets_process(tag, tag_parent, toml_input, det_list):
149 # det_list: detector list
150 output = {}
152 for i_tag in (tag_parent, tag):
153 if i_tag == "": # No tag_parent
154 continue
155 for i_site in ["ARCA", "ORCA"]:
156 for i_det in det_list[i_site]:
157 if i_tag in toml_input.keys():
158 if i_det in toml_input[i_tag]: # Detector explicitly defined
159 output[i_det] = toml_input[i_tag][i_det]
160 elif i_site in toml_input[i_tag]: # Use the site definition
161 output[i_det] = toml_input[i_tag][i_site]
163 return output
165 ####################################################
166 # First retrieve the general caracteristics:
167 # status, description, dataset name, grl...
168 try:
169 toml = read_configuration("dataquality_tag", source)
171 dq_t = {}
172 # The path directory is the name of the data-quality tag
173 dq_t["dir_path"] = tag
174 # Retrieve the general descriptions
175 dq_t["name"] = toml["short_descr"][tag]
176 dq_t["descr"] = toml["descr"][tag]
177 dq_t["status"] = toml["status"][tag]
178 # Dataset name
179 dq_t["dataset"] = toml["dataset"][tag]
180 dq_t["site"] = toml["site"][tag]
181 # GRL criteria
182 dq_t["grl"] = toml["grl"][tag]
183 # Defect tag
184 dq_t["def_tag"] = toml["def_tag"][tag]
186 except KeyError:
187 print("Problem in dataquality_tag.toml file")
188 sys.exit()
190 # Retrieve the dataset description from the dataset.toml file
191 try:
192 toml_ds = read_configuration("dataset", source)
194 dq_t["dataset_descr"] = toml_ds["descr"][dq_t["dataset"]]
195 dq_t["dataset_parent"] = toml_ds["parent"][dq_t["dataset"]]
196 # Name of processing type / versions
197 dq_t["proc"] = single_property_process(
198 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["proc"]
199 )
200 except KeyError:
201 print("Problem in dataset configuration - dataset step")
202 sys.exit()
203 try:
204 # List of detectors per site - Single string
205 dq_t["det"] = all_site_process(
206 dq_t["dataset"],
207 dq_t["dataset_parent"],
208 toml_ds["site_all_dets"],
209 dq_t["site"].split(" ")
210 )
211 for i_site in dq_t["det"]:
212 if dq_t["det"][i_site] != "":
213 dq_t["det"][i_site] = dq_t["det"][i_site].split(" ")
214 else:
215 dq_t["det"][i_site] = []
216 except KeyError:
217 print("Problem in dataset configuration - detector step")
218 sys.exit()
219 try:
220 # Run type - Detector dependant -> specific processing
221 # Empty for v16 and v17 as did not contribute to the naming of file
222 dq_t["run_type"] = all_dets_process(
223 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["run_type"], dq_t["det"]
224 )
225 # QAQC version - Detector dependant -> specific processing
226 # )
227 dq_t["qaqc_version"] = all_dets_process(
228 dq_t["dataset"],
229 dq_t["dataset_parent"],
230 toml_ds["qaqc_version"],
231 dq_t["det"]
232 )
233 # Compile the run type and qaqc version to get the filename
234 dq_t["qaqc_name"] = {}
235 for i_det in dq_t["run_type"]:
236 dq_t["qaqc_name"][i_det] = (
237 f"{dq_t['qaqc_version'][i_det]}_{dq_t['run_type'][i_det]}"
238 )
239 except KeyError:
240 print("Problem in dataset configuration - type/version step")
241 sys.exit()
242 try:
243 # Run range - Detector dependant -> specific processing
244 dq_t["run_range"] = all_dets_process(
245 dq_t["dataset"], dq_t["dataset_parent"], toml_ds["run_range"], dq_t["det"]
246 )
247 except KeyError:
248 print("Problem in dataset configuration - run-range step")
249 sys.exit()
251 # Retrieve the processing type/versions from the processing.toml file
252 # Detector dependant -> specific processing defined in dataset
253 try:
254 toml_proc = read_configuration("processing", source)
256 dq_t["proc_descr"] = toml_proc["descr"][dq_t["proc"]]
257 dq_t["proc_type"] = all_dets_process(
258 dq_t["proc"], "", toml_proc["proc_type"], dq_t["det"]
259 )
260 dq_t["proc_version"] = all_dets_process(
261 dq_t["proc"], "", toml_proc["proc_version"], dq_t["det"]
262 )
263 except KeyError:
264 print("Problem in processing configuration - run-range step")
265 sys.exit()
267 # Retrieve the good run list definition (name and definition)
268 # The threshold retrieval is completed in the script configure_var_thresholds
269 try:
270 toml_grl = read_configuration("grl", source)
272 dq_t["grl_descr"] = toml_grl["descr"][dq_t["grl"]]
273 grl_tag_parent = toml_grl["parent"][dq_t["grl"]]
274 if grl_tag_parent == "":
275 dq_t["grl_name"] = toml_grl["grl_name"][dq_t["grl"]]
276 else:
277 dq_t["grl_name"] = toml_grl["grl_name"][grl_tag_parent]
278 for i_name in dq_t["grl_name"].keys():
279 if i_name in toml_grl["grl_name"][dq_t["grl"]].keys():
280 dq_t["grl_name"][i_name] = toml_grl["grl_name"][dq_t["grl"]][i_name]
281 except KeyError:
282 print("Problem in grl configuration")
283 sys.exit()
285 return dq_t
288###############################################################################
289def configure_defect(source="sftp"):
290 """
291 TOMl configuration of the defects
293 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/),
294 either the directory absolute-path.
295 """
296 toml = read_configuration("defect", source)
298 return {
299 "descr": toml["descr"],
300 "type": list(toml["descr"].keys()),
301 "bit": toml["bit"]
302 }
305###############################################################################
306def configure_def_var_name(v_prop):
307 """Description of defect variables"""
309 # QAQC variables
310 v_prop["description"]["def_daq"] = "DAQ defects"
311 v_prop["description"]["def_operation"] = "Operation defects"
312 v_prop["description"]["def_data_corruption"] = "Data corruption defects"
313 v_prop["description"]["def_calibration"] = "Calibration defects"
314 v_prop["description"]["def_data_processing"] = "Data processing defects"
315 v_prop["description"]["def_analysis"] = "Analysis defects"
316 v_prop["description"]["def_high_level_analysis"] = "Neutrino analysis defects"
317 v_prop["description"]["def_signoff"] = "Missing signoff"
320###############################################################################
321def configure_fact(source="sftp"):
322 """
323 TOMl configuration of the facts
325 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/),
326 either the directory absolute-path.
327 """
328 toml = read_configuration("fact", source)
330 return {"type": toml["type"]}
333###############################################################################
334def configure_det_fact(source="sftp"):
335 """
336 TOMl configuration of the detector facts
338 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/),
339 either the directory absolute-path.
340 """
341 toml = read_configuration("det_fact", source)
343 return {
344 "type": toml["type"],
345 "status": toml["status"],
346 }
349###############################################################################
350def configure_var_unit(v_prop):
351 """Description of variables"""
352 v_prop["unit"] = {
353 "livetime": "s",
354 "timestampdiff": "s",
355 "triggerrate": "Hz",
356 "meanrate": "Hz",
357 "rmsrate": "Hz",
358 }
361###############################################################################
362def configure_var_bit(v_prop):
363 """
364 Bit of variables contributing to the veto//Qscore degradation
365 used to store in the TTree
366 """
367 v_prop["bit"] = {
368 "livetime_s": 0,
369 "timestampdiff": 1,
370 "timestampdiff_r": 2,
371 "triggerrate": 3,
372 "HRV": 4,
373 "DAQ": 5,
374 "WR": 6,
375 "FIFO": 7,
376 "PMTs": 8,
377 "pmts_norm": 9,
378 "MEAN_Rate_Hz": 10,
379 "RMS_Rate_Hz": 11,
380 "out_sync": 12,
381 "out_usync": 13,
382 "JDAQJTProc": 14,
383 "Acoustics": 15,
384 "zero_AHRS": 16,
385 "ahrs_per_mn": 17,
386 "nb_of_meta": 18,
387 "hrv_fifo_failures": 19,
388 "duplic_timeslices": 20,
389 "HV_check": 21,
390 "def_daq": 23,
391 "def_operation": 24,
392 "def_data_corruption": 25,
393 "def_calibration": 26,
394 "def_data_processing": 27,
395 "def_analysis": 28,
396 "def_high_level_analysis": 29,
397 "def_signoff": 30,
398 }
401###############################################################################
402def configure_var_thresholds(v_prop, det, dqt, source="sftp"):
403 """
404 Configure various variables for the analysis of the TTree derived from
405 the JQAQC file
406 TOMl configuration
408 - source: either "sftp" (hosted in https://sftp.km3net.de/data/km3dq_lw_db/Common/),
409 either the directory absolute-path.
410 """
411 toml_grl = read_configuration("grl", source)
412 toml_perf_aux = read_configuration("km3dq_perf_aux", source)
414 grl_tag_parent = toml_grl["parent"][dqt["grl"]]
416 if grl_tag_parent == "":
417 v_prop["veto_thresholds"] = dict(toml_grl["veto_thresholds"][dqt["grl"]])
418 v_prop["qsco_thresholds"] = dict(toml_grl["qsco_thresholds"][dqt["grl"]])
419 v_prop["good_thresholds"] = dict(toml_grl["good_thresholds"][dqt["grl"]])
420 v_prop["poor_thresholds"] = dict(toml_grl["poor_thresholds"][dqt["grl"]])
421 v_prop["bad_thresholds"] = dict(toml_grl["bad_thresholds"][dqt["grl"]])
422 v_prop["discard_thresholds"] = dict(toml_grl["discard_thresholds"][dqt["grl"]])
423 else:
424 v_prop["veto_thresholds"] = dict(toml_grl["veto_thresholds"][grl_tag_parent])
425 v_prop["qsco_thresholds"] = dict(toml_grl["qsco_thresholds"][grl_tag_parent])
426 v_prop["good_thresholds"] = dict(toml_grl["good_thresholds"][grl_tag_parent])
427 v_prop["poor_thresholds"] = dict(toml_grl["poor_thresholds"][grl_tag_parent])
428 v_prop["bad_thresholds"] = dict(toml_grl["bad_thresholds"][grl_tag_parent])
429 v_prop["discard_thresholds"] = dict(
430 toml_grl["discard_thresholds"][grl_tag_parent]
431 )
433 for i_type in (
434 "veto_thresholds",
435 "qsco_thresholds",
436 "good_thresholds",
437 "poor_thresholds",
438 "bad_thresholds",
439 "discard_thresholds",
440 ):
441 if dqt["grl"] in toml_grl[i_type].keys():
442 for i_modif in toml_grl[i_type][dqt["grl"]]:
443 # If empty list, remove the thresholds
444 # defined in quasi_online dqt['grl']
445 if len(toml_grl[i_type][dqt["grl"]][i_modif]) == 0:
446 v_prop[i_type].pop(i_modif)
447 continue
448 # Otherwise update them
449 v_prop[i_type][i_modif] = toml_grl[i_type][dqt["grl"]][i_modif]
451 # DUMMY detector used only by the get_grl_html_descr function
452 # used by km3dq_perf/generatewww.py - No impact on perf
453 if det == "DUMMY":
454 return
456 # Detector dependant treatment
457 try:
458 # Display options - Default value
459 v_prop["basic_plots_options"] = dict(
460 toml_perf_aux["basic_plots_options"]["default"]
461 )
462 v_prop["basic_plots_y_range_display"] = dict(
463 toml_perf_aux["basic_plots_y_range_display"]["default"]
464 )
465 # Default value overriden by detector specific values
466 if "ORCA" in det: # Increased range for bioluminescence
467 v_prop["basic_plots_y_range_display"]["HRV"] = [-0.1, 0.8]
469 try:
470 for i_var, i_var_range in toml_perf_aux["basic_plots_y_range_display"][det].items():
471 v_prop["basic_plots_y_range_display"][i_var] = i_var_range
472 except KeyError:
473 pass
475 # PMTS treatment to be removed at some point??
476 v_prop["basic_plots_y_range_display"]["PMTs"][1] = (
477 get_detx_caract(det)["pmts"] * 1.2
478 )
479 except KeyError:
480 print("config_library.py / configure_var_thresholds: "
481 "Problem in setting y-range display")
484###############################################################################
485def get_grl_html_descr(dq_tag_0):
486 """
487 Returns the grl description in an html for the summary webpage
488 The grl may depend on the detector due to missing variables. This is
489 by-passed by passing a dummy value to the configure_var_thresholds
490 function. In this way, no variable is discarded
491 """
493 var_prop = {}
494 configure_var_thresholds(var_prop, "DUMMY", dq_tag_0)
495 configure_var_unit(var_prop)
496 html_descr = {}
497 for i_key in var_prop.keys():
498 # One assumes that the QQC variables arrive before the defects
499 # in the configuration file
500 html_descr[i_key] = "<b>QAQC variables</b><br>"
501 defect_found = False
502 for i_var in var_prop[i_key].keys():
503 # qaqc variable
504 if any(
505 (
506 isinstance(var_prop[i_key][i_var][0], int),
507 isinstance(var_prop[i_key][i_var][0], float),
508 )
509 ):
510 if var_prop[i_key][i_var][0] == var_prop[i_key][i_var][1]:
511 thresh = f"{var_prop[i_key][i_var][0]}"
512 if i_var == "veto":
513 thresh = f"{var_prop[i_key][i_var][0] == 1}"
514 else:
515 thresh = (
516 f"{var_prop[i_key][i_var][0]} — "
517 f"{var_prop[i_key][i_var][1]}"
518 )
519 if i_var in var_prop["unit"].keys():
520 thresh += f" {var_prop['unit'][i_var]}"
522 html_descr[i_key] += f"{i_var}: {thresh}"
523 # defect
524 elif isinstance(var_prop[i_key][i_var][0], str):
525 if defect_found is False:
526 html_descr[i_key] += "<hr><b>Defects</b><br>"
527 defect_found = True
528 html_descr[i_key] += (
529 f"{i_var.replace('def_', '')}: " f"{var_prop[i_key][i_var][0]}"
530 )
531 for j in range(1, len(var_prop[i_key][i_var])):
532 html_descr[i_key] += f", {var_prop[i_key][i_var][j]}"
534 html_descr[i_key] += "<br>"
536 return html_descr