Coverage for src/km3dq_common/config_library.py: 0%
179 statements
« prev ^ index » next coverage.py v7.6.1, created at 2024-09-25 11:58 +0000
1#! /usr/bin/env python
2###############################################################################
3# Definition of QAQC variable, defects, data-quality tags...
4#
5# Developer: Benjamin Trocme (benjamin.trocme at apc.in2p3.fr) - 2023
7import tomli
8import sys
10from km3dq_common import config_path
12#####################################
13# Definition of the current detectors
14WEEKLY_DETECTORS = {"ORCA": "D0ORCA023",
15 "ARCA": "D0ARCA028"}
16WEEKLY_TAGS = ["default",
17 "calibration"]
20###############################################################################
21def get_detx_caract(det):
22 """ Return the number of DOMs (incl base module so far) and PMts """
24 all_caract = {}
25 all_caract['D0ARCA028'] = {"doms": 504,
26 "pmts": 14341}
27 all_caract['D0ARCA021'] = {"doms": 378,
28 "pmts": 11718}
29 all_caract['D0ARCA020'] = {"doms": 342,
30 "pmts": 10602}
31 all_caract['D0ARCA009'] = {"doms": 162,
32 "pmts": 5022}
34 all_caract['D0ORCA023'] = {"doms": 414,
35 "pmts": 11698}
36 all_caract['D1ORCA019'] = {"doms": 342,
37 "pmts": 9643}
38 all_caract['D0ORCA018'] = {"doms": 324,
39 "pmts": 10044}
40 all_caract['D1ORCA015'] = {"doms": 270,
41 "pmts": 8370}
42 all_caract['D0ORCA015'] = {"doms": 270,
43 "pmts": 8370}
44 all_caract['D1ORCA011'] = {"doms": 198,
45 "pmts": 6138}
46 all_caract['D0ORCA011'] = {"doms": 198,
47 "pmts": 6138}
48 all_caract['D1ORCA013'] = {"doms": 234,
49 "pmts": 7254}
50 all_caract['D0ORCA007'] = {"doms": 126,
51 "pmts": 3906}
52 all_caract['D0ORCA010'] = {"doms": 180,
53 "pmts": 5580}
54 all_caract['D_ORCA006'] = {"doms": 108,
55 "pmts": 3348}
57 return all_caract[det]
60###############################################################################
61def configure_var_name(v_prop):
62 """ Description of JQAQC variables """
64 # QAQC variables
65 v_prop['description'] = {"run": "Run",
66 "livetime": "Livetime (s)",
67 "nb_of_meta": "Nb of metadata",
68 "timestampdiff": "Missing time-slices",
69 "timestampdiff_r": ("Missing time-slices"
70 "/Run duration"),
71 "triggerrate": "Trigger rate(Hz)",
72 "hrv": "High rate veto bit",
73 "daq": "UDP packet-loss (DAQ)",
74 "whiterabbitstatus": "White rabbit status",
75 "fifo": "FIFO almost full",
76 "pmts": "Number of PMTs",
77 "pmts_norm": "Normalized number of PMTs",
78 "meanrate": "Mean rate (Hz)",
79 "rmsrate": "RMS rate (Hz)",
80 "hrv_fifo_failures": "Number of hrv/fifo failures",
81 "duplic_timeslices": "Number of duplicated timeslices",
82 "oos": "Out-of-sync",
83 "out_usync": "Micro out-of-sync",
84 "JDAQJTProc": "Triggered != Retriggered",
85 "acoustics": "Acoustic rate",
86 "ahrs_per_mn": ("Number of AHRS event per minute "
87 "in > 80% of DOMs"),
88 "jdaqevent": "Total",
89 "jtrigger3dshower": "3D shower trigger",
90 "jtriggermxshower": "MX shower trigger",
91 "jtrigger3dmuon": "3D Muon trigger "}
94###############################################################################
95def configure_dataquality_tag(tag):
96 """
97 TOMl configuration of the data quality tags
98 === Arguments ===
99 - tag : tag name - [string] - Ex: "default", "neutrino2024"...
100 === Output ===
101 - Dictionnary with all tag characteristics for all detectors
102 """
103 ####################################################
104 def all_dets_process(tag, toml_input, det_list):
105 # det_list: dictionnary with keys ARCA, ORCA
106 output = {}
108 for i_site in ["ARCA", "ORCA"]:
109 for i_det in det_list[i_site].split(" "):
110 if tag in toml_input.keys():
111 if i_det in toml_input[tag]: # Detector explicitly defined
112 output[i_det] = toml_input[tag][i_det]
113 else: # Use the site definition
114 output[i_det] = toml_input[tag][i_site]
115 else:
116 output[i_det] = "N/A"
118 return output
120 try:
121 with open(config_path("dataquality_tag"), mode="rb") as fconfig:
122 toml = tomli.load(fconfig)
124 dq_t = {}
126 # The path directory is the name of the data-quality tag
127 dq_t['dir_path'] = tag
129 # Retrieve the general descriptions
130 dq_t['name'] = toml['short_descr'][tag]
131 dq_t['descr'] = toml['descr'][tag]
132 dq_t['status'] = toml['status'][tag]
134 # Retrieve the dataset from the dataset.toml file
135 dq_t['dataset'] = toml['dataset'][tag]
137 except:
138 print("Problem in dataquality_tag.toml file")
139 sys.exit()
141 try:
142 with open(config_path("dataset"), mode="rb") as fconfig_ds:
143 toml_ds = tomli.load(fconfig_ds)
144 dq_t['dataset_descr'] = toml_ds['descr'][dq_t['dataset']]
146 # QAQC version - Detector dependant -> specific processing
147 dq_t['qaqc_vers'] = all_dets_process(dq_t['dataset'],
148 toml_ds['qaqc_vers'],
149 toml_ds['site_all_dets'][dq_t['dataset']])
150 # Extract the list of available detectors based on the
151 # definition of the QAQC-file versions and the site_all_dets variable
152 dq_t['det'] = {"ORCA": [],
153 "ARCA": []}
154 for i_det in dq_t['qaqc_vers'].keys():
155 if "ORCA" in i_det and i_det in toml_ds['site_all_dets'][dq_t['dataset']]['ORCA']:
156 dq_t['det']['ORCA'].append(i_det)
157 elif "ARCA" in i_det and i_det in toml_ds['site_all_dets'][dq_t['dataset']]['ARCA']:
158 dq_t['det']['ARCA'].append(i_det)
160 # Retrieve the defect tag
161 dq_t['def_tag'] = toml['def_tag'][tag]
162 except:
163 print("Problem in dataset configuration")
164 sys.exit()
166 # Retrieve the processing type/versions from the processing.toml file
167 # Detector dependant -> specific processing
168 try:
169 dq_t['proc'] = toml_ds['proc'][dq_t['dataset']]
170 with open(config_path("processing"), mode="rb") as fconfig_proc:
171 toml_proc = tomli.load(fconfig_proc)
172 dq_t['proc_descr'] = toml_proc['descr'][dq_t['proc']]
173 dq_t['proc_type'] = all_dets_process(dq_t['proc'],
174 toml_proc['proc_type'],
175 toml_ds['site_all_dets'][dq_t['dataset']])
176 dq_t['proc_version'] = all_dets_process(dq_t['proc'],
177 toml_proc['proc_version'],
178 toml_ds['site_all_dets'][dq_t['dataset']])
179 except:
180 print("Problem in processing configuration")
181 sys.exit()
183 # Retrieve the good run list definition
184 try:
185 dq_t['grl'] = toml['grl'][tag]
186 # Open the grl.toml file to retrieve the grl description
187 with open(config_path("grl"), mode="rb") as fconfig_grl:
188 toml_grl = tomli.load(fconfig_grl)
189 dq_t['grl_descr'] = toml_grl['descr'][dq_t['grl']]
190 # GRL names - If not defined, use the quasi_online one.
191 if dq_t['grl'] in toml_grl['grl_name'].keys():
192 dq_t['grl_name'] = toml_grl['grl_name'][dq_t['grl']]
193 else:
194 dq_t['grl_name'] = toml_grl['grl_name']['quasi_online']
195 except:
196 print("Problem in grl configuration")
197 sys.exit()
199 return dq_t
202###############################################################################
203def configure_defect():
204 """
205 TOMl configuration of the defects
206 """
207 with open(config_path("defect"), mode="rb") as fconfig:
208 toml = tomli.load(fconfig)
210 defect_0 = {}
212 defect_0['descr'] = toml['descr']
213 defect_0['type'] = list(defect_0['descr'].keys())
214 defect_0['bit'] = toml['bit']
216 return defect_0
219###############################################################################
220def configure_def_var_name(v_prop):
221 """ Description of defect variables """
223 # QAQC variables
224 v_prop['description']["def_daq"] = "DAQ defects"
225 v_prop['description']["def_operation"] = "Operation defects"
226 # v_prop['description']["def_oos"] = "Out-of-sync defects"
227 v_prop['description']["def_data_corruption"] = "Data corruption defects"
228 v_prop['description']["def_calibration"] = "Calibration defects"
229 v_prop['description']["def_data_processing"] = "Data processing defects"
230 v_prop['description']["def_analysis"] = "Analysis defects"
231 v_prop['description']["def_high_level_analysis"] = "Neutrino analysis defects"
232 v_prop['description']["def_signoff"] = "Missing signoff"
235###############################################################################
236def configure_fact():
237 """
238 TOMl configuration of the facts
239 """
240 with open(config_path("fact"), mode="rb") as fconfig:
241 toml = tomli.load(fconfig)
243 fact_0 = {}
245 fact_0['type'] = toml['type']
247 return fact_0
250###############################################################################
251def configure_det_fact():
252 """
253 TOMl configuration of the detector facts
254 """
255 with open(config_path("det_fact"), mode="rb") as fconfig:
256 toml = tomli.load(fconfig)
258 det_fact_0 = {}
260 det_fact_0['type'] = toml['type']
261 det_fact_0['status'] = toml['status']
263 return det_fact_0
266###############################################################################
267def configure_var_unit(v_prop):
268 """ Description of variables """
269 v_prop['unit'] = {"livetime": "s",
270 "timestampdiff": "s",
271 "triggerrate": "Hz",
272 "meanrate": "Hz",
273 "rmsrate": "Hz"}
276###############################################################################
277def configure_var_bit(v_prop):
278 """
279 Bit of variables contributing to the veto//Qscore degradation
280 used to store in the TTree
281 """
282 v_prop['bit'] = {"livetime": 0,
283 "timestampdiff": 1,
284 "timestampdiff_r": 2,
285 "triggerrate": 3,
286 "hrv": 4,
287 "daq": 5,
288 "whiterabbitstatus": 6,
289 "fifo": 7,
290 "pmts": 8,
291 "pmts_norm": 9,
292 "meanrate": 10,
293 "rmsrate": 11,
294 "oos": 12,
295 "out_usync": 13,
296 "JDAQJTProc": 14,
297 "acoustics": 15,
298 "ahrs_per_mn": 16,
299 "nb_of_meta": 17,
300 "hrv_fifo_failures": 18,
301 "duplic_timeslices": 19,
302 "def_daq": 22,
303 "def_operation": 23,
304 # "def_oos": 24,
305 "def_data_corruption": 25,
306 "def_calibration": 26,
307 "def_data_processing": 27,
308 "def_analysis": 28,
309 "def_high_level_analysis": 29,
310 "def_signoff": 30}
313###############################################################################
314def configure_var_thresholds(v_prop, det, dqt):
315 """
316 Configure various variables for the analysis of the TTree derived from
317 the JQAQC file
318 TOMl configuration
319 """
320 with open(config_path("grl"), mode="rb") as fconfig:
321 toml = tomli.load(fconfig)
323 # First fill the thresholds with the quasi_online ones
324 # It is mandatory as all thresholds are derived from them
325 v_prop['veto_thresholds'] = dict(toml['veto_thresholds']['quasi_online'])
326 v_prop['qsco_thresholds'] = dict(toml['qsco_thresholds']['quasi_online'])
327 v_prop['good_thresholds'] = dict(toml['good_thresholds']['quasi_online'])
328 v_prop['poor_thresholds'] = dict(toml['poor_thresholds']['quasi_online'])
329 v_prop['bad_thresholds'] = dict(toml['bad_thresholds']['quasi_online'])
330 v_prop['discard_thresholds'] = dict(toml['discard_thresholds']['quasi_online'])
331 v_prop['y_range_display'] = dict(toml['y_range_display']['quasi_online'])
333 # If non quasi_online GRL, update the thresholds with the config file
334 for i_type in ("veto_thresholds", "qsco_thresholds",
335 "good_thresholds", "poor_thresholds",
336 "bad_thresholds", "discard_thresholds",
337 "y_range_display"):
338 if dqt['grl'] != "quasi_online" and dqt['grl'] in toml[i_type].keys():
339 for i_modif in toml[i_type][dqt['grl']]:
340 # If empty list, remove the thresholds defined in quasi_online dqt['grl']
341 if len(toml[i_type][dqt['grl']][i_modif]) == 0:
342 v_prop[i_type].pop(i_modif)
343 continue
344 # Otherwise update them
345 v_prop[i_type][i_modif] = \
346 toml[i_type][dqt['grl']][i_modif]
348 # DUMMY detector used only by the get_grl_html_descr function
349 # used by km3dq_perf/generatewww.py - No impact on perf
350 if det == "DUMMY":
351 return
353 # Special cases impossible to handle with TOML
354 if "v16.0.3" in dqt["qaqc_vers"][det]:
355 if "out_usync" in v_prop['veto_thresholds']:
356 v_prop['veto_thresholds'].pop("out_usync")
357 if "out_usync" in v_prop['qsco_thresholds']:
358 v_prop['qsco_thresholds'].pop("out_usync")
359 if any(("v16.0.3" in dqt["qaqc_vers"][det],
360 "v17.3.2" in dqt["qaqc_vers"][det])):
361 for i_missing in ["nb_of_meta", "hrv_fifo_failures", "duplic_timeslices"]:
362 if i_missing in v_prop['veto_thresholds']:
363 v_prop['veto_thresholds'].pop(i_missing)
364 if i_missing in v_prop['qsco_thresholds']:
365 v_prop['qsco_thresholds'].pop(i_missing)
367 # Display options
368 v_prop['y_range_display']['pmts'][1] = get_detx_caract(det)['pmts']*1.2
369 if "ORCA" in det: # Increased range for bioluminescence
370 v_prop['y_range_display']["hrv"] = [-0.1, 0.8]
373###############################################################################
374def get_grl_html_descr(dq_tag_0):
375 """
376 Returns the grl description in an html for the summary webpage
377 The grl may depend on the detector due to missing variables. This is
378 by-passed by passing a dummy value to the configure_var_thresholds
379 function. In this way, no variable is discarded
380 """
382 var_prop = {}
383 configure_var_thresholds(var_prop, "DUMMY", dq_tag_0)
384 configure_var_unit(var_prop)
385 html_descr = {}
386 for i_key in var_prop.keys():
387 # One assumes that the QQC variables arrive before the defects
388 # in the configuration file
389 html_descr[i_key] = "<b>QAQC variables</b><br>"
390 defect_found = False
391 for i_var in var_prop[i_key].keys():
392 # qaqc variable
393 if any((isinstance(var_prop[i_key][i_var][0], int),
394 isinstance(var_prop[i_key][i_var][0], float))):
395 if var_prop[i_key][i_var][0] == var_prop[i_key][i_var][1]:
396 thresh = f"{var_prop[i_key][i_var][0]}"
397 if i_var == "veto":
398 thresh = f"{var_prop[i_key][i_var][0] == 1}"
399 else:
400 thresh = (f"{var_prop[i_key][i_var][0]} — "
401 f"{var_prop[i_key][i_var][1]}")
402 if i_var in var_prop['unit'].keys():
403 thresh += f" {var_prop['unit'][i_var]}"
405 html_descr[i_key] += (f"{i_var}: {thresh}")
406 # defect
407 elif isinstance(var_prop[i_key][i_var][0], str):
408 if defect_found is False:
409 html_descr[i_key] += "<hr><b>Defects</b><br>"
410 defect_found = True
411 html_descr[i_key] += (f"{i_var.replace('def_', '')}: "
412 f"{var_prop[i_key][i_var][0]}")
413 for j in range(1, len(var_prop[i_key][i_var])):
414 html_descr[i_key] += (f", {var_prop[i_key][i_var][j]}")
416 html_descr[i_key] += ("<br>")
418 return html_descr