Coverage for src/km3dq_common/common_library.py: 52%
286 statements
« prev ^ index » next coverage.py v7.9.1, created at 2025-06-16 14:13 +0000
1#! /usr/bin/env python
2########################################################################################################
3# Various functions used in km3dq_lw_dq and km3dq_core scripts
4#
5# Developer: Benjamin Trocme (benjamin.trocme at apc.in2p3.fr) - 2023
7import os
8import sys
9import time
10import array
11import re
12import urllib.request
13import km3db
14import pandas as pd
15import numpy as np
16import tomli
18from .config_library import configure_var_bit
19from .config_library import configure_var_thresholds
20from .config_library import get_detx_caract
21from .config_library import configure_dataquality_tag
22from .config_library import configure_defect
23from .lw_db_defect_library import read_defect_file
24from .lw_db_defect_library import decode_defect_diag_byte
25from .lw_db_defect_library import check_defect_diag_byte
27# Types of all variables of all versions except the oos variables of 16.0.3 that
28# are renamed on the fly
29# Variable evolution with versions:
30# - 17.3.2: out-sync/in-sync renamed out_sync/in_sync
31# out_usync/in_usync/event_duration added
32# - 19.2.0: nb_of_meta/hrv_fifo_failures/duplic_timeslices/zero_AHRS/mean_AHRS added
33# AHRS removed
35VARIABLE_TYPE = {
36 "livetime_s": float,
37 "DAQ": float,
38 "WR": float,
39 "FIFO": float,
40 "HRV": float,
41 "PMTs": float,
42 "pmts_norm": float,
43 "timestampdiff": float,
44 "timestampdiff_r": float,
45 "triggerrate": float,
46 "Acoustics": float,
47 "Acoustics_per_mn": float,
48 "MEAN_Rate_Hz": float,
49 "RMS_Rate_Hz": float,
50 "AHRS": float,
51 "ahrs_per_mn": float,
52 "mean_AHRS": float,
53 "JDAQJTProc": float,
54 "kton_year": float,
55 "UTCMin_s": float,
56 "UTCMax_s": float,
57 "detector": int,
58 "run": int,
59 "nb_of_meta": int,
60 "trigger3DMuon": int,
61 "triggerMXShower": int,
62 "trigger3DShower": int,
63 "triggerNB": int,
64 "JTriggerReprocessor": int,
65 "JTrigger3dmuon": int,
66 "JTriggermxshower": int,
67 "JTrigger3dshower": int,
68 "JTriggernb": int,
69 "writeL0": int,
70 "writeL1": int,
71 "writeL2": int,
72 "writeSN": int,
73 "JDAQTimeslice": int,
74 "JDAQTimesliceL0": int,
75 "JDAQTimesliceL1": int,
76 "JDAQTimesliceL2": int,
77 "JDAQTimesliceSH": int,
78 "JDAQSummaryslice": int,
79 "JDAQEvent": int,
80 "hrv_fifo_failures": int,
81 "duplic_timeslices": int,
82 "in_sync": int,
83 "zero_AHRS": int,
84 "out_sync": int,
85 "in_usync": int,
86 "out_usync": int,
87 "hv_inconsist":int
88 }
90# Dummy value used when the variable is missing in a run/QAQC file
91MISSING_VALUE = -999999
93########################################################################################################
94def get_det_id(det):
95 """
96 Returns the det id
98 === Input:
99 - det [string]: D1ORCA024, D0ARCA030...
100 === Outputs:
101 - detector id [integer]
102 """
104 return km3db.tools.todetid(det)
107########################################################################################################
108def get_current_detector(site):
109 """
110 Extract the current detector names from the database by checking the highest run number
111 By default, it is retrieved from the KM3NeT DB but in some rare cases (typically at a detector
112 change) it can be forced to a different one by modifying the km3dq_lw_db/Common/current_detector.toml
113 file that is available on sftp.
115 === Input:
116 - site [string]: ARCA / ORCA
117 === Outputs:
118 - detector [string]: D1ORCA024, D0ARCA030...
119 """
121 # First check if the current detector is hardcoded in the current_detector.toml file
122 # Facility especially useful at the detector change to force keeping a detector in Coral
123 with urllib.request.urlopen(
124 "https://sftp.km3net.de/data/km3dq_lw_db/Common/current_detector.toml"
125 ) as s_f:
126 toml = tomli.loads(s_f.read().decode("utf-8"))
128 if toml[site] != "":
129 return toml[site]
131 # If not -> Retrieve it from the database
132 sds = km3db.tools.StreamDS()
134 if site == "ARCA":
135 res = sds.detectors(city="Italy").split("\n")
136 else:
137 res = sds.detectors(city="France").split("\n")
139 max_run = 0
141 for i_det in res[1:]:
142 try:
143 if int(i_det.split("\t")[5]) > max_run:
144 o_det = i_det.split("\t")[0]
145 if o_det.startswith("D") is False:
146 continue
147 max_run = int(i_det.split("\t")[5])
148 except:
149 print("Improper db reply in get_current_detectors")
151 return o_det
154########################################################################################################
155def get_file_paths(detector, qaqc_name):
156 """
157 For a given detector / detector, returns the path for QAQC files both on sps and sftp
159 === Input:
160 - det [string]: D1ORCA024, D0ARCA030...
161 === Outputs:
162 - detector id [integer]
163 """
165 return {
166 "JQAQC_sps": (
167 "/sps/km3net/repo/data/raw/quality/"
168 f"KM3NeT_{km3db.tools.todetid(detector):08d}_QAQC_Jpp_{qaqc_name}.txt"
169 ),
170 "JQAQC_sftp":(
171 "https://sftp.km3net.de/data/quality/"
172 f"KM3NeT_{km3db.tools.todetid(detector):08d}_QAQC_Jpp_{qaqc_name}.txt"
173 )
174 }
177########################################################################################################
178def get_run_properties_from_db(det, filt="PHYS", filt_target="Run"):
179 """
180 Retrieve run properties from the database with the km3db package
182 === Inputs:
183 - det [string]: D1ORCA024, D0ARCA030...
184 - filt [string]: accepted run types (PHYS, COMM...) separated by a space - if == "", no filter
185 - filt_target [string]: accepted target (run, on...) separated by a space - if == "", no filter
186 === Outputs:
187 - properties [dict of dict]: all DB content in a single dictionnary with the run numbers as key
188 """
190 runs = km3db.StreamDS(container="nt").get("runs", detid=det)
192 results = {}
194 for i_run in runs:
195 if filt != "":
196 if all((i_run.runsetupname[0:4] not in filt.split(" "),
197 i_run.runsetupname[0:5] not in filt.split(" "))):
198 continue
200 if filt_target != "":
201 if i_run.jobtarget not in filt_target.split(" "):
202 continue
204 results[i_run.run] ={
205 'UNIXSTARTTIME': i_run.unixstarttime,
206 'STARTTIME_DEFINED': i_run.starttime_defined,
207 'RUNSETUPID': i_run.runsetupid,
208 'RUNSETUPNAME': i_run.runsetupname,
209 'T0_CALIBSETID': i_run.t0_calibsetid,
210 'POS_CALIBSETID': i_run.pos_calibsetid,
211 'ROT_CALIBSETID': i_run.rot_calibsetid,
212 'RUNJOBID': i_run.runjobid,
213 'JOBTARGET': i_run.jobtarget,
214 'JOBPRIORITY': i_run.jobpriority,
215 'UNIXJOBSTART': i_run.unixjobstart,
216 'UNIXJOBEND': i_run.unixjobend,
217 }
219 return results
222########################################################################################################
223def get_run_properties_from_qaqc(
224 det, dq_tag, origin="qaqc_sftp", startrun=0, endrun=1e9
225):
226 """
227 For a given detector, returns the run numbers, their lengths, time counter... as retrieved from the
228 QAQC file
230 === Inputs:
231 - det [string]: D1ORCA024, D0ARCA030...
232 - dq_tag [DQ tag]: as configured in config_library
233 - origin [string]: "qaqc_sftp" or "qaqc_sps"
234 - startrun [integer]: first run
235 - endrun [integer]: last run
236 === Outputs:
237 - run properties: TOBEDETAILED
238 - error: should be empty
239 """
241 runprop = {
242 "runNumber": [],
243 "lvt": [],
244 "lvt_counter": [],
245 "kty": [],
246 "kty_counter": [],
247 "nbRuns": 0,
248 "minRun": 1e10,
249 "maxRun": 0,
250 }
252 kty_counter = 0
253 err_log = ""
255 (n1, err_log) = create_ttree_from_qaqc(
256 det, ["run", "livetime_s", "kton_year"], origin, dq_tag
257 )
259 if len(n1) == 0:
260 return (runprop, "No data in qaqc")
262 if all((startrun != 0, endrun != 1e9)):
263 n1 = n1[(n1["run"] >= startrun) & (n1["run"] <= endrun)]
265 runprop = {
266 "runNumber": n1.run.to_list(),
267 "lvt": (n1.livetime_s / 3600 / 24).to_list(),
268 "kty": n1.kton_year.to_list(),
269 "nbRuns": len(n1),
270 "minRun": int(n1.min()["run"]),
271 "maxRun": int(n1.max()["run"]),
272 }
274 runprop["lvt_counter"] = [0] + (n1.cumsum()["livetime_s"] / 3600 / 24).to_list()
275 runprop["kty_counter"] = [0] + (n1.cumsum()["kton_year"]).to_list()
277 return (runprop, err_log)
280########################################################################################################
281def get_run_timerange(det, filt="PHYS"):
282 """
283 Retrieve the run start/stop by combining db and QAQC quantities to get the most accurate timing.
284 If QAQC data exist (normally for all PHYS runs), use them, otherwise use the DB ones
285 So far used only in km3net_lw_db_interface
287 === Inputs:
288 - det: string like (e.g: D0ARCA030, D1ORCA024...)
289 - filt: string containing the accepted run type (PHYS, COMM...) separated by a space
291 === Outputs:
292 - dictionary with run as key and a list of [run_start, run_stop, source, runtype] with
293 source = 0 (QAQC) or 1 (DB only)
294 """
296 # First retrieve the QAQC variables that will be used to get the exact
297 # run end
298 df, _ = create_ttree_from_qaqc(
299 det, ["run", "livetime_s"], "qaqc_sftp", configure_dataquality_tag("operation")
300 )
301 results = dict([(r, [s, e, 0]) for r, s, e in zip(df["run"], df["UTCMin_s"], df["UTCMax_s"])])
303 # Then retrieve all runs from the database to get the remaining runs
304 runs = km3db.StreamDS(container="nt").get("runs", detid=det)
306 for i_run in runs:
307 i_run_runtype = i_run.runsetupname.split(".")[0]
308 if i_run_runtype not in filt:
309 continue
311 if i_run.run in results: # Timerange already retrieved from the QAQC data - Append run type
312 results[i_run.run].append(i_run_runtype)
313 continue
315 try:
316 results[i_run.run] = [i_run.unixjobstart/1000., i_run.unixjobend/1000., 1, i_run_runtype]
317 except TypeError:
318 pass
320 return results
323########################################################################################################
324def get_last_n_weeks_runs_qaqc(det, nweeks=2):
325 """Extract the all runs acquired in the last nweeks of running"""
327 # QAQC default tag used. As priori OK for this purpose
328 dq_tag = configure_dataquality_tag("operation")
329 min_run = 1e9
330 max_run = 0
332 try:
333 lines = read_qaqc_file(det, "qaqc_sftp", dq_tag["qaqc_name"][det], "")
334 ten_days = (nweeks * 7 + 1) * 3600.0 * 24.0
335 for i_run in range(len(lines["run"])):
336 if (time.time() - float(lines["UTCMin_s"][i_run])) < ten_days:
337 run_number = int(lines["run"][i_run])
338 if run_number < min_run:
339 min_run = run_number
340 if run_number > max_run:
341 max_run = run_number
342 except KeyError:
343 print(f"common_library.py: Missing DQ-tag information for the detector {det}")
345 return (min_run, max_run)
348########################################################################################################
349def get_site(det):
350 """
351 Returns the site (ORCA or ARCA)
353 === Inputs:
354 - det [string]: D1ORCA024, D0ARCA030...
355 === Ouputs:
356 - site [string]: ARCA/ORCA
357 """
359 # A/O detectors corresponds to the current ARCA/ORCA detectors
360 if "ORC" in det or det == "O":
361 out = "ORCA"
362 else:
363 out = "ARCA"
364 return out
367########################################################################################################
368def get_active_dus_range(det):
369 """
370 Retrieve the range of active DUs
371 To be improved with database information
373 === Inputs:
374 - det [string]: D1ORCA024, D0ARCA030...
375 === Ouputs:
376 - site [string]: ARCA/ORCA
377 """
378 lower_du = 0
379 upper_du = 48
381 return (lower_du, upper_du)
384########################################################################################################
385def get_nb_qaqc_variables(qaqc_vers):
386 """
387 Retrieve the number of variables in the QAQC file
389 === Inputs
390 - qaqc_vers [string]: "v16.0.3", "v17.3.2", "V19.2.0"...
392 === Output ===
393 - Number of QAQC variables [integer]
395 """
396 if qaqc_vers in ("v16.0.3", "16.0.3"):
397 return 39
398 if qaqc_vers in ("v17.3.2", "17.3.2", "18.0.0-rc.4-34-g8618072aa-D"):
399 return 42
400 if qaqc_vers in ("v19.2.0", "19.1.0", "19.2.0"):
401 return 46
402 if qaqc_vers in ("19.3.0-rc.2", "19.3.0-rc.5"):
403 return 46
404 if qaqc_vers == "19.3.0-rc.x":
405 return 47
407 print(f"Unknwon qaqc version {qaqc_vers} -> Exiting")
408 sys.exit()
411########################################################################################################
412def create_ttree_from_qaqc(det, var_to_fill, source, tag, append_veto_qsco=False):
413 """
414 Create a ttree from qaqc sftp file and defect variables stored on git
415 It includes some advanced check about the QAQC file integrity.
417 === Inputs
418 - det : detector name - [string] - Ex: "D0ARCA021", "D0ORCA018"...
419 - var_to_fill : QAQC variables or defect to fill - [array of string] -
420 Ex: ['run', 'timestampdiff', 'def_operation', 'def_oos']
421 Obsolete - To be removed at some point
422 - source : QAQC source, a priori "qaqc_sftp" - [string]
423 - tag : data-quality tag (not its name) as created by
424 configure_dataquality_tag
425 - append_veto_qsco: append the veto and Qscore - [boolean]
426 === Outputs
427 - TTree
428 - Error log
430 """
432 error_log = ""
434 content = read_qaqc_file(det, source, tag["qaqc_name"][det], tag["run_range"][det])
436 df = pd.DataFrame(content)
437 # Renaming required for 16.0.3
438 df = df.rename(columns={"out-sync": "out_sync"})
439 df = df.rename(columns={"in-sync": "in_sync"})
441 # Type definition for each column
442 # df = df.astype({key: float for key in variable_float if key in df.columns})
443 # df = df.astype({key: int for key in variable_int if key in df.columns})
444 for i_var, i_type in VARIABLE_TYPE.items():
445 try:
446 df = df.astype({i_var: i_type})
447 except KeyError: # Variable missing in this version -> Dummy column added
448 df[i_var] = len(df) * [MISSING_VALUE]
450 df["timestampdiff"] = df.UTCMax_s - df.UTCMin_s - df.livetime_s
451 df["timestampdiff_r"] = 1.0 - df.livetime_s / (df.UTCMax_s - df.UTCMin_s)
452 df["pmts_norm"] = df.PMTs / get_detx_caract(det)["pmts"]
453 df["triggerrate"] = df.JDAQEvent / df.livetime_s
455 try: # Version > 19.2.0
456 df["ahrs_per_mn"] = df.mean_AHRS / df.livetime_s * 60
457 except AttributeError: # Version 16.0.3 / 17.3.2
458 df["ahrs_per_mn"] = df.AHRS / df.livetime_s * 60
460 df["Acoustics_per_mn"] = df.Acoustics / df.livetime_s * 60
462 df["JDAQJTProc"] = (df.JDAQEvent - df.JTriggerReprocessor) / (df.JDAQEvent + 1e-10)
463 if "ARC" in det:
464 df["kton_year"] = (
465 1e6
466 / (31 * 18 * 2 * 115)
467 * df.PMTs
468 * (1.0 - df.HRV)
469 * df.livetime_s
470 / (365.2425 * 24 * 3600)
471 )
472 else:
473 df["kton_year"] = (
474 6980
475 / (31 * 18 * 115)
476 * df.PMTs
477 * (1.0 - df.HRV)
478 * df.livetime_s
479 / (365.2425 * 24 * 3600)
480 )
483 # Adding defects
484 defect_results = read_defect_file(det, tag["def_tag"])
486 df = df.sort_values("run")
487 # Check if there are some duplicated lines with the same run number
488 # If some are found exit, as this is not acceptable and have to be cured
489 if True in df.duplicated(subset=["run"], keep=False).values:
490 print("Duplicated lines in create_ttree_from_qaqc\n"
491 "Fix required -> Exiting")
492 print(df.mask(~df.duplicated(subset=["run"], keep=False)).dropna())
493 sys.exit()
495 df.reset_index(drop=True)
497 for key, runs in defect_results.items():
498 value = np.zeros(len(df))
499 if len(runs) != 0:
500 # A defect can be assigned to a run missing in the QAQC file (protection added)
501 # The defects_results is not chronologically ordered (sorting added)
502 mask = np.isin(
503 df.run, [x for x in sorted(runs.keys()) if x in df["run"].values]
504 )
505 value[mask] = [
506 runs[x] for x in sorted(runs.keys()) if x in df["run"].values
507 ]
508 df[key] = value
510 # Adding Veto and Q score:
511 if append_veto_qsco:
512 var_properties = {}
513 configure_var_thresholds(var_properties, det, tag)
514 configure_var_bit(var_properties)
516 df["veto"], df["veto_source"], df["qsco"], df["qsco_source"] = (
517 compute_veto_qscore(var_properties, df)
518 )
520 if error_log != "":
521 print(error_log)
522 return (df, error_log)
525########################################################################################################
526def compute_veto_qscore(var_prop, var_val):
527 """
528 Computes the veto/Q-score for a detector/tag using a QAQC source.
529 """
531 def_config = configure_defect()
533 veto = np.zeros(len(var_val), dtype=bool)
534 veto_source = np.zeros(len(var_val), dtype=int)
536 qsco = np.ones(len(var_val), dtype=float) * len(var_prop["qsco_thresholds"])
537 qsco_source = np.zeros(len(var_val), dtype=int)
539 pd.set_option("display.max_rows", None)
540 for i_var in var_prop["veto_thresholds"]:
541 values = var_val[i_var]
542 # Defect variables
543 # The defect variable is defined as def_[defect type]
544 # with [defect type] = daq, operation, analysis...
545 if i_var.startswith("def_"):
546 # first extract the diagnosis
547 defect_type = i_var.replace("def_", "")
548 diag_array = {"present": [], "absent": []}
549 for i_excluded_diag in var_prop["veto_thresholds"][i_var]:
550 if i_excluded_diag.startswith("NOT."):
551 diag_array["absent"].append(
552 def_config["bit"][defect_type][
553 i_excluded_diag.replace("NOT.", "")
554 ]
555 )
556 else:
557 diag_array["present"].append(
558 def_config["bit"][defect_type][i_excluded_diag]
559 )
561 if len(diag_array["absent"]) == 0: # Only present diagnosis
562 def_mask = values != 0
563 elif len(diag_array["present"]) == 0: # Only NOT. diagnosis
564 def_mask = values == 0
565 else: # Mixture of present/absent -> No mask
566 def_mask = values != (-1)
567 if np.sum(def_mask) == 0:
568 continue
570 defect_test_output = np.vectorize(check_defect_diag_byte, excluded=[1])(
571 values[def_mask], diag_array
572 )
573 # The FutureWarning about the incompatible dtype comes from the line below
574 # When checking explicitly the dtype, both are returned as bool hence
575 # the FutureWarning can not be understood.
576 # The trial to fix with explicit casting and other numpy functions
577 # (e.g veto[def_mask] = np.logical_or(veto[def_mask].astype(bool), defect_test_output.astype(bool)))
578 # did not help at all.
579 veto[def_mask] = veto[def_mask] | defect_test_output
580 veto_source[def_mask] += (0b1 & defect_test_output) << var_prop["bit"][
581 i_var
582 ]
584 else: # QAQC variable: range treatement
585 range_mask = (var_val[i_var] != MISSING_VALUE) & (
586 (var_val[i_var] < var_prop["veto_thresholds"][i_var][0]) | (
587 var_val[i_var] > var_prop["veto_thresholds"][i_var][1]
588 )
589 )
590 veto = veto | range_mask
591 veto_source[range_mask] += 0b1 << var_prop["bit"][i_var]
593 for i_var in var_prop["qsco_thresholds"]:
594 values = var_val[i_var]
595 # Defect variables
596 # Defect variables
597 # The defect variable is defined as def_[defect type]
598 # with [defect type] = daq, operation, analysis...
599 if i_var.startswith("def_"):
600 # first extract the set defects
601 def_mask = values != 0
602 if np.sum(def_mask) == 0:
603 continue
605 defect_type = i_var.replace("def_", "")
606 diag_array = {"present": [], "absent": []}
607 for i_excluded_diag in var_prop["qsco_thresholds"][i_var]:
608 if i_excluded_diag.startswith("NOT."):
609 diag_array["absent"].append(
610 def_config["bit"][defect_type][
611 i_excluded_diag.replace("NOT.", "")
612 ]
613 )
614 else:
615 diag_array["present"].append(
616 def_config["bit"][defect_type][i_excluded_diag]
617 )
619 defect_test_output = np.vectorize(check_defect_diag_byte, excluded=[1])(
620 values[def_mask], diag_array
621 )
622 qsco[def_mask] = qsco[def_mask] | defect_test_output
623 qsco_source[def_mask] += (0b1 & defect_test_output) << var_prop["bit"][
624 i_var
625 ]
627 else: # QAQC variable: range treatement
628 range_mask = (var_val[i_var] != MISSING_VALUE) & (
629 (var_val[i_var] < var_prop["qsco_thresholds"][i_var][0]) | (
630 var_val[i_var] > var_prop["qsco_thresholds"][i_var][1]
631 )
632 )
634 qsco[range_mask] -= 1
635 qsco_source[range_mask] += 0b1 << var_prop["bit"][i_var]
637 qsco /= len(var_prop["qsco_thresholds"])
639 return (veto, veto_source, qsco, qsco_source)
642########################################################################################################
643def log_run_range(det, run_0, run_1, log_file):
644 """Write the run range and run list on disk"""
646 os.environ["TZ"] = "Europe/Paris"
647 time.tzset()
649 run_prop = get_run_properties_from_db(det, "PHYS COMM")
651 try:
652 tim_0_unix = int(run_prop[run_0]["UNIXJOBSTART"])
653 tim_0 = time.strftime("%a, %d %b %Y %H:%M", time.localtime(tim_0_unix / 1000))
654 except KeyError:
655 tim_0 = "Unknown"
657 try:
658 tim_1_unix = int(run_prop[run_1]["UNIXJOBEND"])
659 tim_1 = time.strftime("%a, %d %b %Y %H:%M", time.localtime(tim_1_unix / 1000))
660 except KeyError:
661 tim_1 = "Unknown"
663 run_range = f"{int(run_0)} ({tim_0} CET/CEST) - " f"{int(run_1)} ({tim_1} CET/CEST)"
665 log_file.write(f"Run range: {run_range}\n")
668########################################################################################################
669def read_qaqc_file(det, source, qaqc_name, run_range):
670 """
671 sftp QAQC file reading
673 === Arguments ===
674 - det : detector name - [string] - Ex: "D0ARCA021", "D0ORCA018"...
675 - source : QAQC source, a priori "qaqc_sftp" - [string]
676 - qaqc_name :
677 - run_range : string as defined in DQ tag "[min_run]-[max_run]" (inclusive)
679 === Output ===
680 - TTree
681 - Error log
683 """
685 lines = []
686 if source == "qaqc_sftp":
687 with urllib.request.urlopen(
688 get_file_paths(det, qaqc_name)["JQAQC_sftp"]
689 ) as qaqc:
690 tmp = (qaqc.read()).split(b"\n")
691 for i_line in tmp:
692 if i_line != "":
693 lines.append(i_line.decode("utf-8"))
694 else:
695 if source == "qaqc_sps":
696 source_file = get_file_paths(det, qaqc_name)["JQAQC_sps"]
697 else:
698 source_file = source
700 with open(source_file, "r", encoding="utf-8") as s_f:
701 lines = s_f.readlines()
703 labels = lines[0].split()
704 run_index = labels.index("run")
705 if run_range != "":
706 min_run = int(run_range.split("-")[0])
707 max_run = int(run_range.split("-")[1])
708 else:
709 min_run = 0
710 max_run = 1e9
712 content = {}
713 for l in lines[1:]:
714 try:
715 if len(l.split()) != get_nb_qaqc_variables(l.split()[0]):
716 print(f"Bad number of variables in {l}")
717 continue
718 # sys.exit()
719 except IndexError: # Empty line
720 pass
722 try:
723 if all((int(l.split()[run_index]) >= min_run,
724 int(l.split()[run_index]) <= max_run
725 )):
726 for i, key in enumerate(labels):
727 try:
728 try:
729 content[key].append(l.split()[i])
730 except KeyError:
731 content[key] = [l.split()[i]]
732 except IndexError: # Missing variable in QAQC file
733 try:
734 content[key].append(MISSING_VALUE)
735 except KeyError:
736 content[key] = [MISSING_VALUE]
737 except IndexError: # Empty line
738 pass
740 return content
743########################################################################################################
744def decode_source_byte(value, v_prop):
745 """
746 Decode the source of degradation stored in the TTree
747 NB: the source is either a QAQC variable either a defect type (("daq",
748 "operation"...)
749 """
751 source_list = []
752 source_str = ""
754 for i_var, i_bit in v_prop["bit"].items():
755 if (int(value) >> i_bit) & 1:
756 source_list.append(i_var)
757 source_str += f"{i_var} "
759 return (source_list, source_str)