Coverage for src/km3dq_common/common_library.py: 52%
288 statements
« prev ^ index » next coverage.py v7.10.6, created at 2025-09-14 11:06 +0000
1#! /usr/bin/env python
2########################################################################################################
3# Various functions used in km3dq_lw_dq and km3dq_core scripts
4#
5# Developer: Benjamin Trocme (benjamin.trocme at apc.in2p3.fr) - 2023
7import os
8import sys
9import time
10import array
11import re
12import urllib.request
13import km3db
14import pandas as pd
15import numpy as np
16import tomli
18from .config_library import configure_var_bit
19from .config_library import configure_var_thresholds
20from .config_library import get_detx_caract
21from .config_library import configure_dataquality_tag
22from .config_library import configure_defect
23from .lw_db_defect_library import read_defect_file
24from .lw_db_defect_library import decode_defect_diag_byte
25from .lw_db_defect_library import check_defect_diag_byte
27# Types of all variables of all versions except the oos variables of 16.0.3 that
28# are renamed on the fly
29# Variable evolution with versions:
30# - 17.3.2: out-sync/in-sync renamed out_sync/in_sync
31# out_usync/in_usync/event_duration added
32# - 19.2.0: nb_of_meta/hrv_fifo_failures/duplic_timeslices/zero_AHRS/mean_AHRS added
33# AHRS removed
35VARIABLE_TYPE = {
36 "livetime_s": float,
37 "DAQ": float,
38 "WR": float,
39 "FIFO": float,
40 "HRV": float,
41 "PMTs": float,
42 "pmts_norm": float,
43 "timestampdiff": float,
44 "timestampdiff_r": float,
45 "triggerrate": float,
46 "Acoustics": float,
47 "Acoustics_per_mn": float,
48 "MEAN_Rate_Hz": float,
49 "RMS_Rate_Hz": float,
50 "AHRS": float,
51 "ahrs_per_mn": float,
52 "mean_AHRS": float,
53 "JDAQJTProc": float,
54 "kton_year": float,
55 "UTCMin_s": float,
56 "UTCMax_s": float,
57 "detector": int,
58 "run": int,
59 "nb_of_meta": int,
60 "trigger3DMuon": int,
61 "triggerMXShower": int,
62 "trigger3DShower": int,
63 "triggerNB": int,
64 "JTriggerReprocessor": int,
65 "JTrigger3dmuon": int,
66 "JTriggermxshower": int,
67 "JTrigger3dshower": int,
68 "JTriggernb": int,
69 "writeL0": int,
70 "writeL1": int,
71 "writeL2": int,
72 "writeSN": int,
73 "JDAQTimeslice": int,
74 "JDAQTimesliceL0": int,
75 "JDAQTimesliceL1": int,
76 "JDAQTimesliceL2": int,
77 "JDAQTimesliceSH": int,
78 "JDAQSummaryslice": int,
79 "JDAQEvent": int,
80 "hrv_fifo_failures": int,
81 "duplic_timeslices": int,
82 "in_sync": int,
83 "zero_AHRS": int,
84 "out_sync": int,
85 "in_usync": int,
86 "out_usync": int,
87 "HV_check":int
88 }
90# Dummy value used when the variable is missing in a run/QAQC file
91MISSING_VALUE = -999999
93########################################################################################################
94def get_det_id(det):
95 """
96 Returns the det id
98 === Input:
99 - det [string]: D1ORCA024, D0ARCA030...
100 === Outputs:
101 - detector id [integer]
102 """
104 return km3db.tools.todetid(det)
107########################################################################################################
108def get_current_detector(site):
109 """
110 Extract the current detector names from the database by checking the highest run number
111 By default, it is retrieved from the KM3NeT DB but in some rare cases (typically at a detector
112 change) it can be forced to a different one by modifying the km3dq_lw_db/Common/current_detector.toml
113 file that is available on sftp.
115 === Input:
116 - site [string]: ARCA / ORCA
117 === Outputs:
118 - detector [string]: D1ORCA024, D0ARCA030...
119 """
121 # First check if the current detector is hardcoded in the current_detector.toml file
122 # Facility especially useful at the detector change to force keeping a detector in Coral
123 with urllib.request.urlopen(
124 "https://sftp.km3net.de/data/km3dq_lw_db/Common/current_detector.toml"
125 ) as s_f:
126 toml = tomli.loads(s_f.read().decode("utf-8"))
128 if toml[site] != "":
129 return toml[site]
131 # If not -> Retrieve it from the database
132 sds = km3db.tools.StreamDS()
134 if site == "ARCA":
135 res = sds.detectors(city="ARCA+Deployment+site+-+seabed").split("\n")
136 else:
137 res = sds.detectors(city="ORCA+Deployment+site+-+seabed").split("\n")
139 max_run = 0
141 for i_det in res[1:]:
142 try:
143 if int(i_det.split("\t")[5]) > max_run:
144 o_det = i_det.split("\t")[0]
145 if o_det.startswith("D") is False:
146 continue
147 max_run = int(i_det.split("\t")[5])
148 except:
149 o_det = ""
150 print("Improper db reply in get_current_detectors")
152 return o_det
155########################################################################################################
156def get_file_paths(detector, qaqc_name):
157 """
158 For a given detector / detector, returns the path for QAQC files both on sps and sftp
160 === Input:
161 - det [string]: D1ORCA024, D0ARCA030...
162 === Outputs:
163 - detector id [integer]
164 """
166 return {
167 "JQAQC_sps": (
168 "/sps/km3net/repo/data/raw/quality/"
169 f"KM3NeT_{km3db.tools.todetid(detector):08d}_QAQC_Jpp_{qaqc_name}.txt"
170 ),
171 "JQAQC_sftp":(
172 "https://sftp.km3net.de/data/quality/"
173 f"KM3NeT_{km3db.tools.todetid(detector):08d}_QAQC_Jpp_{qaqc_name}.txt"
174 )
175 }
178########################################################################################################
179def get_run_properties_from_db(det, filt="PHYS", filt_target="Run"):
180 """
181 Retrieve run properties from the database with the km3db package
183 === Inputs:
184 - det [string]: D1ORCA024, D0ARCA030...
185 - filt [string]: accepted run types (PHYS, COMM...) separated by a space - if == "", no filter
186 - filt_target [string]: accepted target (run, on...) separated by a space - if == "", no filter
187 === Outputs:
188 - properties [dict of dict]: all DB content in a single dictionnary with the run numbers as key
189 """
191 runs = km3db.StreamDS(container="nt").get("runs", detid=det)
193 results = {}
195 for i_run in runs:
196 if filt != "":
197 if all((i_run.runsetupname[0:4] not in filt.split(" "),
198 i_run.runsetupname[0:5] not in filt.split(" "))):
199 continue
201 if filt_target != "":
202 if i_run.jobtarget not in filt_target.split(" "):
203 continue
205 results[i_run.run] ={
206 'UNIXSTARTTIME': i_run.unixstarttime,
207 'STARTTIME_DEFINED': i_run.starttime_defined,
208 'RUNSETUPID': i_run.runsetupid,
209 'RUNSETUPNAME': i_run.runsetupname,
210 'T0_CALIBSETID': i_run.t0_calibsetid,
211 'POS_CALIBSETID': i_run.pos_calibsetid,
212 'ROT_CALIBSETID': i_run.rot_calibsetid,
213 'RUNJOBID': i_run.runjobid,
214 'JOBTARGET': i_run.jobtarget,
215 'JOBPRIORITY': i_run.jobpriority,
216 'UNIXJOBSTART': i_run.unixjobstart,
217 'UNIXJOBEND': i_run.unixjobend,
218 }
220 return results
223########################################################################################################
224def get_run_properties_from_qaqc(
225 det, dq_tag, origin="qaqc_sftp", startrun=0, endrun=1e9
226):
227 """
228 For a given detector, returns the run numbers, their lengths, time counter... as retrieved from the
229 QAQC file
231 === Inputs:
232 - det [string]: D1ORCA024, D0ARCA030...
233 - dq_tag [DQ tag]: as configured in config_library
234 - origin [string]: "qaqc_sftp" or "qaqc_sps"
235 - startrun [integer]: first run
236 - endrun [integer]: last run
237 === Outputs:
238 - run properties: TOBEDETAILED
239 - error: should be empty
240 """
242 runprop = {
243 "runNumber": [],
244 "lvt": [],
245 "lvt_counter": [],
246 "kty": [],
247 "kty_counter": [],
248 "nbRuns": 0,
249 "minRun": 1e10,
250 "maxRun": 0,
251 }
253 kty_counter = 0
254 err_log = ""
256 (n1, err_log) = create_ttree_from_qaqc(
257 det, ["run", "livetime_s", "kton_year"], origin, dq_tag
258 )
260 if len(n1) == 0:
261 return (runprop, "No data in qaqc")
263 if all((startrun != 0, endrun != 1e9)):
264 n1 = n1[(n1["run"] >= startrun) & (n1["run"] <= endrun)]
266 runprop = {
267 "runNumber": n1.run.to_list(),
268 "lvt": (n1.livetime_s / 3600 / 24).to_list(),
269 "kty": n1.kton_year.to_list(),
270 "nbRuns": len(n1),
271 "minRun": int(n1.min()["run"]),
272 "maxRun": int(n1.max()["run"]),
273 }
275 runprop["lvt_counter"] = [0] + (n1.cumsum()["livetime_s"] / 3600 / 24).to_list()
276 runprop["kty_counter"] = [0] + (n1.cumsum()["kton_year"]).to_list()
278 return (runprop, err_log)
281########################################################################################################
282def get_run_timerange(det, filt="PHYS"):
283 """
284 Retrieve the run start/stop by combining db and QAQC quantities to get the most accurate timing.
285 If QAQC data exist (normally for all PHYS runs), use them, otherwise use the DB ones
286 So far used only in km3net_lw_db_interface
288 === Inputs:
289 - det: string like (e.g: D0ARCA030, D1ORCA024...)
290 - filt: string containing the accepted run type (PHYS, COMM...) separated by a space
292 === Outputs:
293 - dictionary with run as key and a list of [run_start, run_stop, source, runtype] with
294 source = 0 (QAQC) or 1 (DB only)
295 """
297 # First retrieve the QAQC variables that will be used to get the exact
298 # run end
299 df, _ = create_ttree_from_qaqc(
300 det, ["run", "livetime_s"], "qaqc_sftp", configure_dataquality_tag("operation")
301 )
302 results = dict([(r, [s, e, 0]) for r, s, e in zip(df["run"], df["UTCMin_s"], df["UTCMax_s"])])
304 # Then retrieve all runs from the database to get the remaining runs
305 runs = km3db.StreamDS(container="nt").get("runs", detid=det)
307 for i_run in runs:
308 i_run_runtype = i_run.runsetupname.split(".")[0]
309 if i_run_runtype not in filt:
310 continue
312 if i_run.run in results: # Timerange already retrieved from the QAQC data - Append run type
313 results[i_run.run].append(i_run_runtype)
314 continue
316 try:
317 results[i_run.run] = [i_run.unixjobstart/1000., i_run.unixjobend/1000., 1, i_run_runtype]
318 except TypeError:
319 pass
321 return results
324########################################################################################################
325def get_last_n_weeks_runs_qaqc(det, nweeks=2):
326 """Extract the all runs acquired in the last nweeks of running"""
328 # QAQC default tag used. As priori OK for this purpose
329 dq_tag = configure_dataquality_tag("operation")
330 min_run = 1e9
331 max_run = 0
333 try:
334 lines = read_qaqc_file(det, "qaqc_sftp", dq_tag["qaqc_name"][det], "")
335 ten_days = (nweeks * 7 + 1) * 3600.0 * 24.0
336 for i_run in range(len(lines["run"])):
337 if (time.time() - float(lines["UTCMin_s"][i_run])) < ten_days:
338 run_number = int(lines["run"][i_run])
339 if run_number < min_run:
340 min_run = run_number
341 if run_number > max_run:
342 max_run = run_number
343 except KeyError:
344 print(f"common_library.py: Missing DQ-tag information for the detector {det}")
346 return (min_run, max_run)
349########################################################################################################
350def get_site(det):
351 """
352 Returns the site (ORCA or ARCA)
354 === Inputs:
355 - det [string]: D1ORCA024, D0ARCA030...
356 === Ouputs:
357 - site [string]: ARCA/ORCA
358 """
360 # A/O detectors corresponds to the current ARCA/ORCA detectors
361 if "ORC" in det or det == "O":
362 out = "ORCA"
363 else:
364 out = "ARCA"
365 return out
368########################################################################################################
369def get_active_dus_range(det):
370 """
371 Retrieve the range of active DUs
372 To be improved with database information
374 === Inputs:
375 - det [string]: D1ORCA024, D0ARCA030...
376 === Ouputs:
377 - site [string]: ARCA/ORCA
378 """
379 lower_du = 0
380 upper_du = 48
382 return (lower_du, upper_du)
385########################################################################################################
386def get_nb_qaqc_variables(qaqc_vers):
387 """
388 Retrieve the number of variables in the QAQC file
390 === Inputs
391 - qaqc_vers [string]: "v16.0.3", "v17.3.2", "V19.2.0"...
393 === Output ===
394 - Number of QAQC variables [integer]
396 """
397 if qaqc_vers in ("v16.0.3", "16.0.3"):
398 return 39
399 if qaqc_vers in ("v17.3.2", "17.3.2", "18.0.0-rc.4-34-g8618072aa-D"):
400 return 42
401 if qaqc_vers in ("v19.2.0", "19.1.0", "19.2.0"):
402 return 46
403 if qaqc_vers in ("19.3.0-rc.2", "19.3.0-rc.5"):
404 return 46
405 if qaqc_vers == "19.3.0-rc.x":
406 return 47
407 if qaqc_vers == "19.3.0":
408 return 47
410 # print(f"Unknwon qaqc version {qaqc_vers} -> Assuming that it is jpp master -> 47 variables")
411 return 47
413########################################################################################################
414def create_ttree_from_qaqc(det, var_to_fill, source, tag, append_veto_qsco=False):
415 """
416 Create a ttree from qaqc sftp file and defect variables stored on git
417 It includes some advanced check about the QAQC file integrity.
419 === Inputs
420 - det : detector name - [string] - Ex: "D0ARCA021", "D0ORCA018"...
421 - var_to_fill : QAQC variables or defect to fill - [array of string] -
422 Ex: ['run', 'timestampdiff', 'def_operation', 'def_oos']
423 Obsolete - To be removed at some point
424 - source : QAQC source, a priori "qaqc_sftp" - [string]
425 - tag : data-quality tag (not its name) as created by
426 configure_dataquality_tag
427 - append_veto_qsco: append the veto and Qscore - [boolean]
428 === Outputs
429 - TTree
430 - Error log
432 """
434 error_log = ""
436 content = read_qaqc_file(det, source, tag["qaqc_name"][det], tag["run_range"][det])
438 df = pd.DataFrame(content)
439 # Renaming required for 16.0.3
440 df = df.rename(columns={"out-sync": "out_sync"})
441 df = df.rename(columns={"in-sync": "in_sync"})
443 # Type definition for each column
444 # df = df.astype({key: float for key in variable_float if key in df.columns})
445 # df = df.astype({key: int for key in variable_int if key in df.columns})
446 for i_var, i_type in VARIABLE_TYPE.items():
447 try:
448 df = df.astype({i_var: i_type})
449 except KeyError: # Variable missing in this version -> Dummy column added
450 df[i_var] = len(df) * [MISSING_VALUE]
452 df["timestampdiff"] = df.UTCMax_s - df.UTCMin_s - df.livetime_s
453 df["timestampdiff_r"] = 1.0 - df.livetime_s / (df.UTCMax_s - df.UTCMin_s)
454 df["pmts_norm"] = df.PMTs / get_detx_caract(det)["pmts"]
455 df["triggerrate"] = df.JDAQEvent / df.livetime_s
457 try: # Version > 19.2.0
458 df["ahrs_per_mn"] = df.mean_AHRS / df.livetime_s * 60
459 except AttributeError: # Version 16.0.3 / 17.3.2
460 df["ahrs_per_mn"] = df.AHRS / df.livetime_s * 60
462 df["Acoustics_per_mn"] = df.Acoustics / df.livetime_s * 60
464 df["JDAQJTProc"] = (df.JDAQEvent - df.JTriggerReprocessor) / (df.JDAQEvent + 1e-10)
465 if "ARC" in det:
466 df["kton_year"] = (
467 1e6
468 / (31 * 18 * 2 * 115)
469 * df.PMTs
470 * (1.0 - df.HRV)
471 * df.livetime_s
472 / (365.2425 * 24 * 3600)
473 )
474 else:
475 df["kton_year"] = (
476 6980
477 / (31 * 18 * 115)
478 * df.PMTs
479 * (1.0 - df.HRV)
480 * df.livetime_s
481 / (365.2425 * 24 * 3600)
482 )
485 # Adding defects
486 defect_results = read_defect_file(det, tag["def_tag"])
488 df = df.sort_values("run")
489 # Check if there are some duplicated lines with the same run number
490 # If some are found exit, as this is not acceptable and have to be cured
491 if True in df.duplicated(subset=["run"], keep=False).values:
492 print("Duplicated lines in create_ttree_from_qaqc\n"
493 "Fix required -> Exiting")
494 print(df.mask(~df.duplicated(subset=["run"], keep=False)).dropna())
495 sys.exit()
497 df.reset_index(drop=True)
499 for key, runs in defect_results.items():
500 value = np.zeros(len(df))
501 if len(runs) != 0:
502 # A defect can be assigned to a run missing in the QAQC file (protection added)
503 # The defects_results is not chronologically ordered (sorting added)
504 mask = np.isin(
505 df.run, [x for x in sorted(runs.keys()) if x in df["run"].values]
506 )
507 value[mask] = [
508 runs[x] for x in sorted(runs.keys()) if x in df["run"].values
509 ]
510 df[key] = value
512 # Adding Veto and Q score:
513 if append_veto_qsco:
514 var_properties = {}
515 configure_var_thresholds(var_properties, det, tag)
516 configure_var_bit(var_properties)
518 df["veto"], df["veto_source"], df["qsco"], df["qsco_source"] = (
519 compute_veto_qscore(var_properties, df)
520 )
522 if error_log != "":
523 print(error_log)
524 return (df, error_log)
527########################################################################################################
528def compute_veto_qscore(var_prop, var_val):
529 """
530 Computes the veto/Q-score for a detector/tag using a QAQC source.
531 """
533 def_config = configure_defect()
535 veto = np.zeros(len(var_val), dtype=bool)
536 veto_source = np.zeros(len(var_val), dtype=int)
538 qsco = np.ones(len(var_val), dtype=float) * len(var_prop["qsco_thresholds"])
539 qsco_source = np.zeros(len(var_val), dtype=int)
541 pd.set_option("display.max_rows", None)
542 for i_var in var_prop["veto_thresholds"]:
543 values = var_val[i_var]
544 # Defect variables
545 # The defect variable is defined as def_[defect type]
546 # with [defect type] = daq, operation, analysis...
547 if i_var.startswith("def_"):
548 # first extract the diagnosis
549 defect_type = i_var.replace("def_", "")
550 diag_array = {"present": [], "absent": []}
551 for i_excluded_diag in var_prop["veto_thresholds"][i_var]:
552 if i_excluded_diag.startswith("NOT."):
553 diag_array["absent"].append(
554 def_config["bit"][defect_type][
555 i_excluded_diag.replace("NOT.", "")
556 ]
557 )
558 else:
559 diag_array["present"].append(
560 def_config["bit"][defect_type][i_excluded_diag]
561 )
563 if len(diag_array["absent"]) == 0: # Only present diagnosis
564 def_mask = values != 0
565 elif len(diag_array["present"]) == 0: # Only NOT. diagnosis
566 def_mask = values == 0
567 else: # Mixture of present/absent -> No mask
568 def_mask = values != (-1)
569 if np.sum(def_mask) == 0:
570 continue
572 defect_test_output = np.vectorize(check_defect_diag_byte, excluded=[1])(
573 values[def_mask], diag_array
574 )
575 # The FutureWarning about the incompatible dtype comes from the line below
576 # When checking explicitly the dtype, both are returned as bool hence
577 # the FutureWarning can not be understood.
578 # The trial to fix with explicit casting and other numpy functions
579 # (e.g veto[def_mask] = np.logical_or(veto[def_mask].astype(bool), defect_test_output.astype(bool)))
580 # did not help at all.
581 veto[def_mask] = veto[def_mask] | defect_test_output
582 veto_source[def_mask] += (0b1 & defect_test_output) << var_prop["bit"][
583 i_var
584 ]
586 else: # QAQC variable: range treatement
587 range_mask = (var_val[i_var] != MISSING_VALUE) & (
588 (var_val[i_var] < var_prop["veto_thresholds"][i_var][0]) | (
589 var_val[i_var] > var_prop["veto_thresholds"][i_var][1]
590 )
591 )
592 veto = veto | range_mask
593 veto_source[range_mask] += 0b1 << var_prop["bit"][i_var]
595 for i_var in var_prop["qsco_thresholds"]:
596 values = var_val[i_var]
597 # Defect variables
598 # Defect variables
599 # The defect variable is defined as def_[defect type]
600 # with [defect type] = daq, operation, analysis...
601 if i_var.startswith("def_"):
602 # first extract the set defects
603 def_mask = values != 0
604 if np.sum(def_mask) == 0:
605 continue
607 defect_type = i_var.replace("def_", "")
608 diag_array = {"present": [], "absent": []}
609 for i_excluded_diag in var_prop["qsco_thresholds"][i_var]:
610 if i_excluded_diag.startswith("NOT."):
611 diag_array["absent"].append(
612 def_config["bit"][defect_type][
613 i_excluded_diag.replace("NOT.", "")
614 ]
615 )
616 else:
617 diag_array["present"].append(
618 def_config["bit"][defect_type][i_excluded_diag]
619 )
621 defect_test_output = np.vectorize(check_defect_diag_byte, excluded=[1])(
622 values[def_mask], diag_array
623 )
624 qsco[def_mask] = qsco[def_mask] | defect_test_output
625 qsco_source[def_mask] += (0b1 & defect_test_output) << var_prop["bit"][
626 i_var
627 ]
629 else: # QAQC variable: range treatement
630 range_mask = (var_val[i_var] != MISSING_VALUE) & (
631 (var_val[i_var] < var_prop["qsco_thresholds"][i_var][0]) | (
632 var_val[i_var] > var_prop["qsco_thresholds"][i_var][1]
633 )
634 )
636 qsco[range_mask] -= 1
637 qsco_source[range_mask] += 0b1 << var_prop["bit"][i_var]
639 qsco /= len(var_prop["qsco_thresholds"])
641 return (veto, veto_source, qsco, qsco_source)
644########################################################################################################
645def log_run_range(det, run_0, run_1, log_file):
646 """Write the run range and run list on disk"""
648 os.environ["TZ"] = "Europe/Paris"
649 time.tzset()
651 run_prop = get_run_properties_from_db(det, "PHYS COMM")
653 try:
654 tim_0_unix = int(run_prop[run_0]["UNIXJOBSTART"])
655 tim_0 = time.strftime("%a, %d %b %Y %H:%M", time.localtime(tim_0_unix / 1000))
656 except KeyError:
657 tim_0 = "Unknown"
659 try:
660 tim_1_unix = int(run_prop[run_1]["UNIXJOBEND"])
661 tim_1 = time.strftime("%a, %d %b %Y %H:%M", time.localtime(tim_1_unix / 1000))
662 except KeyError:
663 tim_1 = "Unknown"
665 run_range = f"{int(run_0)} ({tim_0} CET/CEST) - " f"{int(run_1)} ({tim_1} CET/CEST)"
667 log_file.write(f"Run range: {run_range}\n")
670########################################################################################################
671def read_qaqc_file(det, source, qaqc_name, run_range):
672 """
673 sftp QAQC file reading
675 === Arguments ===
676 - det : detector name - [string] - Ex: "D0ARCA021", "D0ORCA018"...
677 - source : QAQC source, a priori "qaqc_sftp" - [string]
678 - qaqc_name :
679 - run_range : string as defined in DQ tag "[min_run]-[max_run]" (inclusive)
681 === Output ===
682 - TTree
683 - Error log
685 """
687 lines = []
688 if source == "qaqc_sftp":
689 with urllib.request.urlopen(
690 get_file_paths(det, qaqc_name)["JQAQC_sftp"]
691 ) as qaqc:
692 tmp = (qaqc.read()).split(b"\n")
693 for i_line in tmp:
694 if i_line != "":
695 lines.append(i_line.decode("utf-8"))
696 else:
697 if source == "qaqc_sps":
698 source_file = get_file_paths(det, qaqc_name)["JQAQC_sps"]
699 else:
700 source_file = source
702 with open(source_file, "r", encoding="utf-8") as s_f:
703 lines = s_f.readlines()
705 labels = lines[0].split()
706 run_index = labels.index("run")
707 if run_range != "":
708 min_run = int(run_range.split("-")[0])
709 max_run = int(run_range.split("-")[1])
710 else:
711 min_run = 0
712 max_run = 1e9
714 content = {}
715 for l in lines[1:]:
716 try:
717 if len(l.split()) != get_nb_qaqc_variables(l.split()[0]):
718 print(f"Bad number of variables in {l}")
719 continue
720 # sys.exit()
721 except IndexError: # Empty line
722 pass
724 try:
725 if all((int(l.split()[run_index]) >= min_run,
726 int(l.split()[run_index]) <= max_run
727 )):
728 for i, key in enumerate(labels):
729 try:
730 try:
731 content[key].append(l.split()[i])
732 except KeyError:
733 content[key] = [l.split()[i]]
734 except IndexError: # Missing variable in QAQC file
735 try:
736 content[key].append(MISSING_VALUE)
737 except KeyError:
738 content[key] = [MISSING_VALUE]
739 except IndexError: # Empty line
740 pass
742 return content
745########################################################################################################
746def decode_source_byte(value, v_prop):
747 """
748 Decode the source of degradation stored in the TTree
749 NB: the source is either a QAQC variable either a defect type (("daq",
750 "operation"...)
751 """
753 source_list = []
754 source_str = ""
756 for i_var, i_bit in v_prop["bit"].items():
757 if (int(value) >> i_bit) & 1:
758 source_list.append(i_var)
759 source_str += f"{i_var} "
761 return (source_list, source_str)