Coverage for src/km3dq_common/common_library.py: 51%
292 statements
« prev ^ index » next coverage.py v7.10.7, created at 2025-10-16 11:32 +0000
1#! /usr/bin/env python
2########################################################################################################
3# Various functions used in km3dq_lw_dq and km3dq_core scripts
4#
5# Developer: Benjamin Trocme (benjamin.trocme at apc.in2p3.fr) - 2023
7import os
8import sys
9import time
10import array
11import re
12import urllib.request
13import km3db
14import pandas as pd
15import numpy as np
16import tomli
18from .config_library import configure_var_bit
19from .config_library import configure_var_thresholds
20from .config_library import get_detx_caract
21from .config_library import configure_dataquality_tag
22from .config_library import configure_defect
23from .lw_db_defect_library import read_defect_file
24from .lw_db_defect_library import decode_defect_diag_byte
25from .lw_db_defect_library import check_defect_diag_byte
27# Types of all variables of all versions except the oos variables of 16.0.3 that
28# are renamed on the fly
29# Variable evolution with versions:
30# - 17.3.2: out-sync/in-sync renamed out_sync/in_sync
31# out_usync/in_usync/event_duration added
32# - 19.2.0: nb_of_meta/hrv_fifo_failures/duplic_timeslices/zero_AHRS/mean_AHRS added
33# AHRS removed
35VARIABLE_TYPE = {
36 "livetime_s": float,
37 "DAQ": float,
38 "WR": float,
39 "FIFO": float,
40 "HRV": float,
41 "PMTs": float,
42 "pmts_norm": float,
43 "timestampdiff": float,
44 "timestampdiff_r": float,
45 "triggerrate": float,
46 "Acoustics": float,
47 "Acoustics_per_mn": float,
48 "MEAN_Rate_Hz": float,
49 "RMS_Rate_Hz": float,
50 "AHRS": float,
51 "ahrs_per_mn": float,
52 "mean_AHRS": float,
53 "JDAQJTProc": float,
54 "kton_year": float,
55 "UTCMin_s": float,
56 "UTCMax_s": float,
57 "detector": int,
58 "run": int,
59 "nb_of_meta": int,
60 "trigger3DMuon": int,
61 "triggerMXShower": int,
62 "trigger3DShower": int,
63 "triggerNB": int,
64 "JTriggerReprocessor": int,
65 "JTrigger3dmuon": int,
66 "JTriggermxshower": int,
67 "JTrigger3dshower": int,
68 "JTriggernb": int,
69 "writeL0": int,
70 "writeL1": int,
71 "writeL2": int,
72 "writeSN": int,
73 "JDAQTimeslice": int,
74 "JDAQTimesliceL0": int,
75 "JDAQTimesliceL1": int,
76 "JDAQTimesliceL2": int,
77 "JDAQTimesliceSH": int,
78 "JDAQSummaryslice": int,
79 "JDAQEvent": int,
80 "hrv_fifo_failures": int,
81 "duplic_timeslices": int,
82 "in_sync": int,
83 "zero_AHRS": int,
84 "out_sync": int,
85 "in_usync": int,
86 "out_usync": int,
87 "HV_check":int
88 }
90# Dummy value used when the variable is missing in a run/QAQC file
91MISSING_VALUE = -999999
93########################################################################################################
94def get_det_id(det):
95 """
96 Returns the det id
98 === Input:
99 - det [string]: D1ORCA024, D0ARCA030...
100 === Outputs:
101 - detector id [integer]
102 """
104 return km3db.tools.todetid(det)
107########################################################################################################
108def get_current_detector(site):
109 """
110 Extract the current detector names from the database by checking the highest run number
111 By default, it is retrieved from the KM3NeT DB but in some rare cases (typically at a detector
112 change) it can be forced to a different one by modifying the km3dq_lw_db/Common/current_detector.toml
113 file that is available on sftp.
115 === Input:
116 - site [string]: ARCA / ORCA
117 === Outputs:
118 - detector [string]: D1ORCA024, D0ARCA030...
119 """
121 # First check if the current detector is hardcoded in the current_detector.toml file
122 # Facility especially useful at the detector change to force keeping a detector in Coral
123 with urllib.request.urlopen(
124 "https://sftp.km3net.de/data/km3dq_lw_db/Common/current_detector.toml"
125 ) as s_f:
126 toml = tomli.loads(s_f.read().decode("utf-8"))
128 if toml[site] != "":
129 return toml[site]
131 # If not -> Retrieve it from the database
132 sds = km3db.tools.StreamDS()
134 if site == "ARCA":
135 res = sds.detectors(city="ARCA+Deployment+site+-+seabed").split("\n")
136 else:
137 res = sds.detectors(city="ORCA+Deployment+site+-+seabed").split("\n")
139 max_run = 0
141 for i_det in res[1:]:
142 try:
143 if int(i_det.split("\t")[5]) > max_run:
144 o_det = i_det.split("\t")[0]
145 if o_det.startswith("D") is False:
146 continue
147 max_run = int(i_det.split("\t")[5])
148 except:
149 o_det = ""
150 print("Improper db reply in get_current_detectors")
152 return o_det
155########################################################################################################
156def get_file_paths(detector, qaqc_name):
157 """
158 For a given detector / detector, returns the path for QAQC files both on sps and sftp
160 === Input:
161 - det [string]: D1ORCA024, D0ARCA030...
162 === Outputs:
163 - detector id [integer]
164 """
166 return {
167 "JQAQC_sps": (
168 "/sps/km3net/repo/data/raw/quality/"
169 f"KM3NeT_{km3db.tools.todetid(detector):08d}_QAQC_Jpp_{qaqc_name}.txt"
170 ),
171 "JQAQC_sftp":(
172 "https://sftp.km3net.de/data/quality/"
173 f"KM3NeT_{km3db.tools.todetid(detector):08d}_QAQC_Jpp_{qaqc_name}.txt"
174 )
175 }
178########################################################################################################
179def get_run_properties_from_db(det, filt="PHYS", filt_target="Run"):
180 """
181 Retrieve run properties from the database with the km3db package
183 === Inputs:
184 - det [string]: D1ORCA024, D0ARCA030...
185 - filt [string]: accepted run types (PHYS, COMM...) separated by a space - if == "", no filter
186 - filt_target [string]: accepted target (run, on...) separated by a space - if == "", no filter
187 === Outputs:
188 - properties [dict of dict]: all DB content in a single dictionnary with the run numbers as key
189 """
191 runs = km3db.StreamDS(container="nt").get("runs", detid=det)
193 results = {}
195 for i_run in runs:
196 if filt != "":
197 if all((i_run.runsetupname[0:4] not in filt.split(" "),
198 i_run.runsetupname[0:5] not in filt.split(" "))):
199 continue
201 if filt_target != "":
202 if i_run.jobtarget not in filt_target.split(" "):
203 continue
205 results[i_run.run] ={
206 'UNIXSTARTTIME': i_run.unixstarttime,
207 'STARTTIME_DEFINED': i_run.starttime_defined,
208 'RUNSETUPID': i_run.runsetupid,
209 'RUNSETUPNAME': i_run.runsetupname,
210 'T0_CALIBSETID': i_run.t0_calibsetid,
211 'POS_CALIBSETID': i_run.pos_calibsetid,
212 'ROT_CALIBSETID': i_run.rot_calibsetid,
213 'RUNJOBID': i_run.runjobid,
214 'JOBTARGET': i_run.jobtarget,
215 'JOBPRIORITY': i_run.jobpriority,
216 'UNIXJOBSTART': i_run.unixjobstart,
217 'UNIXJOBEND': i_run.unixjobend,
218 }
220 return results
223########################################################################################################
224def get_run_properties_from_qaqc(
225 det, dq_tag, origin="qaqc_sftp", startrun=0, endrun=1e9
226):
227 """
228 For a given detector, returns the run numbers, their lengths, time counter... as retrieved from the
229 QAQC file
231 === Inputs:
232 - det [string]: D1ORCA024, D0ARCA030...
233 - dq_tag [DQ tag]: as configured in config_library
234 - origin [string]: "qaqc_sftp" or "qaqc_sps"
235 - startrun [integer]: first run
236 - endrun [integer]: last run
237 === Outputs:
238 - run properties: TOBEDETAILED
239 - error: should be empty
240 """
242 runprop = {
243 "runNumber": [],
244 "lvt": [],
245 "lvt_counter": [],
246 "kty": [],
247 "kty_counter": [],
248 "nbRuns": 0,
249 "minRun": 1e10,
250 "maxRun": 0,
251 }
253 kty_counter = 0
254 err_log = ""
256 (n1, err_log) = create_ttree_from_qaqc(
257 det, ["run", "livetime_s", "kton_year"], origin, dq_tag
258 )
260 if len(n1) == 0:
261 return (runprop, "No data in qaqc")
263 if all((startrun != 0, endrun != 1e9)):
264 n1 = n1[(n1["run"] >= startrun) & (n1["run"] <= endrun)]
266 runprop = {
267 "runNumber": n1.run.to_list(),
268 "lvt": (n1.livetime_s / 3600 / 24).to_list(),
269 "kty": n1.kton_year.to_list(),
270 "nbRuns": len(n1),
271 "minRun": int(n1["run"].min()),
272 "maxRun": int(n1["run"].max()),
273 }
275 runprop["lvt_counter"] = [0] + (n1["livetime_s"].cumsum() / 3600 / 24).to_list()
276 runprop["kty_counter"] = [0] + (n1["kton_year"].cumsum()).to_list()
278 return (runprop, err_log)
281########################################################################################################
282def get_run_timerange(det, filt="PHYS"):
283 """
284 Retrieve the run start/stop by combining db and QAQC quantities to get the most accurate timing.
285 If QAQC data exist (normally for all PHYS runs), use them, otherwise use the DB ones
286 So far used only in km3net_lw_db_interface
288 === Inputs:
289 - det: string like (e.g: D0ARCA030, D1ORCA024...)
290 - filt: string containing the accepted run type (PHYS, COMM...) separated by a space
292 === Outputs:
293 - dictionary with run as key and a list of [run_start, run_stop, source, runtype] with
294 source = 0 (QAQC) or 1 (DB only)
295 """
297 # First retrieve the QAQC variables that will be used to get the exact
298 # run end
299 df, _ = create_ttree_from_qaqc(
300 det, ["run", "livetime_s"], "qaqc_sftp", configure_dataquality_tag("operation")
301 )
302 results = dict([(r, [s, e, 0]) for r, s, e in zip(df["run"], df["UTCMin_s"], df["UTCMax_s"])])
304 # Then retrieve all runs from the database to get the remaining runs
305 runs = km3db.StreamDS(container="nt").get("runs", detid=det)
307 for i_run in runs:
308 i_run_runtype = i_run.runsetupname.split(".")[0]
309 if i_run_runtype not in filt:
310 continue
312 if i_run.run in results: # Timerange already retrieved from the QAQC data - Append run type
313 results[i_run.run].append(i_run_runtype)
314 continue
316 try:
317 results[i_run.run] = [i_run.unixjobstart/1000., i_run.unixjobend/1000., 1, i_run_runtype]
318 except TypeError:
319 pass
321 return results
324########################################################################################################
325def get_last_n_weeks_runs_qaqc(det, nweeks=2):
326 """Extract the all runs acquired in the last nweeks of running"""
328 # QAQC default tag used. As priori OK for this purpose
329 dq_tag = configure_dataquality_tag("operation")
330 min_run = 1e9
331 max_run = 0
333 try:
334 lines = read_qaqc_file(det, "qaqc_sftp", dq_tag["qaqc_name"][det], "")
335 ten_days = (nweeks * 7 + 1) * 3600.0 * 24.0
336 for i_run in range(len(lines["run"])):
337 if (time.time() - float(lines["UTCMin_s"][i_run])) < ten_days:
338 run_number = int(lines["run"][i_run])
339 if run_number < min_run:
340 min_run = run_number
341 if run_number > max_run:
342 max_run = run_number
343 except KeyError:
344 print(f"common_library.py: Missing DQ-tag information for the detector {det}")
346 return (min_run, max_run)
349########################################################################################################
350def get_site(det):
351 """
352 Returns the site (ORCA or ARCA)
354 === Inputs:
355 - det [string]: D1ORCA024, D0ARCA030...
356 === Ouputs:
357 - site [string]: ARCA/ORCA
358 """
360 # A/O detectors corresponds to the current ARCA/ORCA detectors
361 if "ORC" in det or det == "O":
362 out = "ORCA"
363 else:
364 out = "ARCA"
365 return out
368########################################################################################################
369def get_active_dus_range(det):
370 """
371 Retrieve the range of active DUs
372 To be improved with database information
374 === Inputs:
375 - det [string]: D1ORCA024, D0ARCA030...
376 === Ouputs:
377 - site [string]: ARCA/ORCA
378 """
379 lower_du = 0
380 upper_du = 48
382 return (lower_du, upper_du)
385########################################################################################################
386def get_nb_qaqc_variables(qaqc_vers):
387 """
388 Retrieve the number of variables in the QAQC file
390 === Inputs
391 - qaqc_vers [string]: "v16.0.3", "v17.3.2", "V19.2.0"...
393 === Output ===
394 - Number of QAQC variables [integer]
396 """
397 if qaqc_vers in ("v16.0.3", "16.0.3"):
398 return 39
399 if qaqc_vers in ("v17.3.2", "17.3.2", "18.0.0-rc.4-34-g8618072aa-D"):
400 return 42
401 if qaqc_vers in ("v19.2.0", "19.1.0", "19.2.0"):
402 return 46
403 if qaqc_vers in ("19.3.0-rc.2", "19.3.0-rc.5"):
404 return 46
405 if qaqc_vers == "19.3.0-rc.x":
406 return 47
407 if qaqc_vers == "19.3.0":
408 return 47
409 if qaqc_vers == "jpp-master-47": # This jpp version with 47 variables was hacked by hand in the QAQC file
410 return 47
411 if qaqc_vers == "20.0.0-rc.2": # This jpp version with 47 variables was hacked by hand in the QAQC file
412 return 47
414 # print(f"Unknwon qaqc version {qaqc_vers} -> Assuming that it is jpp master -> 47 variables")
415 return 48
417########################################################################################################
418def create_ttree_from_qaqc(det, var_to_fill, source, tag, append_veto_qsco=False):
419 """
420 Create a ttree from qaqc sftp file and defect variables stored on git
421 It includes some advanced check about the QAQC file integrity.
423 === Inputs
424 - det : detector name - [string] - Ex: "D0ARCA021", "D0ORCA018"...
425 - var_to_fill : QAQC variables or defect to fill - [array of string] -
426 Ex: ['run', 'timestampdiff', 'def_operation', 'def_oos']
427 Obsolete - To be removed at some point
428 - source : QAQC source, a priori "qaqc_sftp" - [string]
429 - tag : data-quality tag (not its name) as created by
430 configure_dataquality_tag
431 - append_veto_qsco: append the veto and Qscore - [boolean]
432 === Outputs
433 - TTree
434 - Error log
436 """
438 error_log = ""
440 content = read_qaqc_file(det, source, tag["qaqc_name"][det], tag["run_range"][det])
442 df = pd.DataFrame(content)
443 # Renaming required for 16.0.3
444 df = df.rename(columns={"out-sync": "out_sync"})
445 df = df.rename(columns={"in-sync": "in_sync"})
447 # Type definition for each column
448 # df = df.astype({key: float for key in variable_float if key in df.columns})
449 # df = df.astype({key: int for key in variable_int if key in df.columns})
450 for i_var, i_type in VARIABLE_TYPE.items():
451 try:
452 df = df.astype({i_var: i_type})
453 except KeyError: # Variable missing in this version -> Dummy column added
454 df[i_var] = len(df) * [MISSING_VALUE]
456 df["timestampdiff"] = df.UTCMax_s - df.UTCMin_s - df.livetime_s
457 df["timestampdiff_r"] = 1.0 - df.livetime_s / (df.UTCMax_s - df.UTCMin_s)
458 df["pmts_norm"] = df.PMTs / get_detx_caract(det)["pmts"]
459 df["triggerrate"] = df.JDAQEvent / df.livetime_s
461 try: # Version > 19.2.0
462 df["ahrs_per_mn"] = df.mean_AHRS / df.livetime_s * 60
463 except AttributeError: # Version 16.0.3 / 17.3.2
464 df["ahrs_per_mn"] = df.AHRS / df.livetime_s * 60
466 df["Acoustics_per_mn"] = df.Acoustics / df.livetime_s * 60
468 df["JDAQJTProc"] = (df.JDAQEvent - df.JTriggerReprocessor) / (df.JDAQEvent + 1e-10)
469 if "ARC" in det:
470 df["kton_year"] = (
471 1e6
472 / (31 * 18 * 2 * 115)
473 * df.PMTs
474 * (1.0 - df.HRV)
475 * df.livetime_s
476 / (365.2425 * 24 * 3600)
477 )
478 else:
479 df["kton_year"] = (
480 6980
481 / (31 * 18 * 115)
482 * df.PMTs
483 * (1.0 - df.HRV)
484 * df.livetime_s
485 / (365.2425 * 24 * 3600)
486 )
489 # Adding defects
490 defect_results = read_defect_file(det, tag["def_tag"])
492 df = df.sort_values("run")
493 # Check if there are some duplicated lines with the same run number
494 # If some are found exit, as this is not acceptable and have to be cured
495 if True in df.duplicated(subset=["run"], keep=False).values:
496 print("Duplicated lines in create_ttree_from_qaqc\n"
497 "Fix required -> Exiting")
498 print(df.mask(~df.duplicated(subset=["run"], keep=False)).dropna())
499 sys.exit()
501 df.reset_index(drop=True)
503 for key, runs in defect_results.items():
504 value = np.zeros(len(df))
505 if len(runs) != 0:
506 # A defect can be assigned to a run missing in the QAQC file (protection added)
507 # The defects_results is not chronologically ordered (sorting added)
508 mask = np.isin(
509 df.run, [x for x in sorted(runs.keys()) if x in df["run"].values]
510 )
511 value[mask] = [
512 runs[x] for x in sorted(runs.keys()) if x in df["run"].values
513 ]
514 df[key] = value
516 # Adding Veto and Q score:
517 if append_veto_qsco:
518 var_properties = {}
519 configure_var_thresholds(var_properties, det, tag)
520 configure_var_bit(var_properties)
522 df["veto"], df["veto_source"], df["qsco"], df["qsco_source"] = (
523 compute_veto_qscore(var_properties, df)
524 )
526 if error_log != "":
527 print(error_log)
528 return (df, error_log)
531########################################################################################################
532def compute_veto_qscore(var_prop, var_val):
533 """
534 Computes the veto/Q-score for a detector/tag using a QAQC source.
535 """
537 def_config = configure_defect()
539 veto = np.zeros(len(var_val), dtype=bool)
540 veto_source = np.zeros(len(var_val), dtype=int)
542 qsco = np.ones(len(var_val), dtype=float) * len(var_prop["qsco_thresholds"])
543 qsco_source = np.zeros(len(var_val), dtype=int)
545 pd.set_option("display.max_rows", None)
546 for i_var in var_prop["veto_thresholds"]:
547 values = var_val[i_var]
548 # Defect variables
549 # The defect variable is defined as def_[defect type]
550 # with [defect type] = daq, operation, analysis...
551 if i_var.startswith("def_"):
552 # first extract the diagnosis
553 defect_type = i_var.replace("def_", "")
554 diag_array = {"present": [], "absent": []}
555 for i_excluded_diag in var_prop["veto_thresholds"][i_var]:
556 if i_excluded_diag.startswith("NOT."):
557 diag_array["absent"].append(
558 def_config["bit"][defect_type][
559 i_excluded_diag.replace("NOT.", "")
560 ]
561 )
562 else:
563 diag_array["present"].append(
564 def_config["bit"][defect_type][i_excluded_diag]
565 )
567 if len(diag_array["absent"]) == 0: # Only present diagnosis
568 def_mask = values != 0
569 elif len(diag_array["present"]) == 0: # Only NOT. diagnosis
570 def_mask = values == 0
571 else: # Mixture of present/absent -> No mask
572 def_mask = values != (-1)
573 if np.sum(def_mask) == 0:
574 continue
576 defect_test_output = np.vectorize(check_defect_diag_byte, excluded=[1])(
577 values[def_mask], diag_array
578 )
579 # The FutureWarning about the incompatible dtype comes from the line below
580 # When checking explicitly the dtype, both are returned as bool hence
581 # the FutureWarning can not be understood.
582 # The trial to fix with explicit casting and other numpy functions
583 # (e.g veto[def_mask] = np.logical_or(veto[def_mask].astype(bool), defect_test_output.astype(bool)))
584 # did not help at all.
585 veto[def_mask] = veto[def_mask] | defect_test_output
586 veto_source[def_mask] += (0b1 & defect_test_output) << var_prop["bit"][
587 i_var
588 ]
590 else: # QAQC variable: range treatement
591 range_mask = (var_val[i_var] != MISSING_VALUE) & (
592 (var_val[i_var] < var_prop["veto_thresholds"][i_var][0]) | (
593 var_val[i_var] > var_prop["veto_thresholds"][i_var][1]
594 )
595 )
596 veto = veto | range_mask
597 veto_source[range_mask] += 0b1 << var_prop["bit"][i_var]
599 for i_var in var_prop["qsco_thresholds"]:
600 values = var_val[i_var]
601 # Defect variables
602 # Defect variables
603 # The defect variable is defined as def_[defect type]
604 # with [defect type] = daq, operation, analysis...
605 if i_var.startswith("def_"):
606 # first extract the set defects
607 def_mask = values != 0
608 if np.sum(def_mask) == 0:
609 continue
611 defect_type = i_var.replace("def_", "")
612 diag_array = {"present": [], "absent": []}
613 for i_excluded_diag in var_prop["qsco_thresholds"][i_var]:
614 if i_excluded_diag.startswith("NOT."):
615 diag_array["absent"].append(
616 def_config["bit"][defect_type][
617 i_excluded_diag.replace("NOT.", "")
618 ]
619 )
620 else:
621 diag_array["present"].append(
622 def_config["bit"][defect_type][i_excluded_diag]
623 )
625 defect_test_output = np.vectorize(check_defect_diag_byte, excluded=[1])(
626 values[def_mask], diag_array
627 )
628 qsco[def_mask] = qsco[def_mask] | defect_test_output
629 qsco_source[def_mask] += (0b1 & defect_test_output) << var_prop["bit"][
630 i_var
631 ]
633 else: # QAQC variable: range treatement
634 range_mask = (var_val[i_var] != MISSING_VALUE) & (
635 (var_val[i_var] < var_prop["qsco_thresholds"][i_var][0]) | (
636 var_val[i_var] > var_prop["qsco_thresholds"][i_var][1]
637 )
638 )
640 qsco[range_mask] -= 1
641 qsco_source[range_mask] += 0b1 << var_prop["bit"][i_var]
643 qsco /= len(var_prop["qsco_thresholds"])
645 return (veto, veto_source, qsco, qsco_source)
648########################################################################################################
649def log_run_range(det, run_0, run_1, log_file):
650 """Write the run range and run list on disk"""
652 os.environ["TZ"] = "Europe/Paris"
653 time.tzset()
655 run_prop = get_run_properties_from_db(det, "PHYS COMM")
657 try:
658 tim_0_unix = int(run_prop[run_0]["UNIXJOBSTART"])
659 tim_0 = time.strftime("%a, %d %b %Y %H:%M", time.localtime(tim_0_unix / 1000))
660 except KeyError:
661 tim_0 = "Unknown"
663 try:
664 tim_1_unix = int(run_prop[run_1]["UNIXJOBEND"])
665 tim_1 = time.strftime("%a, %d %b %Y %H:%M", time.localtime(tim_1_unix / 1000))
666 except KeyError:
667 tim_1 = "Unknown"
669 run_range = f"{int(run_0)} ({tim_0} CET/CEST) - " f"{int(run_1)} ({tim_1} CET/CEST)"
671 log_file.write(f"Run range: {run_range}\n")
674########################################################################################################
675def read_qaqc_file(det, source, qaqc_name, run_range):
676 """
677 sftp QAQC file reading
679 === Arguments ===
680 - det : detector name - [string] - Ex: "D0ARCA021", "D0ORCA018"...
681 - source : QAQC source, a priori "qaqc_sftp" - [string]
682 - qaqc_name :
683 - run_range : string as defined in DQ tag "[min_run]-[max_run]" (inclusive)
685 === Output ===
686 - TTree
687 - Error log
689 """
691 lines = []
692 if source == "qaqc_sftp":
693 with urllib.request.urlopen(
694 get_file_paths(det, qaqc_name)["JQAQC_sftp"]
695 ) as qaqc:
696 tmp = (qaqc.read()).split(b"\n")
697 for i_line in tmp:
698 if i_line != "":
699 lines.append(i_line.decode("utf-8"))
700 else:
701 if source == "qaqc_sps":
702 source_file = get_file_paths(det, qaqc_name)["JQAQC_sps"]
703 else:
704 source_file = source
706 with open(source_file, "r", encoding="utf-8") as s_f:
707 lines = s_f.readlines()
709 labels = lines[0].split()
710 run_index = labels.index("run")
711 if run_range != "":
712 min_run = int(run_range.split("-")[0])
713 max_run = int(run_range.split("-")[1])
714 else:
715 min_run = 0
716 max_run = 1e9
718 content = {}
719 for l in lines[1:]:
720 try:
721 if len(l.split()) != get_nb_qaqc_variables(l.split()[0]):
722 print(f"Bad number of variables in {l}")
723 continue
724 # sys.exit()
725 except IndexError: # Empty line
726 pass
728 try:
729 if all((int(l.split()[run_index]) >= min_run,
730 int(l.split()[run_index]) <= max_run
731 )):
732 for i, key in enumerate(labels):
733 try:
734 try:
735 content[key].append(l.split()[i])
736 except KeyError:
737 content[key] = [l.split()[i]]
738 except IndexError: # Missing variable in QAQC file
739 try:
740 content[key].append(MISSING_VALUE)
741 except KeyError:
742 content[key] = [MISSING_VALUE]
743 except IndexError: # Empty line
744 pass
746 return content
749########################################################################################################
750def decode_source_byte(value, v_prop):
751 """
752 Decode the source of degradation stored in the TTree
753 NB: the source is either a QAQC variable either a defect type (("daq",
754 "operation"...)
755 """
757 source_list = []
758 source_str = ""
760 for i_var, i_bit in v_prop["bit"].items():
761 if (int(value) >> i_bit) & 1:
762 source_list.append(i_var)
763 source_str += f"{i_var} "
765 return (source_list, source_str)