Coverage for src/km3dq_common/common_library.py: 0%
402 statements
« prev ^ index » next coverage.py v7.6.1, created at 2024-09-25 11:58 +0000
1#! /usr/bin/env python
2###############################################################################
3# Various functions used in km3dq_lw_dq and km3dq_core scripts
4#
5# Developer: Benjamin Trocme (benjamin.trocme at apc.in2p3.fr) - 2023
7import os
8import sys
9import time
10import array
11import re
12import urllib.request
13import km3db
15from km3dq_common.config_library import configure_var_bit
16from km3dq_common.config_library import configure_var_thresholds
17from km3dq_common.config_library import get_detx_caract
18from km3dq_common.config_library import configure_dataquality_tag
19from km3dq_common.config_library import WEEKLY_DETECTORS
20from km3dq_common.lw_db_defect_library import read_defect_file
21from km3dq_common.lw_db_library import decode_defect_diag_byte
24###############################################################################
25def get_file_paths(dataset, qaqc_proc_version, runnumber=0, jra_proc=""):
26 """
27 For a given dataset / detector, returns the path for various files:
28 JMonitor, JDataQuality, JRunAnalyzer
29 The run number and jra_proc are relevant only for the JRunAnalyzer files
30 jra_proc:
31 - priv: analysis level = 1, patches to TH2D coding to recompute the
32 mean/RMS of PMT rates
33 """
35 filename = {}
37 det_id = get_det_id(dataset)
38 if det_id < 100:
39 det_id_str = f"000000{det_id}"
40 else:
41 det_id_str = f"00000{det_id}"
43 # JDataMonitor file
44 if dataset == "D0ARCA021":
45 filename['JDataMonitorName'] = ("/sps/km3net/repo/data_processing/tag/"
46 "v8.1/data_processing/DataQuality/"
47 "KM3NeT_00000133_Monitor_Jpp_v17.3.2"
48 ".root")
50 # QAQC file stored on sps
51 qaqc_sps = "/sps/km3net/repo/data/raw/quality/"
52 filename['JQAQC_sps'] = qaqc_sps \
53 + f"KM3NeT_{det_id_str}_QAQC_Jpp_{qaqc_proc_version}.txt"
55 # QAQC file stored on sftp
56 qaqc_htpps = "https://sftp.km3net.de/data/quality/"
57 filename['JQAQC_sftp'] = qaqc_htpps \
58 + f"KM3NeT_{det_id_str}_QAQC_Jpp_{qaqc_proc_version}.txt"
60 # JDataQuality file - Processing by BT, available on sps
61 dq_file_path_bt = "/sps/km3net/users/trocme/Output-JDataQuality/"
62 dataquality_suffix = qaqc_proc_version.split(".", 1)[0]
63 filename['JDataQualityName'] = dq_file_path_bt\
64 + f"{dataset}/KM3NeT_{det_id_str}_QAQC_Jpp_{dataquality_suffix}.root"
66 # JRA file - Private BT processing - One file per run
67 jra_file_path_bt = "/sps/km3net/users/trocme/Output-JRunAnalyzer/"
68 if runnumber != 0:
69 if jra_proc == "priv":
70 filename['JRAName'] = \
71 f"{jra_file_path_bt}{dataset}/JRA-{runnumber:.0f}.root"
72 else: # Official production
73 filename['JRAName'] = ("/sps/km3net/repo/data/raw/quality/"
74 f"KM3NeT_{det_id_str}/"
75 f"JRA_KM3NeT_{det_id_str}_000{runnumber}"
76 ".root")
77 if os.path.exists(filename['JRAName']) is False:
78 filename['JRAName'] = "Not found"
80 return filename
83###############################################################################
84def get_run_properties_from_db(det, filt="PHYS"):
85 """
86 Retrieve run properties from the database with the km3db package
87 """
88 sds = km3db.tools.StreamDS()
89 runs = sds.runs(detid=get_det_id(det))
90 lines = runs.split("\n")
92 # Extract the available data
93 variable_list = lines[0].split("\t")
94 nb_variables = len(variable_list)
96 regex = r"([0-9]+)"
97 for i_var in range(nb_variables-1):
98 regex += r"\t(.*)"
99 regex += "$"
101 reg_line = re.compile(regex)
103 results = {}
104 for i_line in lines:
105 r_p = reg_line.search(i_line)
106 if r_p:
107 if filt not in r_p.group(5):
108 continue
109 run_nb = int(r_p.group(1))
110 results[run_nb] = {}
111 for i_var in range(2, nb_variables+1):
112 results[run_nb][variable_list[i_var-1]] = r_p.group(i_var)
114 return results
117###############################################################################
118def get_run_properties_from_qaqc(dataset, dq_tag, origin="qaqc_sftp",
119 startrun=0, endrun=1e9):
120 """
121 For a given dataset/detector, returns the run numbers, their lengths,
122 time counter...
123 A dataset may be restricted to a user-defined run range
124 When using the create_ttree_from_qaqc function, a check on the QAQC file
125 is performed
126 Source: QAQC file or JDataQualityFile
127 """
129 from ROOT import TFile
131 runprop = {"runNumber": [],
132 "lvt": [],
133 "lvt_counter": [],
134 "kty": [],
135 "kty_counter": [],
136 "nbRuns": 0, "minRun": 1e10, "maxRun": 0}
138 dataset_time_counter = 0.
139 scale_sec_days = 1/3600./24.
140 kty_counter = 0
142 err_log = ""
143 if origin == "jdq": # Source: JDataquality
144 filename_jdq = get_file_paths(dataset, "")["JDataQualityName"]
145 assert(filename_jdq != "Missing"), \
146 print("Missing DataQuality file - "
147 "Can not retrieve the run properties")
148 f_dq = TFile(filename_jdq)
149 n1_tree = f_dq.Get("n1")
150 elif "qaqc" in origin: # Source: QAQC file on sftp
151 # QAQC default tag used. As priori OK for this purpose
153 (n1_tree, err_log) = create_ttree_from_qaqc(dataset,
154 ["run", "livetime", "kton_year"],
155 origin, dq_tag)
157 entries = n1_tree.GetEntriesFast()
158 for jentry in range(entries):
159 n1_tree.GetEntry(jentry) # Load event
160 if (n1_tree.run < startrun) or (n1_tree.run > endrun):
161 continue
162 runprop["runNumber"].append(n1_tree.run)
163 if origin == "JDataQuality":
164 run_length = n1_tree.a
165 else:
166 run_length = n1_tree.livetime
167 runprop["lvt"].append(run_length*scale_sec_days)
168 runprop["lvt_counter"].append(dataset_time_counter)
169 runprop["kty"].append(n1_tree.kton_year)
170 runprop["kty_counter"].append(kty_counter)
172 dataset_time_counter = dataset_time_counter + run_length*scale_sec_days
173 kty_counter = kty_counter + n1_tree.kton_year
174 runprop["nbRuns"] = runprop["nbRuns"] + 1
175 if n1_tree.run < runprop["minRun"]:
176 runprop["minRun"] = n1_tree.run
177 if n1_tree.run > runprop["maxRun"]:
178 runprop["maxRun"] = n1_tree.run
180 # Add the last time counter to define TH1 with a vector
181 if len(runprop["lvt_counter"]) > 0:
182 runprop["lvt_counter"].append(runprop["lvt_counter"][-1]
183 + runprop["lvt"][-1])
184 if len(runprop["kty_counter"]) > 0:
185 runprop["kty_counter"].append(runprop["kty_counter"][-1]
186 + runprop["kty"][-1])
188 if origin == "JDataQuality":
189 f_dq.Close()
191 return (runprop, err_log)
194###############################################################################
195def get_last_n_weeks_runs_qaqc(dataset, nweeks=2):
196 """ Extract the all runs acquired in the last nweeks of running """
198 # QAQC default tag used. As priori OK for this purpose
199 dq_tag = configure_dataquality_tag("default")
200 lines = read_qaqc_file(dataset, "qaqc_sftp", dq_tag['qaqc_vers'][dataset])
201 min_run = 1e9
202 max_run = 0
204 first_line = True
205 for i in lines:
206 i_split = i.split(" ")
207 if first_line:
208 run_number_index = i_split.index("run")
209 utc_min_s_index = i_split.index("UTCMin_s")
210 first_line = False
211 else:
212 if len(i_split) > utc_min_s_index:
213 ten_days = (nweeks * 7 + 1) * 3600. * 24.
214 if (time.time() - float(i_split[utc_min_s_index])) < ten_days:
215 run_number = int(i_split[run_number_index])
216 if run_number < min_run:
217 min_run = run_number
218 if run_number > max_run:
219 max_run = run_number
221 return (min_run, max_run)
224###############################################################################
225def get_full_detector_name(options, key="detector"):
226 """
227 Get the full detector name with the prefix (D0, D_...)
228 when there is no ambiguity
230 The input is the options dict.
231 NB: options['detector'] can be either a single list either a list of
232 detector/strings.
233 """
235 names = {"D0ARCA009": ["ARCA009", "ARCA9"],
236 "D0ARCA020": ["ARCA020", "ARCA20"],
237 "D0ARCA021": ["ARCA021", "ARCA21"],
238 "D0ARCA028": ["ARCA028", "ARCA28"],
239 "D_ORCA006": ["ORCA006", "ORCA6"],
240 "D0ORCA007": ["ORCA007", "ORCA7"],
241 "D0ORCA010": ["ORCA010", "ORCA10"],
242 "D0ORCA011": [],
243 "D1ORCA011": [],
244 "D1ORCA013": ["ORCA013", "ORCA13"],
245 "D0ORCA015": [],
246 "D1ORCA015": [],
247 "D0ORCA018": ["ORCA018"],
248 "D1ORCA019": ["ORCA019"],
249 "D0ORCA023": ["ORCA023"]}
251 # A and O are used to refer to the current detector
252 names[WEEKLY_DETECTORS["ARCA"]].append("A")
253 names[WEEKLY_DETECTORS["ORCA"]].append("O")
255 # Single detector
256 if isinstance(options[key], str):
257 if options[key] not in names:
258 full_name_found = False
259 for i, name in names.items():
260 for j in name:
261 if options[key] == j:
262 options[key] = i
263 print(f"I am redefining the detector name as {i}")
264 full_name_found = True
265 if not full_name_found:
266 print("Ambiguous %s detector. Please use the full "
267 + "name D*ORCA*** or D*ARCA***")
268 sys.exit()
269 # List of detectors
270 elif isinstance(options[key], list):
271 for k in range(0, len(options[key])):
272 if options[key][k] not in names:
273 full_name_found = False
274 for i, name in names.items():
275 for j in name:
276 if options[key][k] == j:
277 options[key][k] = i
278 print(f"I am redefining the detector name as {i}")
279 full_name_found = True
280 if not full_name_found:
281 print(f"Ambiguous {options['detector'][k]} detector."
282 f"Please use the full"
283 f"name D*ORCA*** or D*ARCA***")
284 sys.exit()
287###############################################################################
288def get_det_id(dataset):
289 """ Return the detector id """
291 det_id = {"D0ARCA009": 94,
292 "D0ARCA020": 116,
293 "D0ARCA021": 133,
294 "D0ARCA028": 160,
295 "D_ORCA006": 49,
296 "D0ORCA007": 110,
297 "D0ORCA010": 100,
298 "D1ORCA013": 117,
299 "D0ORCA011": 123,
300 "D1ORCA011": 132,
301 "D0ORCA015": 138,
302 "D1ORCA015": 146,
303 "D0ORCA018": 148,
304 "D1ORCA019": 172,
305 "D0ORCA023": 196}
307 return det_id[dataset]
310###############################################################################
311def get_site(det):
312 """ Returns the site (ORCA or ARCA) """
314 if "ORCA" in det:
315 out = "ORCA"
316 else:
317 out = "ARCA"
318 return out
321###############################################################################
322def get_active_dus_range(det):
323 """ Retrieve the range of DU active """
324 lower_du = 0
325 upper_du = {"D0ARCA009": 32,
326 "D0ARCA020": 32,
327 "D0ARCA021": 32,
328 "D0ARCA028": 32,
329 "D_ORCA006": 32,
330 "D0ORCA007": 32,
331 "D0ORCA010": 32,
332 "D1ORCA013": 32,
333 "D0ORCA011": 32,
334 "D1ORCA011": 32,
335 "D0ORCA015": 32,
336 "D1ORCA015": 32,
337 "D0ORCA018": 32,
338 "D1ORCA019": 42,
339 "D0ORCA023": 42}
341 return (lower_du, upper_du[det])
344###############################################################################
345def get_nb_qaqc_variables(qaqc_vers):
346 """
347 Retrieve the number of variables in the QAQC file
348 NB: in a near future, the det argument should be replaced by a Jpp version
349 === Arguments ===
350 - det : detector name - [string] - Ex: "D0ARCA021", "D0ORCA018"...
352 === Output ===
353 - Number of QAQC variables
355 """
356 if "v16.0.3" in qaqc_vers:
357 return 39
358 if "v17.3.2" in qaqc_vers:
359 return 42
361 return 45
364###############################################################################
365def create_ttree_from_qaqc(det, var_to_fill, source, tag, append_veto_qsco=False):
366 """
367 Create a ttree from qaqc sftp file and defect variables stored on git
368 It includes some advanced check about the QAQC file integrity.
369 === Arguments ===
370 - det : detector name - [string] - Ex: "D0ARCA021", "D0ORCA018"...
371 - var_to_fill : QAQC variables or defect to fill - [array of string] -
372 Ex: ['run', 'timestampdiff', 'def_operation', 'def_oos']
373 - source : QAQC source, a priori "qaqc_sftp" - [string]
374 - tag : data-quality tag (not its name) as created by
375 configure_dataquality_tag
376 - append_veto_qsco: append the veto and Qscore - [boolean]
378 === Output ===
379 - TTree
380 - Error log
382 """
384 from ROOT import TTree
386 error_log = ""
388 # Correspondance between the name in the QAQC file (first line) and the
389 # variable name in the script
390 # In the case of empty variable, this corresponds to "composite" variable
391 # derived from the primary variables with an explicit computation in the
392 # function
394 variable_name_corresp = {"git": "GIT",
395 "jpp": "JPP",
396 "nb_of_meta": "nb_of_meta",
397 "uuid": "UUID",
398 "detectorid": "detector",
399 "run": "run",
400 "livetime": "livetime_s",
401 "utcmin": "UTCMin_s",
402 "utcmax": "UTCMax_s",
403 "trigger3dmuon": "trigger3DMuon",
404 "trigger3dshower": "trigger3DShower",
405 "triggermxshower": "triggerMXShower",
406 "triggernb": "triggerNB",
407 "writel0": "writeL0",
408 "writel1": "writeL1",
409 "writel2": "writeL2",
410 "writesn": "writeSN",
411 "jdaqtimeslice": "JDAQTimeslice",
412 "jdaqtimeslicel0": "JDAQTimesliceL0",
413 "jdaqtimeslicel1": "JDAQTimesliceL1",
414 "jdaqtimeslicel2": "JDAQTimesliceL2",
415 "jdaqtimeslicesn": "JDAQTimesliceSN",
416 "jdaqsummaryslice": "JDAQSummaryslice",
417 "jdaqevent": "JDAQEvent",
418 "jtriggerreprocessor": "JTriggerReprocessor",
419 "jtrigger3dshower": "JTrigger3DShower",
420 "jtriggermxshower": "JTriggerMXShower",
421 "jtrigger3dmuon": "JTrigger3DMuon",
422 "jtriggernb": "JTriggerNB",
423 "in_sync": "in_sync",
424 "oos": "out_sync",
425 "daq": "DAQ",
426 "whiterabbitstatus": "WR",
427 "hrv": "HRV",
428 "fifo": "FIFO",
429 "pmts": "PMTs",
430 "meanrate": "MEAN_Rate_Hz",
431 "rmsrate": "RMS_Rate_Hz",
432 "hrv_fifo_failures": "hrv_fifo_failures",
433 "duplic_timeslices": "duplic_timeslices",
434 "Acoustics": "Acoustics",
435 "AHRS": "AHRS",
436 "in_usync": "in_usync",
437 "out_usync": "out_usync",
438 "event_duration": "event_duration",
439 "timestampdiff": "",
440 "timestampdiff_r": "",
441 "triggerrate": "",
442 "pmts_norm": "",
443 "acoustics": "",
444 "ahrs_per_mn": "",
445 "JDAQJTProc": "",
446 "kton_year": ""}
448 if len(var_to_fill) == 0:
449 var_to_fill = list(variable_name_corresp.keys())
451 if tag["qaqc_vers"][det] == "v16.0.3":
452 # Processing version 16: name change
453 # 3 missing variables
454 variable_name_corresp['oos'] = "out-sync"
455 variable_name_corresp['in_sync'] = "in-sync"
456 for i_missing in ["in_usync", "out_usync", "event_duration"]:
457 variable_name_corresp[i_missing] = "MISSING"
458 if i_missing in var_to_fill:
459 var_to_fill.remove(i_missing)
461 if any((tag["qaqc_vers"][det] == "v16.0.3",
462 tag["qaqc_vers"][det] == "v17.3.2")):
463 # Processing version 16/17: 3 other missing variables
464 for i_missing in ["nb_of_meta", "hrv_fifo_failures", "duplic_timeslices"]:
465 variable_name_corresp[i_missing] = "MISSING"
466 if i_missing in var_to_fill:
467 var_to_fill.remove(i_missing)
469 variable_float = ["livetime", "daq", "whiterabbitstatus", "fifo", "hrv",
470 "pmts", "pmts_norm", "timestampdiff", "timestampdiff_r",
471 "triggerrate", "acoustics",
472 "ahrs_per_mn", "JDAQJTProc", "kton_year"]
474 var_properties = {}
475 configure_var_thresholds(var_properties, det, tag)
476 configure_var_bit(var_properties)
478 new_tree = TTree("t1", "t1")
480 lines = read_qaqc_file(det, source, tag['qaqc_vers'][det])
481 defect_results = read_defect_file(det, tag['def_tag'])
483 # Define the regular expression for the QAQC file
484 regex0 = r""
485 nb_qaqc_variables = get_nb_qaqc_variables(tag['qaqc_vers'][det])
486 for i in range(nb_qaqc_variables - 1):
487 regex0 += r"\s*(\S+)\s+"
488 regex0 += r"(\S+)\n*\Z"
489 qaqc_pattern = re.compile(regex0)
490 ####################################################################
491 # Check the regular expression of the first line of the QAQC file to
492 # retrieve the list of available variables
493 qaqc_first_line = qaqc_pattern.match(lines[0])
494 if not qaqc_first_line:
495 error_log += "Wrong regex for the list of variable of the QAQC file\n"
496 error_log += "Exiting..."
497 return (new_tree, error_log)
499 var_list_qaqc = []
500 for i_var in range(1, nb_qaqc_variables + 1):
501 var_list_qaqc.append(qaqc_first_line.group(i_var))
502 # var_list_qaqc = (lines[0].replace("\n", "")).split(" ")
504 qaqc_index = {}
505 var_value = {}
506 ######################################################################
507 # Loop on all QAQC variables to fill, create all relevant branches and
508 # associate them to the array used for tree filling.
509 for i_var in var_to_fill:
510 if i_var.startswith("def_") is True:
511 continue # Defect variable are treated hereafter
512 if variable_name_corresp[i_var] == "MISSING":
513 # Missing variable (in v16 and v17 compared to v19)
514 continue
515 if any((variable_name_corresp[i_var] in var_list_qaqc,
516 variable_name_corresp[i_var] == "")):
517 if variable_name_corresp[i_var] in var_list_qaqc:
518 qaqc_index[i_var] = var_list_qaqc\
519 .index(variable_name_corresp[i_var])
520 else:
521 qaqc_index[i_var] = 1000 # Composite variable
522 if i_var in variable_float:
523 var_value[i_var] = array.array('f', [0.])
524 else:
525 var_value[i_var] = array.array('i', [0])
526 if i_var in variable_float:
527 new_tree.Branch(f"{i_var}", var_value[i_var], f"{i_var}/F")
528 else:
529 new_tree.Branch(f"{i_var}", var_value[i_var], f"{i_var}/i")
530 else:
531 err = (f"ERROR: {variable_name_corresp[i_var]}"
532 f" not in QAQC fill -> Exiting")
533 error_log = error_log + err
534 if error_log != "":
535 print(error_log)
536 return (new_tree, error_log)
538 ########################################################################
539 # Loop on all defect variables to fill, create all relevant branches and
540 # associate them to the array used for tree filling.
541 for i_var in var_to_fill:
542 if i_var.startswith("def_") is False:
543 continue # QAQC variable are treated herebefore
544 var_value[i_var] = array.array('i', [0])
545 new_tree.Branch(f"{i_var}", var_value[i_var], f"{i_var}/i")
547 ###########################################
548 # Create the branch to fill the veto/Qscore
549 if append_veto_qsco:
550 for i_var in ("veto", "veto_source", "qsco_source"):
551 var_value[i_var] = array.array('i', [0])
552 new_tree.Branch(f"{i_var}", var_value[i_var], f"{i_var}/i")
553 var_value['qsco'] = array.array('f', [0])
554 new_tree.Branch("qsco", var_value['qsco'], "qsco/f")
556 #######################################################################
557 # Extract the QAQC index of the QAQC variables used for the computation
558 # of composite variables
559 for i_var in ["UTCMax_s", "UTCMin_s", "livetime_s", "JDAQEvent", "PMTs",
560 "Acoustics", "AHRS", "JTriggerReprocessor", "HRV"]:
561 if i_var in var_list_qaqc:
562 qaqc_index[i_var] = var_list_qaqc.index(i_var)
563 else:
564 err = f"Missing {i_var} in QAQC file"
565 error_log = error_log + err
566 qaqc_index[i_var] = -1
568 #####################################
569 # Loop on all runs to fill the arrays
570 run_stored = []
571 for i_line in range(1, len(lines)):
572 if lines[i_line] == "":
573 continue
574 qaqc_regex = qaqc_pattern.match(lines[i_line])
575 # Bad line format -> Line skipped
576 if not qaqc_regex:
577 error_log += ("QAQC line with a bad format:"
578 f"{lines[i_line]}")
579 error_log += "->Line skipped\n"
580 if error_log != "":
581 print(error_log)
582 continue
584 var_value_re = []
585 for i in range(1, nb_qaqc_variables + 1):
586 var_value_re.append(qaqc_regex.group(i))
588 # Run duplicated in QAQC file -> Line skipped
589 run_number = var_value_re[qaqc_index["run"]]
590 if run_number in run_stored:
591 if f"Duplicated run {run_number}" not in error_log:
592 error_log += f"Duplicated run {run_number} in QAQC file"
593 error_log += "(1st occurence kept)\n"
594 continue
596 # Run with a zero livetime -> Line skipped
597 livetime = float(var_value_re[qaqc_index["livetime_s"]])
598 if livetime == 0.:
599 if f"Run {run_number} has a livetime = 0" not in error_log:
600 error_log += f"Run {run_number} has a livetime = 0"
601 continue
603 # Filling the QAQC variable values
604 for i_var in var_to_fill:
605 if i_var.startswith("def_") is True:
606 continue # Defect variable are treated hereafter
608 if qaqc_index[i_var] < len(var_value_re):
609 str_fl = var_value_re[qaqc_index[i_var]]
610 if i_var in variable_float:
611 try:
612 var_value[i_var][0] = float(str_fl)
613 except ValueError:
614 err = (f"Corrupted variable in QAQC file: {i_var}\n "
615 f"- {lines[i_line]}\n")
616 error_log = error_log + (f"{err}\n")
617 continue
618 else:
619 try:
620 var_value[i_var][0] = int(str_fl.split(".")[0])
621 except ValueError:
622 err = (f"Corrupted variable {i_var} found in qaqc file"
623 f"- {lines[i_line]} -"
624 "Variable skipped for this run")
625 error_log += f"{err}\n"
626 continue
628 elif qaqc_index[i_var] == 1000: # Composite variable
629 tmp_utcmax = float(var_value_re[qaqc_index["UTCMax_s"]])
630 tmp_utcmin = float(var_value_re[qaqc_index["UTCMin_s"]])
631 tmp_livetime = float(var_value_re[qaqc_index["livetime_s"]])
632 tmp_pmts = float(var_value_re[qaqc_index["PMTs"]])
633 tmp_JDAQEvent = float(var_value_re[qaqc_index["JDAQEvent"]])
634 tmp_Acoustics = float(var_value_re[qaqc_index["Acoustics"]])
635 tmp_AHRS = float(var_value_re[qaqc_index["AHRS"]])
636 tmp_JTriggerReprocessor = \
637 float(var_value_re[qaqc_index["JTriggerReprocessor"]])
638 tmp_kton_year = 6980 / (31 * 18 * 115) \
639 * float(var_value_re[qaqc_index["PMTs"]]) \
640 * (1 - float(var_value_re[qaqc_index["HRV"]])) \
641 * tmp_livetime / (365.2425 * 24 * 3600)
642 if i_var == "timestampdiff":
643 var_value[i_var][0] = (tmp_utcmax - tmp_utcmin)\
644 - tmp_livetime
645 if i_var == "timestampdiff_r":
646 var_value[i_var][0] = 1. - tmp_livetime / \
647 (tmp_utcmax - tmp_utcmin)
648 if i_var == "pmts_norm": # NOT YET NORMALIZED :-(
649 var_value[i_var][0] = tmp_pmts / \
650 get_detx_caract(det)['pmts']
651 elif i_var == "triggerrate":
652 var_value[i_var][0] = tmp_JDAQEvent / tmp_livetime
653 elif i_var == "acoustics":
654 var_value[i_var][0] = tmp_Acoustics
655 elif i_var == "ahrs_per_mn":
656 var_value[i_var][0] = tmp_AHRS / tmp_livetime * 60
657 elif i_var == "JDAQJTProc":
658 var_value[i_var][0] = \
659 (tmp_JDAQEvent - tmp_JTriggerReprocessor) / \
660 (tmp_JDAQEvent + 1.0e-10)
661 elif i_var == "kton_year":
662 var_value[i_var][0] = tmp_kton_year
663 else:
664 err = (f"Corrupted lines found in qaqc file - {lines[i_line]}"
665 " - Line skipped")
666 error_log = error_log + (f"{err}\n")
668 # Filling the QAQC variable values
669 for i_var in var_to_fill:
670 if i_var.startswith("def_") is False:
671 continue # QAQC variable are treated herebefore
672 var_value[i_var][0] = 0 # No defect
673 if int(run_number) in list(defect_results[i_var].keys()):
674 var_value[i_var][0] = defect_results[i_var][int(run_number)]
676 if append_veto_qsco:
677 (var_value['veto'][0], var_value['veto_source'][0],
678 var_value['qsco'][0], var_value['qsco_source'][0]) = \
679 compute_veto_qscore(var_properties, var_value)
681 run_stored.append(run_number)
683 new_tree.Fill()
685 new_tree.ResetBranchAddresses()
687 if error_log != "":
688 print(error_log)
689 return (new_tree, error_log)
692###############################################################################
693def compute_veto_qscore(var_prop, var_val):
694 """
695 Computes the veto/Q-score for a detector/tag using a QAQC source.
696 """
698 veto = False
699 veto_source = 0b0
701 qsco = 1.
702 qsco_source = 0b0
704 for i_var in var_prop['veto_thresholds']:
705 # Defect variables
706 if i_var.startswith("def_"):
707 # first extract the set defects
708 decoded_defects = decode_defect_diag_byte(var_val[i_var][0], i_var)
709 # Then loop on the various defects used in veto
710 # Treatement of NOT. defect added on February, 2. Not fully tested.
711 for i_excluded_def in var_prop['veto_thresholds'][i_var]:
712 defect_test = False
713 if i_excluded_def in decoded_defects:
714 defect_test = True
715 if i_excluded_def.startswith("NOT."):
716 if i_excluded_def.replace("NOT.", "") not in decoded_defects:
717 defect_test = True
719 if defect_test:
720 veto = True
721 veto_source = veto_source | 0b1 << var_prop['bit'][i_var]
722 continue
724 # QAQC variables
725 if any((var_val[i_var][0] < var_prop['veto_thresholds'][i_var][0],
726 var_val[i_var][0] > var_prop['veto_thresholds'][i_var][1])):
727 veto = True
728 veto_source += 0b1 << var_prop['bit'][i_var]
730 for i_var in var_prop['qsco_thresholds']:
731 # Defect variables
732 if i_var.startswith("def_"):
733 # first extract the set defects
734 decoded_defects = decode_defect_diag_byte(var_val[i_var][0], i_var)
735 # Then loop on the various defects used in veto
736 for i_excluded_def in var_prop['qsco_thresholds'][i_var]:
737 if i_excluded_def in decoded_defects:
738 qsco -= 1./float(len(var_prop['qsco_thresholds']))
739 qsco_source = qsco_source | 0b1 << var_prop['bit'][i_var]
740 continue
742 if any((var_val[i_var][0] < var_prop['qsco_thresholds'][i_var][0],
743 var_val[i_var][0] > var_prop['qsco_thresholds'][i_var][1])):
744 qsco -= 1./float(len(var_prop['qsco_thresholds']))
745 qsco_source += 0b1 << var_prop['bit'][i_var]
747 return (int(veto), int(veto_source), qsco, int(qsco_source))
750###############################################################################
751def log_run_range(det, run_0, run_1, log_file):
752 """ Write the run range and run list on disk """
754 os.environ['TZ'] = 'Europe/Paris'
755 time.tzset()
757 run_prop = get_run_properties_from_db(det)
759 if run_0 in run_prop.keys():
760 tim_0_unix = int(run_prop[run_0]['UNIXJOBSTART'])
761 tim_0 = time.strftime("%a, %d %b %Y %H:%M",
762 time.localtime(tim_0_unix/1000))
763 else:
764 tim_0 = "Unknown"
766 if run_1 in run_prop.keys():
767 tim_1_unix = int(run_prop[run_1]['UNIXJOBEND'])
768 tim_1 = time.strftime("%a, %d %b %Y %H:%M",
769 time.localtime(tim_1_unix/1000))
770 else:
771 tim_1 = "Unknown"
773 run_range = (f"{int(run_0)} ({tim_0} CET/CEST) - "
774 f"{int(run_1)} ({tim_1} CET/CEST)")
776 log_file.write(f"Run range: {run_range}\n")
779###############################################################################
780def read_qaqc_file(det, source="qaqc_sftp", qaqc_proc_version=""):
781 """
782 sftp QAQC file reading
783 """
785 lines = []
786 if source == "qaqc_sftp":
787 with urllib.request.urlopen(
788 get_file_paths(det, qaqc_proc_version)["JQAQC_sftp"]) as qaqc:
789 tmp = ((qaqc.read()).split(b"\n"))
790 for i_line in tmp:
791 if i_line != "":
792 lines.append(i_line.decode("utf-8"))
793 else:
794 if source == "qaqc_sps":
795 source_file = get_file_paths(det, qaqc_proc_version)["JQAQC_sps"]
796 else:
797 source_file = source
799 with open(source_file, "r", encoding="utf-8") as s_f:
800 lines = s_f.readlines()
802 return lines
805###############################################################################
806def decode_source_byte(value, v_prop):
807 """
808 Decode the source of degradation stored in the TTree
809 NB: the source is either a QAQC variable either a defect type (("daq",
810 "operation"...)
811 """
813 from ROOT import TTree
815 source_list = []
816 source_str = ""
818 for (i_var, i_bit) in v_prop['bit'].items():
819 if (int(value) >> i_bit) & 1:
820 source_list.append(i_var)
821 source_str += f"{i_var} "
823 return (source_list, source_str)