#! /usr/bin/env python
###############################################################################
# Various functions used in km3dq_lw_dq and km3dq_core scripts
#
# Developer: Benjamin Trocme (benjamin.trocme at apc.in2p3.fr) - 2023
import os
import sys
import time
import array
import re
import urllib.request
import km3db
from km3dq_common.config_library import configure_var_bit
from km3dq_common.config_library import configure_var_thresholds
from km3dq_common.config_library import get_detx_caract
from km3dq_common.config_library import configure_dataquality_tag
from km3dq_common.config_library import WEEKLY_DETECTORS
from km3dq_common.lw_db_defect_library import read_defect_file
from km3dq_common.lw_db_library import decode_defect_diag_byte
###############################################################################
[docs]
def get_file_paths(dataset, qaqc_proc_version, runnumber=0, jra_proc=""):
"""
For a given dataset / detector, returns the path for various files:
JMonitor, JDataQuality, JRunAnalyzer
The run number and jra_proc are relevant only for the JRunAnalyzer files
jra_proc:
- priv: analysis level = 1, patches to TH2D coding to recompute the
mean/RMS of PMT rates
"""
filename = {}
det_id = get_det_id(dataset)
if det_id < 100:
det_id_str = f"000000{det_id}"
else:
det_id_str = f"00000{det_id}"
# JDataMonitor file
if dataset == "D0ARCA021":
filename['JDataMonitorName'] = ("/sps/km3net/repo/data_processing/tag/"
"v8.1/data_processing/DataQuality/"
"KM3NeT_00000133_Monitor_Jpp_v17.3.2"
".root")
# QAQC file stored on sps
qaqc_sps = "/sps/km3net/repo/data/raw/quality/"
filename['JQAQC_sps'] = qaqc_sps \
+ f"KM3NeT_{det_id_str}_QAQC_Jpp_{qaqc_proc_version}.txt"
# QAQC file stored on sftp
qaqc_htpps = "https://sftp.km3net.de/data/quality/"
filename['JQAQC_sftp'] = qaqc_htpps \
+ f"KM3NeT_{det_id_str}_QAQC_Jpp_{qaqc_proc_version}.txt"
# JDataQuality file - Processing by BT, available on sps
dq_file_path_bt = "/sps/km3net/users/trocme/Output-JDataQuality/"
dataquality_suffix = qaqc_proc_version.split(".", 1)[0]
filename['JDataQualityName'] = dq_file_path_bt\
+ f"{dataset}/KM3NeT_{det_id_str}_QAQC_Jpp_{dataquality_suffix}.root"
# JRA file - Private BT processing - One file per run
jra_file_path_bt = "/sps/km3net/users/trocme/Output-JRunAnalyzer/"
if runnumber != 0:
if jra_proc == "priv":
filename['JRAName'] = \
f"{jra_file_path_bt}{dataset}/JRA-{runnumber:.0f}.root"
else: # Official production
filename['JRAName'] = ("/sps/km3net/repo/data/raw/quality/"
f"KM3NeT_{det_id_str}/"
f"JRA_KM3NeT_{det_id_str}_000{runnumber}"
".root")
if os.path.exists(filename['JRAName']) is False:
filename['JRAName'] = "Not found"
return filename
###############################################################################
[docs]
def get_run_properties_from_db(det, filt="PHYS"):
"""
Retrieve run properties from the database with the km3db package
"""
sds = km3db.tools.StreamDS()
runs = sds.runs(detid=get_det_id(det))
lines = runs.split("\n")
# Extract the available data
variable_list = lines[0].split("\t")
nb_variables = len(variable_list)
regex = r"([0-9]+)"
for i_var in range(nb_variables-1):
regex += r"\t(.*)"
regex += "$"
reg_line = re.compile(regex)
results = {}
for i_line in lines:
r_p = reg_line.search(i_line)
if r_p:
if filt not in r_p.group(5):
continue
run_nb = int(r_p.group(1))
results[run_nb] = {}
for i_var in range(2, nb_variables+1):
results[run_nb][variable_list[i_var-1]] = r_p.group(i_var)
return results
###############################################################################
[docs]
def get_run_properties_from_qaqc(dataset, dq_tag, origin="qaqc_sftp",
startrun=0, endrun=1e9):
"""
For a given dataset/detector, returns the run numbers, their lengths,
time counter...
A dataset may be restricted to a user-defined run range
When using the create_ttree_from_qaqc function, a check on the QAQC file
is performed
Source: QAQC file or JDataQualityFile
"""
from ROOT import TFile
runprop = {"runNumber": [],
"lvt": [],
"lvt_counter": [],
"kty": [],
"kty_counter": [],
"nbRuns": 0, "minRun": 1e10, "maxRun": 0}
dataset_time_counter = 0.
scale_sec_days = 1/3600./24.
kty_counter = 0
err_log = ""
if origin == "jdq": # Source: JDataquality
filename_jdq = get_file_paths(dataset, "")["JDataQualityName"]
assert(filename_jdq != "Missing"), \
print("Missing DataQuality file - "
"Can not retrieve the run properties")
f_dq = TFile(filename_jdq)
n1_tree = f_dq.Get("n1")
elif "qaqc" in origin: # Source: QAQC file on sftp
# QAQC default tag used. As priori OK for this purpose
(n1_tree, err_log) = create_ttree_from_qaqc(dataset,
["run", "livetime", "kton_year"],
origin, dq_tag)
entries = n1_tree.GetEntriesFast()
for jentry in range(entries):
n1_tree.GetEntry(jentry) # Load event
if (n1_tree.run < startrun) or (n1_tree.run > endrun):
continue
runprop["runNumber"].append(n1_tree.run)
if origin == "JDataQuality":
run_length = n1_tree.a
else:
run_length = n1_tree.livetime
runprop["lvt"].append(run_length*scale_sec_days)
runprop["lvt_counter"].append(dataset_time_counter)
runprop["kty"].append(n1_tree.kton_year)
runprop["kty_counter"].append(kty_counter)
dataset_time_counter = dataset_time_counter + run_length*scale_sec_days
kty_counter = kty_counter + n1_tree.kton_year
runprop["nbRuns"] = runprop["nbRuns"] + 1
if n1_tree.run < runprop["minRun"]:
runprop["minRun"] = n1_tree.run
if n1_tree.run > runprop["maxRun"]:
runprop["maxRun"] = n1_tree.run
# Add the last time counter to define TH1 with a vector
if len(runprop["lvt_counter"]) > 0:
runprop["lvt_counter"].append(runprop["lvt_counter"][-1]
+ runprop["lvt"][-1])
if len(runprop["kty_counter"]) > 0:
runprop["kty_counter"].append(runprop["kty_counter"][-1]
+ runprop["kty"][-1])
if origin == "JDataQuality":
f_dq.Close()
return (runprop, err_log)
###############################################################################
[docs]
def get_last_n_weeks_runs_qaqc(dataset, nweeks=2):
""" Extract the all runs acquired in the last nweeks of running """
# QAQC default tag used. As priori OK for this purpose
dq_tag = configure_dataquality_tag("default")
lines = read_qaqc_file(dataset, "qaqc_sftp", dq_tag['qaqc_vers'][dataset])
min_run = 1e9
max_run = 0
first_line = True
for i in lines:
i_split = i.split(" ")
if first_line:
run_number_index = i_split.index("run")
utc_min_s_index = i_split.index("UTCMin_s")
first_line = False
else:
if len(i_split) > utc_min_s_index:
ten_days = (nweeks * 7 + 1) * 3600. * 24.
if (time.time() - float(i_split[utc_min_s_index])) < ten_days:
run_number = int(i_split[run_number_index])
if run_number < min_run:
min_run = run_number
if run_number > max_run:
max_run = run_number
return (min_run, max_run)
###############################################################################
[docs]
def get_full_detector_name(options, key="detector"):
"""
Get the full detector name with the prefix (D0, D_...)
when there is no ambiguity
The input is the options dict.
NB: options['detector'] can be either a single list either a list of
detector/strings.
"""
names = {"D0ARCA009": ["ARCA009", "ARCA9"],
"D0ARCA020": ["ARCA020", "ARCA20"],
"D0ARCA021": ["ARCA021", "ARCA21"],
"D0ARCA028": ["ARCA028", "ARCA28"],
"D_ORCA006": ["ORCA006", "ORCA6"],
"D0ORCA007": ["ORCA007", "ORCA7"],
"D0ORCA010": ["ORCA010", "ORCA10"],
"D0ORCA011": [],
"D1ORCA011": [],
"D1ORCA013": ["ORCA013", "ORCA13"],
"D0ORCA015": [],
"D1ORCA015": [],
"D0ORCA018": ["ORCA018"],
"D1ORCA019": ["ORCA019"],
"D0ORCA023": ["ORCA023"]}
# A and O are used to refer to the current detector
names[WEEKLY_DETECTORS["ARCA"]].append("A")
names[WEEKLY_DETECTORS["ORCA"]].append("O")
# Single detector
if isinstance(options[key], str):
if options[key] not in names:
full_name_found = False
for i, name in names.items():
for j in name:
if options[key] == j:
options[key] = i
print(f"I am redefining the detector name as {i}")
full_name_found = True
if not full_name_found:
print("Ambiguous %s detector. Please use the full "
+ "name D*ORCA*** or D*ARCA***")
sys.exit()
# List of detectors
elif isinstance(options[key], list):
for k in range(0, len(options[key])):
if options[key][k] not in names:
full_name_found = False
for i, name in names.items():
for j in name:
if options[key][k] == j:
options[key][k] = i
print(f"I am redefining the detector name as {i}")
full_name_found = True
if not full_name_found:
print(f"Ambiguous {options['detector'][k]} detector."
f"Please use the full"
f"name D*ORCA*** or D*ARCA***")
sys.exit()
###############################################################################
[docs]
def get_det_id(dataset):
""" Return the detector id """
det_id = {"D0ARCA009": 94,
"D0ARCA020": 116,
"D0ARCA021": 133,
"D0ARCA028": 160,
"D_ORCA006": 49,
"D0ORCA007": 110,
"D0ORCA010": 100,
"D1ORCA013": 117,
"D0ORCA011": 123,
"D1ORCA011": 132,
"D0ORCA015": 138,
"D1ORCA015": 146,
"D0ORCA018": 148,
"D1ORCA019": 172,
"D0ORCA023": 196}
return det_id[dataset]
###############################################################################
[docs]
def get_site(det):
""" Returns the site (ORCA or ARCA) """
if "ORCA" in det:
out = "ORCA"
else:
out = "ARCA"
return out
###############################################################################
[docs]
def get_active_dus_range(det):
""" Retrieve the range of DU active """
lower_du = 0
upper_du = {"D0ARCA009": 32,
"D0ARCA020": 32,
"D0ARCA021": 32,
"D0ARCA028": 32,
"D_ORCA006": 32,
"D0ORCA007": 32,
"D0ORCA010": 32,
"D1ORCA013": 32,
"D0ORCA011": 32,
"D1ORCA011": 32,
"D0ORCA015": 32,
"D1ORCA015": 32,
"D0ORCA018": 32,
"D1ORCA019": 42,
"D0ORCA023": 42}
return (lower_du, upper_du[det])
###############################################################################
[docs]
def get_nb_qaqc_variables(qaqc_vers):
"""
Retrieve the number of variables in the QAQC file
NB: in a near future, the det argument should be replaced by a Jpp version
=== Arguments ===
- det : detector name - [string] - Ex: "D0ARCA021", "D0ORCA018"...
=== Output ===
- Number of QAQC variables
"""
if "v16.0.3" in qaqc_vers:
return 39
if "v17.3.2" in qaqc_vers:
return 42
return 45
###############################################################################
[docs]
def create_ttree_from_qaqc(det, var_to_fill, source, tag, append_veto_qsco=False):
"""
Create a ttree from qaqc sftp file and defect variables stored on git
It includes some advanced check about the QAQC file integrity.
=== Arguments ===
- det : detector name - [string] - Ex: "D0ARCA021", "D0ORCA018"...
- var_to_fill : QAQC variables or defect to fill - [array of string] -
Ex: ['run', 'timestampdiff', 'def_operation', 'def_oos']
- source : QAQC source, a priori "qaqc_sftp" - [string]
- tag : data-quality tag (not its name) as created by
configure_dataquality_tag
- append_veto_qsco: append the veto and Qscore - [boolean]
=== Output ===
- TTree
- Error log
"""
from ROOT import TTree
error_log = ""
# Correspondance between the name in the QAQC file (first line) and the
# variable name in the script
# In the case of empty variable, this corresponds to "composite" variable
# derived from the primary variables with an explicit computation in the
# function
variable_name_corresp = {"git": "GIT",
"jpp": "JPP",
"nb_of_meta": "nb_of_meta",
"uuid": "UUID",
"detectorid": "detector",
"run": "run",
"livetime": "livetime_s",
"utcmin": "UTCMin_s",
"utcmax": "UTCMax_s",
"trigger3dmuon": "trigger3DMuon",
"trigger3dshower": "trigger3DShower",
"triggermxshower": "triggerMXShower",
"triggernb": "triggerNB",
"writel0": "writeL0",
"writel1": "writeL1",
"writel2": "writeL2",
"writesn": "writeSN",
"jdaqtimeslice": "JDAQTimeslice",
"jdaqtimeslicel0": "JDAQTimesliceL0",
"jdaqtimeslicel1": "JDAQTimesliceL1",
"jdaqtimeslicel2": "JDAQTimesliceL2",
"jdaqtimeslicesn": "JDAQTimesliceSN",
"jdaqsummaryslice": "JDAQSummaryslice",
"jdaqevent": "JDAQEvent",
"jtriggerreprocessor": "JTriggerReprocessor",
"jtrigger3dshower": "JTrigger3DShower",
"jtriggermxshower": "JTriggerMXShower",
"jtrigger3dmuon": "JTrigger3DMuon",
"jtriggernb": "JTriggerNB",
"in_sync": "in_sync",
"oos": "out_sync",
"daq": "DAQ",
"whiterabbitstatus": "WR",
"hrv": "HRV",
"fifo": "FIFO",
"pmts": "PMTs",
"meanrate": "MEAN_Rate_Hz",
"rmsrate": "RMS_Rate_Hz",
"hrv_fifo_failures": "hrv_fifo_failures",
"duplic_timeslices": "duplic_timeslices",
"Acoustics": "Acoustics",
"AHRS": "AHRS",
"in_usync": "in_usync",
"out_usync": "out_usync",
"event_duration": "event_duration",
"timestampdiff": "",
"timestampdiff_r": "",
"triggerrate": "",
"pmts_norm": "",
"acoustics": "",
"ahrs_per_mn": "",
"JDAQJTProc": "",
"kton_year": ""}
if len(var_to_fill) == 0:
var_to_fill = list(variable_name_corresp.keys())
if tag["qaqc_vers"][det] == "v16.0.3":
# Processing version 16: name change
# 3 missing variables
variable_name_corresp['oos'] = "out-sync"
variable_name_corresp['in_sync'] = "in-sync"
for i_missing in ["in_usync", "out_usync", "event_duration"]:
variable_name_corresp[i_missing] = "MISSING"
if i_missing in var_to_fill:
var_to_fill.remove(i_missing)
if any((tag["qaqc_vers"][det] == "v16.0.3",
tag["qaqc_vers"][det] == "v17.3.2")):
# Processing version 16/17: 3 other missing variables
for i_missing in ["nb_of_meta", "hrv_fifo_failures", "duplic_timeslices"]:
variable_name_corresp[i_missing] = "MISSING"
if i_missing in var_to_fill:
var_to_fill.remove(i_missing)
variable_float = ["livetime", "daq", "whiterabbitstatus", "fifo", "hrv",
"pmts", "pmts_norm", "timestampdiff", "timestampdiff_r",
"triggerrate", "acoustics",
"ahrs_per_mn", "JDAQJTProc", "kton_year"]
var_properties = {}
configure_var_thresholds(var_properties, det, tag)
configure_var_bit(var_properties)
new_tree = TTree("t1", "t1")
lines = read_qaqc_file(det, source, tag['qaqc_vers'][det])
defect_results = read_defect_file(det, tag['def_tag'])
# Define the regular expression for the QAQC file
regex0 = r""
nb_qaqc_variables = get_nb_qaqc_variables(tag['qaqc_vers'][det])
for i in range(nb_qaqc_variables - 1):
regex0 += r"\s*(\S+)\s+"
regex0 += r"(\S+)\n*\Z"
qaqc_pattern = re.compile(regex0)
####################################################################
# Check the regular expression of the first line of the QAQC file to
# retrieve the list of available variables
qaqc_first_line = qaqc_pattern.match(lines[0])
if not qaqc_first_line:
error_log += "Wrong regex for the list of variable of the QAQC file\n"
error_log += "Exiting..."
return (new_tree, error_log)
var_list_qaqc = []
for i_var in range(1, nb_qaqc_variables + 1):
var_list_qaqc.append(qaqc_first_line.group(i_var))
# var_list_qaqc = (lines[0].replace("\n", "")).split(" ")
qaqc_index = {}
var_value = {}
######################################################################
# Loop on all QAQC variables to fill, create all relevant branches and
# associate them to the array used for tree filling.
for i_var in var_to_fill:
if i_var.startswith("def_") is True:
continue # Defect variable are treated hereafter
if variable_name_corresp[i_var] == "MISSING":
# Missing variable (in v16 and v17 compared to v19)
continue
if any((variable_name_corresp[i_var] in var_list_qaqc,
variable_name_corresp[i_var] == "")):
if variable_name_corresp[i_var] in var_list_qaqc:
qaqc_index[i_var] = var_list_qaqc\
.index(variable_name_corresp[i_var])
else:
qaqc_index[i_var] = 1000 # Composite variable
if i_var in variable_float:
var_value[i_var] = array.array('f', [0.])
else:
var_value[i_var] = array.array('i', [0])
if i_var in variable_float:
new_tree.Branch(f"{i_var}", var_value[i_var], f"{i_var}/F")
else:
new_tree.Branch(f"{i_var}", var_value[i_var], f"{i_var}/i")
else:
err = (f"ERROR: {variable_name_corresp[i_var]}"
f" not in QAQC fill -> Exiting")
error_log = error_log + err
if error_log != "":
print(error_log)
return (new_tree, error_log)
########################################################################
# Loop on all defect variables to fill, create all relevant branches and
# associate them to the array used for tree filling.
for i_var in var_to_fill:
if i_var.startswith("def_") is False:
continue # QAQC variable are treated herebefore
var_value[i_var] = array.array('i', [0])
new_tree.Branch(f"{i_var}", var_value[i_var], f"{i_var}/i")
###########################################
# Create the branch to fill the veto/Qscore
if append_veto_qsco:
for i_var in ("veto", "veto_source", "qsco_source"):
var_value[i_var] = array.array('i', [0])
new_tree.Branch(f"{i_var}", var_value[i_var], f"{i_var}/i")
var_value['qsco'] = array.array('f', [0])
new_tree.Branch("qsco", var_value['qsco'], "qsco/f")
#######################################################################
# Extract the QAQC index of the QAQC variables used for the computation
# of composite variables
for i_var in ["UTCMax_s", "UTCMin_s", "livetime_s", "JDAQEvent", "PMTs",
"Acoustics", "AHRS", "JTriggerReprocessor", "HRV"]:
if i_var in var_list_qaqc:
qaqc_index[i_var] = var_list_qaqc.index(i_var)
else:
err = f"Missing {i_var} in QAQC file"
error_log = error_log + err
qaqc_index[i_var] = -1
#####################################
# Loop on all runs to fill the arrays
run_stored = []
for i_line in range(1, len(lines)):
if lines[i_line] == "":
continue
qaqc_regex = qaqc_pattern.match(lines[i_line])
# Bad line format -> Line skipped
if not qaqc_regex:
error_log += ("QAQC line with a bad format:"
f"{lines[i_line]}")
error_log += "->Line skipped\n"
if error_log != "":
print(error_log)
continue
var_value_re = []
for i in range(1, nb_qaqc_variables + 1):
var_value_re.append(qaqc_regex.group(i))
# Run duplicated in QAQC file -> Line skipped
run_number = var_value_re[qaqc_index["run"]]
if run_number in run_stored:
if f"Duplicated run {run_number}" not in error_log:
error_log += f"Duplicated run {run_number} in QAQC file"
error_log += "(1st occurence kept)\n"
continue
# Run with a zero livetime -> Line skipped
livetime = float(var_value_re[qaqc_index["livetime_s"]])
if livetime == 0.:
if f"Run {run_number} has a livetime = 0" not in error_log:
error_log += f"Run {run_number} has a livetime = 0"
continue
# Filling the QAQC variable values
for i_var in var_to_fill:
if i_var.startswith("def_") is True:
continue # Defect variable are treated hereafter
if qaqc_index[i_var] < len(var_value_re):
str_fl = var_value_re[qaqc_index[i_var]]
if i_var in variable_float:
try:
var_value[i_var][0] = float(str_fl)
except ValueError:
err = (f"Corrupted variable in QAQC file: {i_var}\n "
f"- {lines[i_line]}\n")
error_log = error_log + (f"{err}\n")
continue
else:
try:
var_value[i_var][0] = int(str_fl.split(".")[0])
except ValueError:
err = (f"Corrupted variable {i_var} found in qaqc file"
f"- {lines[i_line]} -"
"Variable skipped for this run")
error_log += f"{err}\n"
continue
elif qaqc_index[i_var] == 1000: # Composite variable
tmp_utcmax = float(var_value_re[qaqc_index["UTCMax_s"]])
tmp_utcmin = float(var_value_re[qaqc_index["UTCMin_s"]])
tmp_livetime = float(var_value_re[qaqc_index["livetime_s"]])
tmp_pmts = float(var_value_re[qaqc_index["PMTs"]])
tmp_JDAQEvent = float(var_value_re[qaqc_index["JDAQEvent"]])
tmp_Acoustics = float(var_value_re[qaqc_index["Acoustics"]])
tmp_AHRS = float(var_value_re[qaqc_index["AHRS"]])
tmp_JTriggerReprocessor = \
float(var_value_re[qaqc_index["JTriggerReprocessor"]])
tmp_kton_year = 6980 / (31 * 18 * 115) \
* float(var_value_re[qaqc_index["PMTs"]]) \
* (1 - float(var_value_re[qaqc_index["HRV"]])) \
* tmp_livetime / (365.2425 * 24 * 3600)
if i_var == "timestampdiff":
var_value[i_var][0] = (tmp_utcmax - tmp_utcmin)\
- tmp_livetime
if i_var == "timestampdiff_r":
var_value[i_var][0] = 1. - tmp_livetime / \
(tmp_utcmax - tmp_utcmin)
if i_var == "pmts_norm": # NOT YET NORMALIZED :-(
var_value[i_var][0] = tmp_pmts / \
get_detx_caract(det)['pmts']
elif i_var == "triggerrate":
var_value[i_var][0] = tmp_JDAQEvent / tmp_livetime
elif i_var == "acoustics":
var_value[i_var][0] = tmp_Acoustics
elif i_var == "ahrs_per_mn":
var_value[i_var][0] = tmp_AHRS / tmp_livetime * 60
elif i_var == "JDAQJTProc":
var_value[i_var][0] = \
(tmp_JDAQEvent - tmp_JTriggerReprocessor) / \
(tmp_JDAQEvent + 1.0e-10)
elif i_var == "kton_year":
var_value[i_var][0] = tmp_kton_year
else:
err = (f"Corrupted lines found in qaqc file - {lines[i_line]}"
" - Line skipped")
error_log = error_log + (f"{err}\n")
# Filling the QAQC variable values
for i_var in var_to_fill:
if i_var.startswith("def_") is False:
continue # QAQC variable are treated herebefore
var_value[i_var][0] = 0 # No defect
if int(run_number) in list(defect_results[i_var].keys()):
var_value[i_var][0] = defect_results[i_var][int(run_number)]
if append_veto_qsco:
(var_value['veto'][0], var_value['veto_source'][0],
var_value['qsco'][0], var_value['qsco_source'][0]) = \
compute_veto_qscore(var_properties, var_value)
run_stored.append(run_number)
new_tree.Fill()
new_tree.ResetBranchAddresses()
if error_log != "":
print(error_log)
return (new_tree, error_log)
###############################################################################
[docs]
def compute_veto_qscore(var_prop, var_val):
"""
Computes the veto/Q-score for a detector/tag using a QAQC source.
"""
veto = False
veto_source = 0b0
qsco = 1.
qsco_source = 0b0
for i_var in var_prop['veto_thresholds']:
# Defect variables
if i_var.startswith("def_"):
# first extract the set defects
decoded_defects = decode_defect_diag_byte(var_val[i_var][0], i_var)
# Then loop on the various defects used in veto
# Treatement of NOT. defect added on February, 2. Not fully tested.
for i_excluded_def in var_prop['veto_thresholds'][i_var]:
defect_test = False
if i_excluded_def in decoded_defects:
defect_test = True
if i_excluded_def.startswith("NOT."):
if i_excluded_def.replace("NOT.", "") not in decoded_defects:
defect_test = True
if defect_test:
veto = True
veto_source = veto_source | 0b1 << var_prop['bit'][i_var]
continue
# QAQC variables
if any((var_val[i_var][0] < var_prop['veto_thresholds'][i_var][0],
var_val[i_var][0] > var_prop['veto_thresholds'][i_var][1])):
veto = True
veto_source += 0b1 << var_prop['bit'][i_var]
for i_var in var_prop['qsco_thresholds']:
# Defect variables
if i_var.startswith("def_"):
# first extract the set defects
decoded_defects = decode_defect_diag_byte(var_val[i_var][0], i_var)
# Then loop on the various defects used in veto
for i_excluded_def in var_prop['qsco_thresholds'][i_var]:
if i_excluded_def in decoded_defects:
qsco -= 1./float(len(var_prop['qsco_thresholds']))
qsco_source = qsco_source | 0b1 << var_prop['bit'][i_var]
continue
if any((var_val[i_var][0] < var_prop['qsco_thresholds'][i_var][0],
var_val[i_var][0] > var_prop['qsco_thresholds'][i_var][1])):
qsco -= 1./float(len(var_prop['qsco_thresholds']))
qsco_source += 0b1 << var_prop['bit'][i_var]
return (int(veto), int(veto_source), qsco, int(qsco_source))
###############################################################################
[docs]
def log_run_range(det, run_0, run_1, log_file):
""" Write the run range and run list on disk """
os.environ['TZ'] = 'Europe/Paris'
time.tzset()
run_prop = get_run_properties_from_db(det)
if run_0 in run_prop.keys():
tim_0_unix = int(run_prop[run_0]['UNIXJOBSTART'])
tim_0 = time.strftime("%a, %d %b %Y %H:%M",
time.localtime(tim_0_unix/1000))
else:
tim_0 = "Unknown"
if run_1 in run_prop.keys():
tim_1_unix = int(run_prop[run_1]['UNIXJOBEND'])
tim_1 = time.strftime("%a, %d %b %Y %H:%M",
time.localtime(tim_1_unix/1000))
else:
tim_1 = "Unknown"
run_range = (f"{int(run_0)} ({tim_0} CET/CEST) - "
f"{int(run_1)} ({tim_1} CET/CEST)")
log_file.write(f"Run range: {run_range}\n")
###############################################################################
[docs]
def read_qaqc_file(det, source="qaqc_sftp", qaqc_proc_version=""):
"""
sftp QAQC file reading
"""
lines = []
if source == "qaqc_sftp":
with urllib.request.urlopen(
get_file_paths(det, qaqc_proc_version)["JQAQC_sftp"]) as qaqc:
tmp = ((qaqc.read()).split(b"\n"))
for i_line in tmp:
if i_line != "":
lines.append(i_line.decode("utf-8"))
else:
if source == "qaqc_sps":
source_file = get_file_paths(det, qaqc_proc_version)["JQAQC_sps"]
else:
source_file = source
with open(source_file, "r", encoding="utf-8") as s_f:
lines = s_f.readlines()
return lines
###############################################################################
[docs]
def decode_source_byte(value, v_prop):
"""
Decode the source of degradation stored in the TTree
NB: the source is either a QAQC variable either a defect type (("daq",
"operation"...)
"""
from ROOT import TTree
source_list = []
source_str = ""
for (i_var, i_bit) in v_prop['bit'].items():
if (int(value) >> i_bit) & 1:
source_list.append(i_var)
source_str += f"{i_var} "
return (source_list, source_str)