Coverage for src/km3dq_common/common_library.py: 51%

292 statements  

« prev     ^ index     » next       coverage.py v7.10.7, created at 2025-10-16 11:32 +0000

1#! /usr/bin/env python 

2######################################################################################################## 

3# Various functions used in km3dq_lw_dq and km3dq_core scripts 

4# 

5# Developer: Benjamin Trocme (benjamin.trocme at apc.in2p3.fr) - 2023 

6 

7import os 

8import sys 

9import time 

10import array 

11import re 

12import urllib.request 

13import km3db 

14import pandas as pd 

15import numpy as np 

16import tomli 

17 

18from .config_library import configure_var_bit 

19from .config_library import configure_var_thresholds 

20from .config_library import get_detx_caract 

21from .config_library import configure_dataquality_tag 

22from .config_library import configure_defect 

23from .lw_db_defect_library import read_defect_file 

24from .lw_db_defect_library import decode_defect_diag_byte 

25from .lw_db_defect_library import check_defect_diag_byte 

26 

27# Types of all variables of all versions except the oos variables of 16.0.3 that 

28# are renamed on the fly 

29# Variable evolution with versions: 

30# - 17.3.2: out-sync/in-sync renamed out_sync/in_sync 

31# out_usync/in_usync/event_duration added 

32# - 19.2.0: nb_of_meta/hrv_fifo_failures/duplic_timeslices/zero_AHRS/mean_AHRS added 

33# AHRS removed 

34 

35VARIABLE_TYPE = { 

36 "livetime_s": float, 

37 "DAQ": float, 

38 "WR": float, 

39 "FIFO": float, 

40 "HRV": float, 

41 "PMTs": float, 

42 "pmts_norm": float, 

43 "timestampdiff": float, 

44 "timestampdiff_r": float, 

45 "triggerrate": float, 

46 "Acoustics": float, 

47 "Acoustics_per_mn": float, 

48 "MEAN_Rate_Hz": float, 

49 "RMS_Rate_Hz": float, 

50 "AHRS": float, 

51 "ahrs_per_mn": float, 

52 "mean_AHRS": float, 

53 "JDAQJTProc": float, 

54 "kton_year": float, 

55 "UTCMin_s": float, 

56 "UTCMax_s": float, 

57 "detector": int, 

58 "run": int, 

59 "nb_of_meta": int, 

60 "trigger3DMuon": int, 

61 "triggerMXShower": int, 

62 "trigger3DShower": int, 

63 "triggerNB": int, 

64 "JTriggerReprocessor": int, 

65 "JTrigger3dmuon": int, 

66 "JTriggermxshower": int, 

67 "JTrigger3dshower": int, 

68 "JTriggernb": int, 

69 "writeL0": int, 

70 "writeL1": int, 

71 "writeL2": int, 

72 "writeSN": int, 

73 "JDAQTimeslice": int, 

74 "JDAQTimesliceL0": int, 

75 "JDAQTimesliceL1": int, 

76 "JDAQTimesliceL2": int, 

77 "JDAQTimesliceSH": int, 

78 "JDAQSummaryslice": int, 

79 "JDAQEvent": int, 

80 "hrv_fifo_failures": int, 

81 "duplic_timeslices": int, 

82 "in_sync": int, 

83 "zero_AHRS": int, 

84 "out_sync": int, 

85 "in_usync": int, 

86 "out_usync": int, 

87 "HV_check":int 

88 } 

89 

90# Dummy value used when the variable is missing in a run/QAQC file 

91MISSING_VALUE = -999999 

92 

93######################################################################################################## 

94def get_det_id(det): 

95 """ 

96 Returns the det id 

97 

98 === Input: 

99 - det [string]: D1ORCA024, D0ARCA030... 

100 === Outputs: 

101 - detector id [integer] 

102 """ 

103 

104 return km3db.tools.todetid(det) 

105 

106 

107######################################################################################################## 

108def get_current_detector(site): 

109 """ 

110 Extract the current detector names from the database by checking the highest run number 

111 By default, it is retrieved from the KM3NeT DB but in some rare cases (typically at a detector 

112 change) it can be forced to a different one by modifying the km3dq_lw_db/Common/current_detector.toml 

113 file that is available on sftp. 

114  

115 === Input: 

116 - site [string]: ARCA / ORCA 

117 === Outputs: 

118 - detector [string]: D1ORCA024, D0ARCA030... 

119 """ 

120 

121 # First check if the current detector is hardcoded in the current_detector.toml file 

122 # Facility especially useful at the detector change to force keeping a detector in Coral 

123 with urllib.request.urlopen( 

124 "https://sftp.km3net.de/data/km3dq_lw_db/Common/current_detector.toml" 

125 ) as s_f: 

126 toml = tomli.loads(s_f.read().decode("utf-8")) 

127 

128 if toml[site] != "": 

129 return toml[site] 

130 

131 # If not -> Retrieve it from the database 

132 sds = km3db.tools.StreamDS() 

133 

134 if site == "ARCA": 

135 res = sds.detectors(city="ARCA+Deployment+site+-+seabed").split("\n") 

136 else: 

137 res = sds.detectors(city="ORCA+Deployment+site+-+seabed").split("\n") 

138 

139 max_run = 0 

140 

141 for i_det in res[1:]: 

142 try: 

143 if int(i_det.split("\t")[5]) > max_run: 

144 o_det = i_det.split("\t")[0] 

145 if o_det.startswith("D") is False: 

146 continue 

147 max_run = int(i_det.split("\t")[5]) 

148 except: 

149 o_det = "" 

150 print("Improper db reply in get_current_detectors") 

151 

152 return o_det 

153 

154 

155######################################################################################################## 

156def get_file_paths(detector, qaqc_name): 

157 """ 

158 For a given detector / detector, returns the path for QAQC files both on sps and sftp 

159 

160 === Input: 

161 - det [string]: D1ORCA024, D0ARCA030... 

162 === Outputs: 

163 - detector id [integer] 

164 """ 

165 

166 return { 

167 "JQAQC_sps": ( 

168 "/sps/km3net/repo/data/raw/quality/" 

169 f"KM3NeT_{km3db.tools.todetid(detector):08d}_QAQC_Jpp_{qaqc_name}.txt" 

170 ), 

171 "JQAQC_sftp":( 

172 "https://sftp.km3net.de/data/quality/" 

173 f"KM3NeT_{km3db.tools.todetid(detector):08d}_QAQC_Jpp_{qaqc_name}.txt" 

174 ) 

175 } 

176 

177 

178######################################################################################################## 

179def get_run_properties_from_db(det, filt="PHYS", filt_target="Run"): 

180 """ 

181 Retrieve run properties from the database with the km3db package 

182 

183 === Inputs: 

184 - det [string]: D1ORCA024, D0ARCA030... 

185 - filt [string]: accepted run types (PHYS, COMM...) separated by a space - if == "", no filter 

186 - filt_target [string]: accepted target (run, on...) separated by a space - if == "", no filter 

187 === Outputs: 

188 - properties [dict of dict]: all DB content in a single dictionnary with the run numbers as key 

189 """ 

190 

191 runs = km3db.StreamDS(container="nt").get("runs", detid=det) 

192 

193 results = {} 

194 

195 for i_run in runs: 

196 if filt != "": 

197 if all((i_run.runsetupname[0:4] not in filt.split(" "), 

198 i_run.runsetupname[0:5] not in filt.split(" "))): 

199 continue 

200 

201 if filt_target != "": 

202 if i_run.jobtarget not in filt_target.split(" "): 

203 continue 

204 

205 results[i_run.run] ={ 

206 'UNIXSTARTTIME': i_run.unixstarttime, 

207 'STARTTIME_DEFINED': i_run.starttime_defined, 

208 'RUNSETUPID': i_run.runsetupid, 

209 'RUNSETUPNAME': i_run.runsetupname, 

210 'T0_CALIBSETID': i_run.t0_calibsetid, 

211 'POS_CALIBSETID': i_run.pos_calibsetid, 

212 'ROT_CALIBSETID': i_run.rot_calibsetid, 

213 'RUNJOBID': i_run.runjobid, 

214 'JOBTARGET': i_run.jobtarget, 

215 'JOBPRIORITY': i_run.jobpriority, 

216 'UNIXJOBSTART': i_run.unixjobstart, 

217 'UNIXJOBEND': i_run.unixjobend, 

218 } 

219 

220 return results 

221 

222 

223######################################################################################################## 

224def get_run_properties_from_qaqc( 

225 det, dq_tag, origin="qaqc_sftp", startrun=0, endrun=1e9 

226): 

227 """ 

228 For a given detector, returns the run numbers, their lengths, time counter... as retrieved from the 

229 QAQC file 

230  

231 === Inputs: 

232 - det [string]: D1ORCA024, D0ARCA030... 

233 - dq_tag [DQ tag]: as configured in config_library 

234 - origin [string]: "qaqc_sftp" or "qaqc_sps" 

235 - startrun [integer]: first run 

236 - endrun [integer]: last run  

237 === Outputs: 

238 - run properties: TOBEDETAILED 

239 - error: should be empty 

240 """ 

241 

242 runprop = { 

243 "runNumber": [], 

244 "lvt": [], 

245 "lvt_counter": [], 

246 "kty": [], 

247 "kty_counter": [], 

248 "nbRuns": 0, 

249 "minRun": 1e10, 

250 "maxRun": 0, 

251 } 

252 

253 kty_counter = 0 

254 err_log = "" 

255 

256 (n1, err_log) = create_ttree_from_qaqc( 

257 det, ["run", "livetime_s", "kton_year"], origin, dq_tag 

258 ) 

259 

260 if len(n1) == 0: 

261 return (runprop, "No data in qaqc") 

262 

263 if all((startrun != 0, endrun != 1e9)): 

264 n1 = n1[(n1["run"] >= startrun) & (n1["run"] <= endrun)] 

265 

266 runprop = { 

267 "runNumber": n1.run.to_list(), 

268 "lvt": (n1.livetime_s / 3600 / 24).to_list(), 

269 "kty": n1.kton_year.to_list(), 

270 "nbRuns": len(n1), 

271 "minRun": int(n1["run"].min()), 

272 "maxRun": int(n1["run"].max()), 

273 } 

274 

275 runprop["lvt_counter"] = [0] + (n1["livetime_s"].cumsum() / 3600 / 24).to_list() 

276 runprop["kty_counter"] = [0] + (n1["kton_year"].cumsum()).to_list() 

277 

278 return (runprop, err_log) 

279 

280 

281######################################################################################################## 

282def get_run_timerange(det, filt="PHYS"): 

283 """ 

284 Retrieve the run start/stop by combining db and QAQC quantities to get the most accurate timing. 

285 If QAQC data exist (normally for all PHYS runs), use them, otherwise use the DB ones 

286 So far used only in km3net_lw_db_interface 

287  

288 === Inputs: 

289 - det: string like (e.g: D0ARCA030, D1ORCA024...) 

290 - filt: string containing the accepted run type (PHYS, COMM...) separated by a space 

291 

292 === Outputs: 

293 - dictionary with run as key and a list of [run_start, run_stop, source, runtype] with 

294 source = 0 (QAQC) or 1 (DB only) 

295 """ 

296 

297 # First retrieve the QAQC variables that will be used to get the exact 

298 # run end 

299 df, _ = create_ttree_from_qaqc( 

300 det, ["run", "livetime_s"], "qaqc_sftp", configure_dataquality_tag("operation") 

301 ) 

302 results = dict([(r, [s, e, 0]) for r, s, e in zip(df["run"], df["UTCMin_s"], df["UTCMax_s"])]) 

303 

304 # Then retrieve all runs from the database to get the remaining runs 

305 runs = km3db.StreamDS(container="nt").get("runs", detid=det) 

306 

307 for i_run in runs: 

308 i_run_runtype = i_run.runsetupname.split(".")[0] 

309 if i_run_runtype not in filt: 

310 continue 

311 

312 if i_run.run in results: # Timerange already retrieved from the QAQC data - Append run type 

313 results[i_run.run].append(i_run_runtype) 

314 continue 

315 

316 try: 

317 results[i_run.run] = [i_run.unixjobstart/1000., i_run.unixjobend/1000., 1, i_run_runtype] 

318 except TypeError: 

319 pass 

320 

321 return results 

322 

323 

324######################################################################################################## 

325def get_last_n_weeks_runs_qaqc(det, nweeks=2): 

326 """Extract the all runs acquired in the last nweeks of running""" 

327 

328 # QAQC default tag used. As priori OK for this purpose 

329 dq_tag = configure_dataquality_tag("operation") 

330 min_run = 1e9 

331 max_run = 0 

332 

333 try: 

334 lines = read_qaqc_file(det, "qaqc_sftp", dq_tag["qaqc_name"][det], "") 

335 ten_days = (nweeks * 7 + 1) * 3600.0 * 24.0 

336 for i_run in range(len(lines["run"])): 

337 if (time.time() - float(lines["UTCMin_s"][i_run])) < ten_days: 

338 run_number = int(lines["run"][i_run]) 

339 if run_number < min_run: 

340 min_run = run_number 

341 if run_number > max_run: 

342 max_run = run_number 

343 except KeyError: 

344 print(f"common_library.py: Missing DQ-tag information for the detector {det}") 

345 

346 return (min_run, max_run) 

347 

348 

349######################################################################################################## 

350def get_site(det): 

351 """ 

352 Returns the site (ORCA or ARCA) 

353 

354 === Inputs: 

355 - det [string]: D1ORCA024, D0ARCA030... 

356 === Ouputs: 

357 - site [string]: ARCA/ORCA 

358 """ 

359 

360 # A/O detectors corresponds to the current ARCA/ORCA detectors 

361 if "ORC" in det or det == "O": 

362 out = "ORCA" 

363 else: 

364 out = "ARCA" 

365 return out 

366 

367 

368######################################################################################################## 

369def get_active_dus_range(det): 

370 """ 

371 Retrieve the range of active DUs 

372 To be improved with database information 

373 

374 === Inputs: 

375 - det [string]: D1ORCA024, D0ARCA030... 

376 === Ouputs: 

377 - site [string]: ARCA/ORCA 

378 """ 

379 lower_du = 0 

380 upper_du = 48 

381 

382 return (lower_du, upper_du) 

383 

384 

385######################################################################################################## 

386def get_nb_qaqc_variables(qaqc_vers): 

387 """ 

388 Retrieve the number of variables in the QAQC file 

389 

390 === Inputs 

391 - qaqc_vers [string]: "v16.0.3", "v17.3.2", "V19.2.0"... 

392 

393 === Output === 

394 - Number of QAQC variables [integer] 

395 

396 """ 

397 if qaqc_vers in ("v16.0.3", "16.0.3"): 

398 return 39 

399 if qaqc_vers in ("v17.3.2", "17.3.2", "18.0.0-rc.4-34-g8618072aa-D"): 

400 return 42 

401 if qaqc_vers in ("v19.2.0", "19.1.0", "19.2.0"): 

402 return 46 

403 if qaqc_vers in ("19.3.0-rc.2", "19.3.0-rc.5"): 

404 return 46 

405 if qaqc_vers == "19.3.0-rc.x": 

406 return 47 

407 if qaqc_vers == "19.3.0": 

408 return 47 

409 if qaqc_vers == "jpp-master-47": # This jpp version with 47 variables was hacked by hand in the QAQC file 

410 return 47 

411 if qaqc_vers == "20.0.0-rc.2": # This jpp version with 47 variables was hacked by hand in the QAQC file 

412 return 47 

413 

414 # print(f"Unknwon qaqc version {qaqc_vers} -> Assuming that it is jpp master -> 47 variables") 

415 return 48 

416 

417######################################################################################################## 

418def create_ttree_from_qaqc(det, var_to_fill, source, tag, append_veto_qsco=False): 

419 """ 

420 Create a ttree from qaqc sftp file and defect variables stored on git 

421 It includes some advanced check about the QAQC file integrity. 

422 

423 === Inputs 

424 - det : detector name - [string] - Ex: "D0ARCA021", "D0ORCA018"... 

425 - var_to_fill : QAQC variables or defect to fill - [array of string] - 

426 Ex: ['run', 'timestampdiff', 'def_operation', 'def_oos'] 

427 Obsolete - To be removed at some point 

428 - source : QAQC source, a priori "qaqc_sftp" - [string] 

429 - tag : data-quality tag (not its name) as created by 

430 configure_dataquality_tag 

431 - append_veto_qsco: append the veto and Qscore - [boolean] 

432 === Outputs 

433 - TTree 

434 - Error log 

435 

436 """ 

437 

438 error_log = "" 

439 

440 content = read_qaqc_file(det, source, tag["qaqc_name"][det], tag["run_range"][det]) 

441 

442 df = pd.DataFrame(content) 

443 # Renaming required for 16.0.3 

444 df = df.rename(columns={"out-sync": "out_sync"}) 

445 df = df.rename(columns={"in-sync": "in_sync"}) 

446 

447 # Type definition for each column 

448 # df = df.astype({key: float for key in variable_float if key in df.columns}) 

449 # df = df.astype({key: int for key in variable_int if key in df.columns}) 

450 for i_var, i_type in VARIABLE_TYPE.items(): 

451 try: 

452 df = df.astype({i_var: i_type}) 

453 except KeyError: # Variable missing in this version -> Dummy column added 

454 df[i_var] = len(df) * [MISSING_VALUE] 

455 

456 df["timestampdiff"] = df.UTCMax_s - df.UTCMin_s - df.livetime_s 

457 df["timestampdiff_r"] = 1.0 - df.livetime_s / (df.UTCMax_s - df.UTCMin_s) 

458 df["pmts_norm"] = df.PMTs / get_detx_caract(det)["pmts"] 

459 df["triggerrate"] = df.JDAQEvent / df.livetime_s 

460 

461 try: # Version > 19.2.0 

462 df["ahrs_per_mn"] = df.mean_AHRS / df.livetime_s * 60 

463 except AttributeError: # Version 16.0.3 / 17.3.2 

464 df["ahrs_per_mn"] = df.AHRS / df.livetime_s * 60 

465 

466 df["Acoustics_per_mn"] = df.Acoustics / df.livetime_s * 60 

467 

468 df["JDAQJTProc"] = (df.JDAQEvent - df.JTriggerReprocessor) / (df.JDAQEvent + 1e-10) 

469 if "ARC" in det: 

470 df["kton_year"] = ( 

471 1e6 

472 / (31 * 18 * 2 * 115) 

473 * df.PMTs 

474 * (1.0 - df.HRV) 

475 * df.livetime_s 

476 / (365.2425 * 24 * 3600) 

477 ) 

478 else: 

479 df["kton_year"] = ( 

480 6980 

481 / (31 * 18 * 115) 

482 * df.PMTs 

483 * (1.0 - df.HRV) 

484 * df.livetime_s 

485 / (365.2425 * 24 * 3600) 

486 ) 

487 

488 

489 # Adding defects 

490 defect_results = read_defect_file(det, tag["def_tag"]) 

491 

492 df = df.sort_values("run") 

493 # Check if there are some duplicated lines with the same run number 

494 # If some are found exit, as this is not acceptable and have to be cured 

495 if True in df.duplicated(subset=["run"], keep=False).values: 

496 print("Duplicated lines in create_ttree_from_qaqc\n" 

497 "Fix required -> Exiting") 

498 print(df.mask(~df.duplicated(subset=["run"], keep=False)).dropna()) 

499 sys.exit() 

500 

501 df.reset_index(drop=True) 

502 

503 for key, runs in defect_results.items(): 

504 value = np.zeros(len(df)) 

505 if len(runs) != 0: 

506 # A defect can be assigned to a run missing in the QAQC file (protection added) 

507 # The defects_results is not chronologically ordered (sorting added) 

508 mask = np.isin( 

509 df.run, [x for x in sorted(runs.keys()) if x in df["run"].values] 

510 ) 

511 value[mask] = [ 

512 runs[x] for x in sorted(runs.keys()) if x in df["run"].values 

513 ] 

514 df[key] = value 

515 

516 # Adding Veto and Q score: 

517 if append_veto_qsco: 

518 var_properties = {} 

519 configure_var_thresholds(var_properties, det, tag) 

520 configure_var_bit(var_properties) 

521 

522 df["veto"], df["veto_source"], df["qsco"], df["qsco_source"] = ( 

523 compute_veto_qscore(var_properties, df) 

524 ) 

525 

526 if error_log != "": 

527 print(error_log) 

528 return (df, error_log) 

529 

530 

531######################################################################################################## 

532def compute_veto_qscore(var_prop, var_val): 

533 """ 

534 Computes the veto/Q-score for a detector/tag using a QAQC source. 

535 """ 

536 

537 def_config = configure_defect() 

538 

539 veto = np.zeros(len(var_val), dtype=bool) 

540 veto_source = np.zeros(len(var_val), dtype=int) 

541 

542 qsco = np.ones(len(var_val), dtype=float) * len(var_prop["qsco_thresholds"]) 

543 qsco_source = np.zeros(len(var_val), dtype=int) 

544 

545 pd.set_option("display.max_rows", None) 

546 for i_var in var_prop["veto_thresholds"]: 

547 values = var_val[i_var] 

548 # Defect variables 

549 # The defect variable is defined as def_[defect type] 

550 # with [defect type] = daq, operation, analysis... 

551 if i_var.startswith("def_"): 

552 # first extract the diagnosis 

553 defect_type = i_var.replace("def_", "") 

554 diag_array = {"present": [], "absent": []} 

555 for i_excluded_diag in var_prop["veto_thresholds"][i_var]: 

556 if i_excluded_diag.startswith("NOT."): 

557 diag_array["absent"].append( 

558 def_config["bit"][defect_type][ 

559 i_excluded_diag.replace("NOT.", "") 

560 ] 

561 ) 

562 else: 

563 diag_array["present"].append( 

564 def_config["bit"][defect_type][i_excluded_diag] 

565 ) 

566 

567 if len(diag_array["absent"]) == 0: # Only present diagnosis 

568 def_mask = values != 0 

569 elif len(diag_array["present"]) == 0: # Only NOT. diagnosis 

570 def_mask = values == 0 

571 else: # Mixture of present/absent -> No mask 

572 def_mask = values != (-1) 

573 if np.sum(def_mask) == 0: 

574 continue 

575 

576 defect_test_output = np.vectorize(check_defect_diag_byte, excluded=[1])( 

577 values[def_mask], diag_array 

578 ) 

579 # The FutureWarning about the incompatible dtype comes from the line below 

580 # When checking explicitly the dtype, both are returned as bool hence 

581 # the FutureWarning can not be understood. 

582 # The trial to fix with explicit casting and other numpy functions 

583 # (e.g veto[def_mask] = np.logical_or(veto[def_mask].astype(bool), defect_test_output.astype(bool))) 

584 # did not help at all. 

585 veto[def_mask] = veto[def_mask] | defect_test_output 

586 veto_source[def_mask] += (0b1 & defect_test_output) << var_prop["bit"][ 

587 i_var 

588 ] 

589 

590 else: # QAQC variable: range treatement 

591 range_mask = (var_val[i_var] != MISSING_VALUE) & ( 

592 (var_val[i_var] < var_prop["veto_thresholds"][i_var][0]) | ( 

593 var_val[i_var] > var_prop["veto_thresholds"][i_var][1] 

594 ) 

595 ) 

596 veto = veto | range_mask 

597 veto_source[range_mask] += 0b1 << var_prop["bit"][i_var] 

598 

599 for i_var in var_prop["qsco_thresholds"]: 

600 values = var_val[i_var] 

601 # Defect variables 

602 # Defect variables 

603 # The defect variable is defined as def_[defect type] 

604 # with [defect type] = daq, operation, analysis... 

605 if i_var.startswith("def_"): 

606 # first extract the set defects 

607 def_mask = values != 0 

608 if np.sum(def_mask) == 0: 

609 continue 

610 

611 defect_type = i_var.replace("def_", "") 

612 diag_array = {"present": [], "absent": []} 

613 for i_excluded_diag in var_prop["qsco_thresholds"][i_var]: 

614 if i_excluded_diag.startswith("NOT."): 

615 diag_array["absent"].append( 

616 def_config["bit"][defect_type][ 

617 i_excluded_diag.replace("NOT.", "") 

618 ] 

619 ) 

620 else: 

621 diag_array["present"].append( 

622 def_config["bit"][defect_type][i_excluded_diag] 

623 ) 

624 

625 defect_test_output = np.vectorize(check_defect_diag_byte, excluded=[1])( 

626 values[def_mask], diag_array 

627 ) 

628 qsco[def_mask] = qsco[def_mask] | defect_test_output 

629 qsco_source[def_mask] += (0b1 & defect_test_output) << var_prop["bit"][ 

630 i_var 

631 ] 

632 

633 else: # QAQC variable: range treatement 

634 range_mask = (var_val[i_var] != MISSING_VALUE) & ( 

635 (var_val[i_var] < var_prop["qsco_thresholds"][i_var][0]) | ( 

636 var_val[i_var] > var_prop["qsco_thresholds"][i_var][1] 

637 ) 

638 ) 

639 

640 qsco[range_mask] -= 1 

641 qsco_source[range_mask] += 0b1 << var_prop["bit"][i_var] 

642 

643 qsco /= len(var_prop["qsco_thresholds"]) 

644 

645 return (veto, veto_source, qsco, qsco_source) 

646 

647 

648######################################################################################################## 

649def log_run_range(det, run_0, run_1, log_file): 

650 """Write the run range and run list on disk""" 

651 

652 os.environ["TZ"] = "Europe/Paris" 

653 time.tzset() 

654 

655 run_prop = get_run_properties_from_db(det, "PHYS COMM") 

656 

657 try: 

658 tim_0_unix = int(run_prop[run_0]["UNIXJOBSTART"]) 

659 tim_0 = time.strftime("%a, %d %b %Y %H:%M", time.localtime(tim_0_unix / 1000)) 

660 except KeyError: 

661 tim_0 = "Unknown" 

662 

663 try: 

664 tim_1_unix = int(run_prop[run_1]["UNIXJOBEND"]) 

665 tim_1 = time.strftime("%a, %d %b %Y %H:%M", time.localtime(tim_1_unix / 1000)) 

666 except KeyError: 

667 tim_1 = "Unknown" 

668 

669 run_range = f"{int(run_0)} ({tim_0} CET/CEST) - " f"{int(run_1)} ({tim_1} CET/CEST)" 

670 

671 log_file.write(f"Run range: {run_range}\n") 

672 

673 

674######################################################################################################## 

675def read_qaqc_file(det, source, qaqc_name, run_range): 

676 """ 

677 sftp QAQC file reading 

678  

679 === Arguments === 

680 - det : detector name - [string] - Ex: "D0ARCA021", "D0ORCA018"... 

681 - source : QAQC source, a priori "qaqc_sftp" - [string] 

682 - qaqc_name : 

683 - run_range : string as defined in DQ tag "[min_run]-[max_run]" (inclusive) 

684 

685 === Output === 

686 - TTree 

687 - Error log 

688 

689 """ 

690 

691 lines = [] 

692 if source == "qaqc_sftp": 

693 with urllib.request.urlopen( 

694 get_file_paths(det, qaqc_name)["JQAQC_sftp"] 

695 ) as qaqc: 

696 tmp = (qaqc.read()).split(b"\n") 

697 for i_line in tmp: 

698 if i_line != "": 

699 lines.append(i_line.decode("utf-8")) 

700 else: 

701 if source == "qaqc_sps": 

702 source_file = get_file_paths(det, qaqc_name)["JQAQC_sps"] 

703 else: 

704 source_file = source 

705 

706 with open(source_file, "r", encoding="utf-8") as s_f: 

707 lines = s_f.readlines() 

708 

709 labels = lines[0].split() 

710 run_index = labels.index("run") 

711 if run_range != "": 

712 min_run = int(run_range.split("-")[0]) 

713 max_run = int(run_range.split("-")[1]) 

714 else: 

715 min_run = 0 

716 max_run = 1e9 

717 

718 content = {} 

719 for l in lines[1:]: 

720 try: 

721 if len(l.split()) != get_nb_qaqc_variables(l.split()[0]): 

722 print(f"Bad number of variables in {l}") 

723 continue 

724 # sys.exit() 

725 except IndexError: # Empty line 

726 pass 

727 

728 try: 

729 if all((int(l.split()[run_index]) >= min_run, 

730 int(l.split()[run_index]) <= max_run 

731 )): 

732 for i, key in enumerate(labels): 

733 try: 

734 try: 

735 content[key].append(l.split()[i]) 

736 except KeyError: 

737 content[key] = [l.split()[i]] 

738 except IndexError: # Missing variable in QAQC file 

739 try: 

740 content[key].append(MISSING_VALUE) 

741 except KeyError: 

742 content[key] = [MISSING_VALUE] 

743 except IndexError: # Empty line 

744 pass 

745 

746 return content 

747 

748 

749######################################################################################################## 

750def decode_source_byte(value, v_prop): 

751 """ 

752 Decode the source of degradation stored in the TTree 

753 NB: the source is either a QAQC variable either a defect type (("daq", 

754 "operation"...) 

755 """ 

756 

757 source_list = [] 

758 source_str = "" 

759 

760 for i_var, i_bit in v_prop["bit"].items(): 

761 if (int(value) >> i_bit) & 1: 

762 source_list.append(i_var) 

763 source_str += f"{i_var} " 

764 

765 return (source_list, source_str)