Coverage for src/km3dq_common/lw_db_signoff_library.py: 0%

1#! /usr/bin/env python 

2############################################################################### 

3import re 

4import urllib.request 

5 

6from .common_library import create_ttree_from_qaqc 

7 

8from .config_library import configure_dataquality_tag 

9from .config_library import configure_defect 

10 

11from .lw_db_defect_library import decode_defect_diag_byte 

12 

13 

14############################################################################### 

15def get_unsignedoff_runs(det, level = "detailed", dq_tag_name="operation"): 

16 """ 

17 Return the unsignedoff PHYS runs based on the defect files stored on SFTP 

18 ("signoff" defect type) 

19 

20 === Arguments === 

21 - det : detector name - [string] - Ex: "D0ARCA021", "D0ORCA018"... 

22 - dq_tag_name : data quality tag name - [string] - Ex: "default", "neutrino2024" 

23 

24 === Output === 

25 - Dictionnary with def_[defect] as keys and a value for each affected run. 

26 This value will be decoded by the decode_defect_diag_byte function. 

27 Ex: {'def_daq': {}, 'def_operation': {16501: 1, 16503: 2}...} 

28 

29 """ 

30 # The DQ tag is needed as the signoff depends on the QAQC file that 

31 # depends itself on the DQ tag 

32 dq_tag = configure_dataquality_tag(dq_tag_name) 

33 defects0 = configure_defect() 

34 

35 if "ORCA" in det: 

36 site = "ORCA" 

37 else: 

38 site = "ARCA" 

39 

40 (n1, _) = create_ttree_from_qaqc(det, ["run"], "qaqc_sftp", dq_tag) 

41 

42 re_line = re.compile(r"\s*([0-9]+)\s*\|.*") 

43 

44 res = {} 

45 # Initialize the output for all runs as not yet signed off 

46 for _, irun in n1.iterrows(): # One entry per run 

47 res[irun["run"]] = [] 

48 for i_diag in defects0["bit"]["signoff"].keys(): 

49 res[irun["run"]].append(i_diag) 

50 

51 # Now scan the defect files and removed the run signed off 

52 for i_diag in defects0["bit"]["signoff"].keys(): 

53 lines = [] 

54 defect_url = ( 

55 "https://sftp.km3net.de/data/km3dq_lw_db/" 

56 f"{site}/{det}/Defects/{dq_tag['def_tag']}/" 

57 f"signoff_{i_diag}.txt" 

58 ) 

59 try: 

60 with urllib.request.urlopen(defect_url) as def_file: 

61 tmp = (def_file.read()).split(b"\n") 

62 for i_line in tmp: 

63 if i_line != "": 

64 lines.append(i_line.decode("utf-8")) 

65 except urllib.error.URLError: 

66 # No defect file found 

67 continue 

68 

69 for i_line in lines: 

70 r_m = re_line.match(i_line) 

71 if r_m is not None: 

72 run_nb = int(r_m.group(1)) 

73 # Run signed off not (yet?) in QAQC 

74 if run_nb not in res.keys(): 

75 continue 

76 signoff_nature_bit = 1 << defects0["bit"]["signoff"][i_diag] 

77 signoff_nature = decode_defect_diag_byte(signoff_nature_bit, "signoff") 

78 for i_nature in signoff_nature.split(" / "): 

79 if i_nature != "": 

80 res[run_nb].remove(i_nature) 

81 

82 if len(res[run_nb]) == 0: 

83 res.pop(run_nb) 

84 

85 if level == "detailed": # Return all unsigned run 

86 return res 

87 

88 if level == "compact": # Returns intervals 

89 unsign = list(res.keys()) 

90 unsign.sort() 

91 res_str = "" 

92 

93 for i_r in unsign: 

94 try: 

95 if i_r == interv_upp + 1: 

96 interv_upp = i_r 

97 else: 

98 res_str += F"{interv_low} - {interv_upp} / " 

99 interv_low = i_r 

100 interv_upp = i_r 

101 except NameError: 

102 interv_low = i_r 

103 interv_upp = i_r 

104 

105 try: 

106 res_str += F"{interv_low} - {interv_upp} / " 

107 except NameError: 

108 pass 

109 

110 return res_str