Coverage for src/km3dq_common/aux_library.py: 0%
86 statements
« prev ^ index » next coverage.py v7.6.1, created at 2024-09-25 11:58 +0000
1#! /usr/bin/env python
2###############################################################################
3# Developer: Benjamin Trocme (benjamin.trocme at apc.in2p3.fr) - 2024
5import os
6import subprocess
7import glob
8import re
10from km3dq_common.common_library import get_det_id
13###############################################################################
14def complete_with_zero(integer, total_length):
16 output = ""
17 for _ in range(total_length - len(str(integer))):
18 output += "0"
19 output += f"{integer}"
21 return output
24###############################################################################
25def get_raw_data_full_path(dataset, run_number):
26 """
27 Return the full iRODS path
28 Functional only at CC-IN2P3
29 """
31 raw_data_file = {"std": "",
32 "root": "",
33 "details": ""}
35 d_id = get_det_id(dataset)
36 regular_dir = (f"/in2p3/km3net/data/raw/sea/"
37 f"KM3NeT_{complete_with_zero(d_id, 8)}")
38 regular_name = (f"KM3NeT_{complete_with_zero(d_id, 8)}"
39 f"_{complete_with_zero(run_number, 8)}.root")
41 cmd = f"ils {regular_dir}"
43 out = os.popen(cmd)
44 for i_out in out:
45 sub_dir = i_out.split("C- ")
46 if len(sub_dir) > 1:
47 sub_dir_clean = sub_dir[1].replace("\n", "")
48 cmd = f"ils -l {sub_dir_clean}"
49 out2 = os.popen(cmd)
50 for i_out2 in out2:
51 if regular_name in i_out2:
52 d = sub_dir_clean.replace("/in2p3/", "/")
53 raw_data_file['std'] = (f"/in2p3{d}/{regular_name}")
54 raw_data_file['root'] = (f"root://ccxroot:1999//hpss/"
55 f"in2p3.fr/group{d}/"
56 f"{regular_name}")
57 irods_details = i_out2.split(f"& {regular_name}")[0]
58 raw_data_file['details'] += f"{irods_details} / "
60 return raw_data_file
63###############################################################################
64def get_dst_full_path(dataset, run_number, data_type, version, where):
65 """
66 Return the full SPS/irods path
67 Functional only at CC-IN2P3
68 - dataset: detector name
69 - run_number: if 0, returns only the directory
70 - data_type: "jppmuon_aashower_dynamic" or "jppmuon_jppshower-upgoing_dynamic"
71 - version: array of version
72 Example: {"data":"v9.0","mc_noise":"v9.0", "mc_mupage":"v9.0", "mc_neutr":"v9.0"}
73 - where: string containing "sps" and/or "irods"
74 """
76 dst_file = {}
78 d_id_str = complete_with_zero(get_det_id(dataset), 8)
79 run_number_str = complete_with_zero(run_number, 8)
81 sub_directory_list = ["", "_priority", "_secondary"]
83 filename = {"data": (f"KM3NeT_{d_id_str}_{run_number_str}.data."
84 f"{data_type}.offline.dst.{version['data']}.root"),
85 "mc_noise": ((f"KM3NeT_{d_id_str}_{run_number_str}.mc."
86 "pure_noise*.root")),
87 "mc_mupage": ((f"KM3NeT_{d_id_str}_{run_number_str}.mc."
88 "mupage*.root")),
89 "mc_neutr": ((f"KM3NeT_{d_id_str}_{run_number_str}.mc."
90 "gsg*.root"))}
92 # Check the file availability on hpss
93 if "sps" in where:
94 for i_file in filename.keys():
95 if run_number == 0:
96 dst_file[f"dir_{i_file}"] = ("/sps/km3net/repo/data_processing/tag/"
97 f"{version[i_file]}/workdirs/"
98 f"KM3NeT_{d_id_str}{sub_directory_list}/"
99 f"results/{d_id_str}/dst")
100 else:
101 dst_file[i_file] = ""
102 for i_dir_data_type in sub_directory_list:
103 full_path = ("/sps/km3net/repo/data_processing/tag/"
104 f"{version[i_file]}/workdirs/"
105 f"KM3NeT_{d_id_str}{i_dir_data_type}/"
106 f"results/{d_id_str}/dst/{filename[i_file]}")
107 # Check that the file exists
108 if len(glob.glob(full_path)) == 1:
109 dst_file[i_file] = glob.glob(full_path)[0]
111 # Check the file availability on irods
112 # Only data implemented so far
113 if "irods" in where:
114 for i_file in ["data"]:
115 if run_number == 0:
116 dst_file[f"dir_{i_file}"] = ("/in2p3/km3net/data"
117 f"KM3NeT_{d_id_str}{sub_directory_list}/"
118 f"{version[i_file]}/dst")
119 else:
120 dst_file[f'{i_file}_irods'] = ""
121 for i_dir_data_type in sub_directory_list:
122 full_path = ("/in2p3/km3net/data/"
123 f"KM3NeT_{d_id_str}{i_dir_data_type}/"
124 f"{version[i_file]}/dst/{filename[i_file]}")
125 # Check that the file exists
126 sub = subprocess.run(["ils", full_path],
127 capture_output=True)
128 ils_out = sub.stdout.decode("utf-8").replace(" ", "")\
129 .replace("\n", "")
130 if filename[i_file] in ils_out:
131 dst_file[f'{i_file}_irods'] = full_path
133 return dst_file
136###############################################################################
137# def get_dst_data_full_path(dataset, run_number, suffix, version, where):
138# """
139# Return the full SPS/irods path
140# Functional only at CC-IN2P3
141# """
142#
143# dst_data_file = {"sps": "missing",
144# "irods": "missing",
145# "sps_mc": ""}
146#
147# d_id = get_det_id(dataset)
148# if d_id < 100:
149# d_id_str = f"000000{d_id:2d}"
150# else:
151# d_id_str = f"00000{d_id:3d}"
152#
153# if run_number < 10000:
154# run_number_str = f"0000{run_number}"
155# elif run_number < 100000:
156# run_number_str = f"000{run_number}"
157# elif run_number < 1000000:
158# run_number_str = f"00{run_number}"
159#
160# filename = (f"KM3NeT_{d_id_str}_{run_number_str}.data."
161# f"{suffix}.offline.dst.{version}.root")
162# filename_mc_wildc = (f"KM3NeT_{d_id_str}_{run_number_str}.mc."
163# "*.root")
164#
165# if "sps" in where:
166# for i_dir_suffix in ("", "_priority", "_secondary", "_priority_viper"):
167# sps_path = ("/sps/km3net/repo/data_processing/tag/"
168# f"{version}/workdirs/KM3NeT_{d_id_str}{i_dir_suffix}/"
169# f"results/{d_id_str}/dst/{filename}")
170#
171# if os.path.exists(sps_path):
172# dst_data_file['sps'] = sps_path
173# else:
174# sps_path_mc = ("/sps/km3net/repo/data_processing/tag/"
175# f"{version}/workdirs/"
176# f"KM3NeT_{d_id_str}{i_dir_suffix}/"
177# f"results/{d_id_str}/dst/{filename_mc_wildc}")
178# if len(glob.glob(sps_path_mc)):
179# dst_data_file['sps_mc'] = "some exists"
180#
181# if "irods" in where:
182# irods_path = (f"/in2p3/km3net/data/KM3NeT_{d_id_str}/{version}/"
183# f"dst/{filename}")
184#
185# sub = subprocess.run(["ils", irods_path],
186# capture_output=True)
187# ils_out = sub.stdout.decode("utf-8").replace(" ", "").replace("\n", "")
188# if filename in ils_out:
189# dst_data_file['irods'] = irods_path
190#
191# return dst_data_file
194###############################################################################
195def check_raw_data_availability(dataset, run_number_list):
196 """
197 Check that the raw data are available on iRODS
198 Function optimised for long list (typically JQAQC one)
199 """
201 d_id = get_det_id(dataset)
202 regular_dir = (f"/in2p3/km3net/data/raw/sea/"
203 f"KM3NeT_{complete_with_zero(d_id, 8)}")
205 cmd = f"ils {regular_dir}"
206 out = os.popen(cmd)
208 nb_of_files = {}
210 for i_out in out:
211 sub_dir = i_out.split("C- ")
212 if len(sub_dir) > 1:
213 sub_dir_clean = sub_dir[1].replace("\n", "")
214 cmd = f"ils {sub_dir_clean}"
215 out2 = os.popen(cmd)
216 for i_out2 in out2:
217 re_file = re.compile(rf"\s*KM3NeT_0+{d_id}_0+(\d+).root\s*")
218 re_file_match = re_file.match(i_out2)
220 if re_file_match:
221 run_number = re_file_match.group(1)
223 if run_number in nb_of_files.keys():
224 nb_of_files[run_number] += 1
225 else:
226 nb_of_files[run_number] = 1
228 results = {}
229 for v_run in run_number_list:
230 if f"{v_run}" not in nb_of_files.keys():
231 results[v_run] = 0
232 elif nb_of_files[f"{v_run}"] > 1:
233 results[v_run] = nb_of_files[f"{v_run}"]
235 return results