PandA-2024.02
test_panda.py
Go to the documentation of this file.
1 #!/usr/bin/python
2 
3 import argparse
4 import datetime
5 import distutils.spawn
6 import logging
7 import os
8 import re
9 import shlex
10 import shutil
11 import signal
12 import subprocess
13 import sys
14 import threading
15 import xml.dom.minidom
16 from collections import deque
17 
18 line_index = 0
19 failure = False
20 
21 def positive_integer(value):
22  pos_int = int(value)
23  if pos_int <= 0:
24  raise argparse.ArgumentTypeError("%s must be a positive integer" % value)
25  return pos_int
26 
27 class StoreOrUpdateMin(argparse.Action):
28  first_parsed = True
29  def __call__(self, parser, namespace, values, option_string=None):
30  if self.first_parsed == True :
31  self.first_parsed = False
32  setattr(namespace, self.dest, values)
33  else :
34  setattr(namespace, self.dest, min(namespace.j, values))
35 
36 
37 #Return children of a process
38 def GetChildren(parent_pid):
39  ret = set()
40  ps_command = subprocess.Popen("ps -o pid --ppid %d --noheaders" % parent_pid, shell=True, stdout=subprocess.PIPE)
41  ps_output = ps_command.stdout.read()
42  ps_command.wait()
43  for pid_str in ps_output.split("\n")[:-1]:
44  ret.add(int(pid_str))
45  return ret
46 
47 #Kill a process than kill its children
48 def kill_proc_tree(pid):
49  children = GetChildren(pid)
50  os.kill(pid, signal.SIGKILL)
51  for child in children:
52  kill_proc_tree(child)
53 #Process benchmark in list
54 def execute_tests(named_list,thread_index):
55  global passed_benchmark
56  global total_benchmark
57  global line_index
58  global children
59  global failure
60  lines = open(named_list).readlines()
61  with lock:
62  local_index = line_index
63  line_index += 1
64  while local_index < len(lines) and not (failure and args.stop):
65  cwd = ComputeDirectory(lines[local_index])
66  failed_output_file_name = os.path.join(cwd, args.tool + "_failed_output")
67  if os.path.exists(failed_output_file_name):
68  os.remove(failed_output_file_name)
69  tool_return_value_file_name = os.path.join(cwd, args.tool + "_return_value")
70  if args.restart and os.path.exists(os.path.join(cwd, args.tool + "_return_value")):
71  tool_return_value_file = open(tool_return_value_file_name, "r")
72  return_value = tool_return_value_file.read()
73  tool_return_value_file.close()
74  if return_value == "0":
75  with lock:
76  total_benchmark += 1
77  passed_benchmark += 1
78  logging.info(" SKIPPING --- OVERALL: " + str(passed_benchmark) + " passed, " + str(total_benchmark-passed_benchmark) + " failed, " + str(len(lines)-total_benchmark) + " queued --- " + lines[local_index].replace("\\", ""))
79  local_index = line_index
80  line_index += 1
81  continue
82  HLS_output_directory = os.path.join(cwd, "HLS_output")
83  if os.path.exists(HLS_output_directory):
84  shutil.rmtree(HLS_output_directory)
85  output_file_name = os.path.join(cwd, args.tool + "_execution_output")
86  output_file = open(output_file_name, "w")
87  local_args = lines[local_index]
88  if local_args[0] == "\"":
89  local_args = local_args[1:-1]
90  if args.tool != "bambu" and args.tool != "zebu":
91  tokens = shlex.split(lines[local_index])
92  args_without_benchmark_name = ""
93  for token in tokens:
94  if token.find("--benchmark-name") == -1:
95  args_without_benchmark_name += token + " "
96  local_args = args_without_benchmark_name
97  local_command = "ulimit " + args.ulimit + "; exec timeout " + args.timeout + " " + tool_exe
98  local_command = local_command + " " + local_args
99  output_file.write("#" * 80 + "\n")
100  output_file.write("cd " + cwd + "; ")
101  output_file.write(local_command + "\n")
102  output_file.write("#" * 80 + "\n")
103  output_file.flush()
104  return_value = -1
105  with lock_creation_destruction:
106  if not (failure and args.stop):
107  children[thread_index] = subprocess.Popen(local_command, stderr=output_file, stdout=output_file, cwd=cwd, shell=True, executable="/bin/bash")
108  try:
109  return_value = children[thread_index].wait()
110  except:
111  pass
112  with lock_creation_destruction:
113  if return_value != 0 and (args.stop or args.returnfail):
114  failure = True
115  if failure and args.stop:
116  for local_thread_index in range(n_jobs):
117  if children[local_thread_index] != None:
118  if children[local_thread_index].poll() == None:
119  try:
120  kill_proc_tree(children[local_thread_index].pid)
121  except OSError:
122  pass
123  os.fsync(output_file.fileno())
124  output_file.close()
125  tool_return_value_file = open(tool_return_value_file_name, "w")
126  tool_return_value_file.write(str(return_value))
127  tool_return_value_file.close()
128  args_file = open(os.path.join(cwd, "args"), "w")
129  args_file.write(lines[local_index])
130  args_file.close()
131  if return_value == 0 and os.path.exists(os.path.join(cwd, args.tool + "_results_0.xml")):
132  tool_results_file_name = os.path.join(cwd, args.tool + "_results")
133  tool_results_file = open(tool_results_file_name, "w")
134  tool_results_string = ""
135  xml_document = xml.dom.minidom.parse(os.path.join(cwd, args.tool + "_results_0.xml"))
136  if len(xml_document.getElementsByTagName("CYCLES")) > 0:
137  cycles_tag = xml_document.getElementsByTagName("CYCLES")[0]
138  tool_results_string = tool_results_string + cycles_tag.attributes["value"].value + " CYCLES"
139  if len(xml_document.getElementsByTagName("CLOCK_SLACK")) > 0:
140  slack_tag = xml_document.getElementsByTagName("CLOCK_SLACK")[0]
141  tool_results_string = tool_results_string + " *** " + slack_tag.attributes["value"].value + "ns"
142  tool_results_file.write(tool_results_string)
143  tool_results_file.close()
144  if not (failure and args.stop) or (return_value != -9 and return_value != 0):
145  if return_value != 0:
146  shutil.copy(output_file_name, str(os.path.join(os.path.dirname(output_file_name), args.tool + "_failed_output")))
147  with lock:
148  total_benchmark += 1
149  if return_value == 0:
150  passed_benchmark += 1
151  if not args.no_clean:
152  for sub in os.listdir(cwd):
153  if os.path.isdir(os.path.join(cwd, sub)):
154  shutil.rmtree(os.path.join(cwd, sub))
155  else:
156  if sub != args.tool + "_return_value" and sub != args.tool + "_execution_output" and sub != args.tool + "_results_0.xml" and sub != "args":
157  os.remove(os.path.join(cwd, sub))
158  if os.path.exists(os.path.join(cwd, args.tool + "_results_0.xml")):
159  logging.info(" SUCCESS (" + tool_results_string + ") --- OVERALL: " + str(passed_benchmark) + " passed, " + str(total_benchmark-passed_benchmark) + " failed, " + str(len(lines)-total_benchmark) + " queued --- " + lines[local_index].replace("\\", ""))
160  else:
161  logging.info(" SUCCESS --- OVERALL: " + str(passed_benchmark) + " passed, " + str(total_benchmark-passed_benchmark) + " failed, " + str(len(lines)-total_benchmark) + " queued --- " + lines[local_index].replace("\\", ""))
162  elif return_value == 124:
163  logging.info(" FAILURE (Timeout) --- OVERALL: " + str(passed_benchmark) + " passed, " + str(total_benchmark-passed_benchmark) + " failed, " + str(len(lines)-total_benchmark) + " queued --- " + lines[local_index].replace("\\", ""))
164  elif return_value == 153:
165  logging.info(" FAILURE (File size limit exceeded) --- OVERALL: " + str(passed_benchmark) + " passed, " + str(total_benchmark-passed_benchmark) + " failed, " + str(len(lines)-total_benchmark) + " queued --- " + lines[local_index].replace("\\", ""))
166  else:
167  logging.info(" FAILURE --- OVERALL: " + str(passed_benchmark) + " passed, " + str(total_benchmark-passed_benchmark) + " failed, " + str(len(lines)-total_benchmark) + " queued --- " + lines[local_index].replace("\\", ""))
168  with lock:
169  local_index = line_index
170  line_index += 1
171 
172 #Computing relative path
173 def ComputeDirectory(line):
174  configuration_name = ""
175  benchmark_name = ""
176  tokens = shlex.split(line)
177  for token in tokens:
178  if token.find("--configuration-name") != -1:
179  configuration_name = token[len("--configuration-name="):]
180  if token.find("--benchmark-name") != -1:
181  benchmark_name = token[len("--benchmark-name="):]
182  new_dir = os.path.join(abs_path, configuration_name, benchmark_name)
183  return new_dir
184 
185 #Search c files
186 def SearchCFiles(directory):
187  logging.info(" Looking for file in " + str(directory))
188  files = set()
189  for element in os.listdir(directory):
190  if os.path.isdir(os.path.join(directory, element)):
191  files = files.union(SearchCFiles(os.path.join(directory, element)))
192  elif (element[-2:] == ".c") or (element[-2:] == ".C") or (element[-4:] == ".CPP") or (element[-4:] == ".cpp") or (element[-4:] == ".cxx") or (element[-3:] == ".cc") or (element[-4:] == ".c++"):
193  files.add(os.path.join(directory, element))
194  return files
195 
196 #Collecting results
197 def CollectResults(directory):
198  #Skip if this is a leaf directory
199  if os.path.exists(os.path.join(directory, args.tool + "_return_value")) or os.listdir(directory) == []:
200  return
201  subdirs = [s for s in sorted(os.listdir(directory)) if os.path.isdir(os.path.join(directory,s)) and s != "panda-temp" and s != "HLS_output"]
202  for subdir in subdirs:
203  CollectResults(os.path.join(directory, subdir))
204  tool_failed_output = open(os.path.join(directory, args.tool + "_failed_output"), "w")
205  for subdir in subdirs:
206  if os.path.exists(os.path.join(directory, subdir, args.tool + "_failed_output")):
207  tool_failed_output.write(open(os.path.join(directory, subdir, args.tool + "_failed_output")).read())
208  if os.path.exists(os.path.join(directory, subdir, args.tool + "_execution_output")):
209  tool_failed_output.write("\n")
210  tool_failed_output.write("\n")
211  tool_failed_output.write("\n")
212  tool_failed_output.close()
213  report_file = open(os.path.join(directory, "report"), "w")
214  for subdir in subdirs:
215  if os.path.exists(os.path.join(directory, subdir, args.tool + "_return_value")):
216  return_value_file_name = os.path.join(directory, subdir, args.tool + "_return_value")
217  return_value_file = open(return_value_file_name)
218  return_value = return_value_file.read()
219  return_value_file.close()
220  args_file = open(os.path.join(directory, subdir, "args"))
221  command_args = args_file.readlines()[0]
222  command_args = command_args.replace(abs_benchmarks_root + "/", "")
223  args_file.close()
224  if return_value == "0":
225  tool_results_file_name = os.path.join(directory, subdir, args.tool + "_results")
226  if os.path.exists(tool_results_file_name):
227  report_file.write("SUCCESS (" + open(tool_results_file_name).read() + " cycles) " + command_args.replace("\\", ""))
228  else:
229  report_file.write("SUCCESS: " + command_args.replace("\\", ""))
230  else:
231  if return_value == "124":
232  report_file.write("FAILURE(Timeout): " + command_args.replace("\\", ""))
233  else:
234  report_file.write("FAILURE: " + command_args.replace("\\", ""))
235  report_file.write("\n")
236  elif os.path.exists(os.path.join(directory, subdir, "report")):
237  local_report_file = open(os.path.join(directory, subdir, "report"))
238  report_file.write(local_report_file.read())
239  local_report_file.close()
240  report_file.close()
241  if args.tool == "bambu":
242  local_args = ""
243  named_list_name = os.path.join(abs_path, "named_list")
244  lines = open(named_list_name).readlines()
245  for line in lines:
246  local_dir = ComputeDirectory(line)
247  if os.path.exists(os.path.join(local_dir, args.tool + "_results_0.xml")):
248  local_args = local_args + " " + os.path.join(local_dir, args.tool + "_results_0.xml")
249  if len(local_args) > 0:
250  #Generate experimental setup xml
251  experimental_setup_file_name = os.path.join(abs_path, "experimental_setup.xml")
252  temp_list = open(experimental_setup_file_name, "w")
253  bambu_version_file_name = os.path.join(abs_path, "bambu_version")
254  bambu_version_file = open(bambu_version_file_name, "w")
255  bambu_version_command = [tool_exe]
256  bambu_version_command.extend(shlex.split("--version"))
257  subprocess.call(bambu_version_command, stdout=bambu_version_file)
258  bambu_version_file.close()
259  bambu_version_file = open(bambu_version_file_name, "r")
260  bambu_version = bambu_version_file.readlines()[-2].rstrip()
261  bambu_version_file.close()
262  if args.commonargs != None:
263  bambu_arguments = ' '.join(' '.join(map(str,l)) for l in args.commonargs)
264  else:
265  bambu_arguments = ""
266  temp_list.write("<?xml version=\"1.0\"?>\n")
267  temp_list.write("<experimental_setup>\n")
268  temp_list.write(" <bambu_version value=\"" + bambu_version + "\"/>\n")
269  temp_list.write(" <timestamp value=\"" + str(datetime.datetime.now()) + "\"/>\n")
270  temp_list.write(" <script value=\"" + args.script + "\"/>\n")
271  temp_list.write(" <bambu_arguments value=\"" + bambu_arguments + "\"/>\n")
272  temp_list.write(" <benchmarks>\n")
273  reordered_list_name = os.path.join(abs_path, "reordered_list")
274  reordered_list = open(reordered_list_name, "r")
275  for line in reordered_list.readlines():
276  temp_list.write(" <benchmark value=\"" + line.rstrip().replace("\\\\\\-","-").replace("\\\\\\=", "=").replace("\\\\\\_", "_").replace("\\\\\\/", "/").replace("\\\\\\\"", "&quot;").replace("\\\\\\.", ".") + "\"/>\n")
277  temp_list.write(" </benchmarks>\n")
278  temp_list.write("</experimental_setup>\n")
279  temp_list.close();
280  local_args = local_args + " " + experimental_setup_file_name
281  if os.path.exists(args.spider_style) :
282  local_args = local_args + " " + args.spider_style + " " + table
283  else:
284  local_args = local_args + " " + os.path.join(os.path.dirname(spider), args.spider_style) + " " + table
285 # logging.info(" Executing " + spider + " " + local_args)
286  logging.info(" Executing " + spider)
287  local_command = [spider]
288  local_command.extend(shlex.split(local_args))
289  return_value = subprocess.call(local_command)
290  logging.info("Collected results of " + directory)
291 
292 
293 #Create Junit Body
294 def CreateJunitBody(directory,ju_file):
295  #Skip if this is a leaf directory
296  if os.path.exists(os.path.join(directory, args.tool + "_return_value")) or os.listdir(directory) == []:
297  return
298  subdirs = [s for s in sorted(os.listdir(directory)) if os.path.isdir(os.path.join(directory,s)) and s != "panda-temp" and s != "HLS_output"]
299  print_testsuite = False
300  for subdir in subdirs:
301  if os.path.exists(os.path.join(directory, subdir, args.tool + "_return_value")):
302  print_testsuite = True
303  CreateJunitBody(os.path.join(directory, subdir),ju_file)
304  failed_counter_file_name = os.path.join(abs_path, "failed_counter")
305  failed_counter = "0"
306  if os.path.exists(failed_counter_file_name):
307  failed_counter_file = open(failed_counter_file_name)
308  failed_counter = failed_counter_file.read()
309 
310  if print_testsuite and len(subdirs) > 0:
311  ju_file.write(" <testsuite disabled=\"0\" errors=\"0\" failures=\""+failed_counter+"\" name=\""+directory+"\" tests=\""+ str(len(subdirs)) + "\" timestamp=\"" + str(datetime.datetime.now()) + "\">\n")
312  for subdir in subdirs:
313  if os.path.exists(os.path.join(directory, subdir, args.tool + "_return_value")):
314  return_value_file_name = os.path.join(directory, subdir, args.tool + "_return_value")
315  return_value_file = open(return_value_file_name)
316  return_value = return_value_file.read()
317  return_value_file.close()
318  args_file = open(os.path.join(directory, subdir, "args"))
319  command_args = args_file.readlines()[0]
320  command_args = command_args.replace(abs_benchmarks_root + "/", "")
321  args_file.close()
322  if return_value == "0":
323  ju_file.write(" <testcase classname=\"PandA-bambu-tests\" name=\"" + command_args.replace("\\", "") + "\">\n")
324  else:
325  if return_value == "124":
326  ju_file.write(" <testcase classname=\"PandA-bambu-tests\" name=\"" + command_args.replace("\\", "") + "\">\n")
327  ju_file.write(" <failure type=\"FAILURE(Timeout)\"></failure>\n")
328  ju_file.write(" <system-out>\n")
329  ju_file.write("<![CDATA[\n")
330  else:
331  ju_file.write(" <testcase classname=\"PandA-bambu-tests\" name=\"" + command_args.replace("\\", "") + "\">\n")
332  ju_file.write(" <failure type=\"FAILURE\"></failure>\n")
333  ju_file.write(" <system-out>\n")
334  ju_file.write("<![CDATA[\n")
335  with open(os.path.join(directory, args.tool + "_failed_output")) as f:
336  for line in deque(f, maxlen=15):
337  ju_file.write(line)
338  ju_file.write("]]>\n")
339  ju_file.write(" </system-out>\n")
340  ju_file.write(" </testcase>\n")
341  if print_testsuite and len(subdirs) > 0:
342  ju_file.write(" </testsuite>\n")
343 
344 
345 #Create PerfPublisher Body
346 def CreatePerfPublisherBody(directory,pp_file):
347  #Skip if this is a leaf directory
348  if os.path.exists(os.path.join(directory, args.tool + "_return_value")) or os.listdir(directory) == []:
349  return
350  subdirs = [s for s in sorted(os.listdir(directory)) if os.path.isdir(os.path.join(directory,s)) and s != "panda-temp" and s != "HLS_output"]
351  print_testsuite = False
352  for subdir in subdirs:
353  if os.path.exists(os.path.join(directory, subdir, args.tool + "_return_value")):
354  print_testsuite = True
355  CreatePerfPublisherBody(os.path.join(directory, subdir),pp_file)
356 
357  for subdir in subdirs:
358  if os.path.exists(os.path.join(directory, subdir, args.tool + "_return_value")):
359  pp_file.write(" <test name=\""+ str(directory)+"/"+ str(subdir) +"\" executed=\"yes\">\n")
360  pp_file.write(" <result>\n")
361  return_value_file_name = os.path.join(directory, subdir, args.tool + "_return_value")
362  return_value_file = open(return_value_file_name)
363  return_value = return_value_file.read()
364  return_value_file.close()
365  args_file = open(os.path.join(directory, subdir, "args"))
366  command_args = args_file.readlines()[0]
367  command_args = command_args.replace(abs_benchmarks_root + "/", "")
368  args_file.close()
369  if return_value == "0":
370  pp_file.write(" <success passed=\"yes\" state=\"100\" hasTimedOut=\"false\"/>\n")
371  cycles_tag = ""
372  areatime_tag = ""
373  slice_tag = ""
374  sliceluts_tag = ""
375  registers_tag = ""
376  dsps_tag = ""
377  brams_tag = ""
378  period_tag = ""
379  dsps_tag = ""
380  slack_tag = ""
381  frequency_tag = ""
382  HLS_execution_time_tag = ""
383  if os.path.exists(os.path.join(directory, subdir, args.tool + "_results_0.xml")):
384  xml_document = xml.dom.minidom.parse(os.path.join(directory, subdir, args.tool + "_results_0.xml"))
385  if len(xml_document.getElementsByTagName("CYCLES")) > 0:
386  cycles_tag = str(xml_document.getElementsByTagName("CYCLES")[0].attributes["value"].value)
387  if len(xml_document.getElementsByTagName("AREAxTIME")) > 0:
388  areatime_tag = str(xml_document.getElementsByTagName("AREAxTIME")[0].attributes["value"].value)
389  if len(xml_document.getElementsByTagName("SLICE")) > 0:
390  slice_tag = str(xml_document.getElementsByTagName("SLICE")[0].attributes["value"].value)
391  if len(xml_document.getElementsByTagName("SLICE_LUTS")) > 0:
392  sliceluts_tag = str(xml_document.getElementsByTagName("SLICE_LUTS")[0].attributes["value"].value)
393  if len(xml_document.getElementsByTagName("REGISTERS")) > 0:
394  registers_tag = str(xml_document.getElementsByTagName("REGISTERS")[0].attributes["value"].value)
395  if len(xml_document.getElementsByTagName("DSPS")) > 0:
396  dsps_tag = str(xml_document.getElementsByTagName("DSPS")[0].attributes["value"].value)
397  if len(xml_document.getElementsByTagName("BRAMS")) > 0:
398  brams_tag = str(xml_document.getElementsByTagName("BRAMS")[0].attributes["value"].value)
399  if len(xml_document.getElementsByTagName("PERIOD")) > 0:
400  period_tag = str(xml_document.getElementsByTagName("PERIOD")[0].attributes["value"].value)
401  if len(xml_document.getElementsByTagName("CLOCK_SLACK")) > 0:
402  slack_tag = str(xml_document.getElementsByTagName("CLOCK_SLACK")[0].attributes["value"].value)
403  if len(xml_document.getElementsByTagName("FREQUENCY")) > 0:
404  frequency_tag = str(xml_document.getElementsByTagName("FREQUENCY")[0].attributes["value"].value)
405  if len(xml_document.getElementsByTagName("HLS_execution_time")) > 0:
406  HLS_execution_time_tag = str(xml_document.getElementsByTagName("HLS_execution_time")[0].attributes["value"].value)
407 
408  if cycles_tag != "":
409  pp_file.write(" <performance unit=\"cycles\" mesure=\""+ cycles_tag + "\" isRelevant=\"true\"/>\n")
410  if HLS_execution_time_tag != "":
411  pp_file.write(" <executiontime unit=\"s\" mesure=\""+ HLS_execution_time_tag + "\" isRelevant=\"true\"/>\n")
412 
413  if areatime_tag != "" or slice_tag != "" or sliceluts_tag != "" or registers_tag != "" or dsps_tag != "" or brams_tag != "" or period_tag != "" or dsps_tag != "" or slack_tag != "" or frequency_tag != "":
414  pp_file.write(" <metrics>\n")
415  if areatime_tag != "":
416  pp_file.write(" <areatime unit=\"lutxns\" mesure=\""+ areatime_tag + "\" isRelevant=\"true\"/>\n")
417  if slice_tag != "":
418  pp_file.write(" <slices unit=\"ns\" mesure=\""+ slice_tag + "\" isRelevant=\"true\"/>\n")
419  if sliceluts_tag != "":
420  pp_file.write(" <sliceluts unit=\"slice\" mesure=\""+ sliceluts_tag + "\" isRelevant=\"true\"/>\n")
421  if registers_tag != "":
422  pp_file.write(" <registers unit=\"registers\" mesure=\""+ registers_tag + "\" isRelevant=\"true\"/>\n")
423  if dsps_tag != "":
424  pp_file.write(" <dsps unit=\"dsp\" mesure=\""+ dsps_tag + "\" isRelevant=\"true\"/>\n")
425  if brams_tag != "":
426  pp_file.write(" <brams unit=\"bram\" mesure=\""+ brams_tag + "\" isRelevant=\"true\"/>\n")
427  if period_tag != "":
428  pp_file.write(" <period unit=\"ns\" mesure=\""+ period_tag + "\" isRelevant=\"true\"/>\n")
429  if slack_tag != "":
430  pp_file.write(" <slack unit=\"ns\" mesure=\""+ slack_tag + "\" isRelevant=\"true\"/>\n")
431  if frequency_tag != "":
432  pp_file.write(" <frequency unit=\"MHz\" mesure=\""+ frequency_tag + "\" isRelevant=\"true\"/>\n")
433  pp_file.write(" </metrics>\n")
434  else:
435  if return_value == "124":
436  pp_file.write(" <success passed=\"no\" state=\"0\" hasTimedOut=\"true\"/>\n")
437  else:
438  pp_file.write(" <success passed=\"no\" state=\"0\" hasTimedOut=\"false\"/>\n")
439  pp_file.write(" </result>\n")
440  pp_file.write(" </test>\n")
441 
442 parser = argparse.ArgumentParser(description="Performs panda tests", fromfile_prefix_chars='@')
443 parser.add_argument("files", help="The files to be tested: they can be configuration files, directories containing benchmarks or source code files.", nargs='*', action="append")
444 parser.add_argument('-l', "--benchmarks_list", help="The file containing the list of tests to be performed", nargs='*', action="append")
445 parser.add_argument('-b', "--benchmarks_root", help="The directory containing benchmarks")
446 parser.add_argument('-o', "--output", help="The directory where output files we be put (default=\"output\")", default="output")
447 parser.add_argument('-j', help="The number of jobs which execute the benchmarks (default=\"1\")", default=1, type=positive_integer, action=StoreOrUpdateMin)
448 parser.add_argument("--bambu", help="The bambu executable (default=/opt/panda/bin/bambu)", default="/opt/panda/bin/bambu")
449 parser.add_argument("--spider", help="The spider executable (default=/opt/panda/bin/spider)", default="/opt/panda/bin/spider")
450 parser.add_argument("--spider-style", help="The spider table style relative to the spider executable (default=../lib/latex_format_bambu_results.xml)", default="../lib/latex_format_bambu_results.xml")
451 parser.add_argument("--zebu", help="The zebu executable (default=/opt/panda/bin/zebu)", default="/opt/panda/bin/zebu")
452 parser.add_argument('-t', "--timeout", help="Timeout for tool execution (default=60m)", default="60m")
453 parser.add_argument('-a', "--args", help="A set of arguments to be passed to the tool", nargs='*', action='append')
454 parser.add_argument('-c', "--commonargs", help="A set of arguments to be passed to the tool", nargs='*', action='append')
455 parser.add_argument("--table", help="Print the results in tex format", default="results.tex")
456 parser.add_argument("--tool", help="The tool to be tested", default="bambu")
457 parser.add_argument("--ulimit", help="The ulimit options", default="-f 2097152 -v 8388608 -s 16384")
458 parser.add_argument("--stop", help="Stop the execution on first error (default=false)", default=False, action="store_true")
459 parser.add_argument("--returnfail", help="Return FAILURE in case at least one test fails (default=false)", default=False, action="store_true")
460 parser.add_argument("--mail", help="Send a mail with the result")
461 parser.add_argument("--name", help="Set the name of this regression (default=Bambu regression)", nargs='*', action='append')
462 parser.add_argument("--no-clean", help="Do not clean produced files", default=False, action="store_true")
463 parser.add_argument("--restart", help="Restart last execution (default=false)", default=False, action="store_true")
464 parser.add_argument("--script", help="Set the bash script in the generated tex", default="")
465 parser.add_argument("--junitdir", help="Set the JUnit directory", default="")
466 parser.add_argument("--perfpublisherdir", help="Set the PerfPublisher directory", default="")
467 
468 args = parser.parse_args()
469 n_jobs = args.j # set this, because it will be overwritten by the parse of modified_argv
470 logging.basicConfig(level=logging.INFO,format='%(levelname)s: %(message)s')
471 
472 #The absolute path of current script
473 abs_script = os.path.abspath(sys.argv[0])
474 
475 #Expand configuration files
476 modified_argv = []
477 modified_argv.append(sys.argv[0])
478 abs_configuration_dir=""
479 for arg in sys.argv[1:]:
480  if arg in args.files[0]:
481  #.c/c++ file
482  if (arg[-2:] == ".c") or (arg[-2:] == ".C") or (arg[-4:] == ".CPP") or (arg[-4:] == ".cpp") or (arg[-4:] == ".cxx") or (arg[-3:] == ".cc") or (arg[-4:] == ".c++"):
483  modified_argv.append(arg)
484  #Check if it is a directory
485  elif os.path.exists(arg) and os.path.isdir(arg):
486  modified_argv.append(arg)
487  elif args.benchmarks_root != None and os.path.exists(os.path.join(os.path.abspath(args.benchmarks_root), arg)) and os.path.isdir(os.path.join(os.path.abspath(args.benchmarks_root), arg)):
488  modified_argv.append(arg)
489  elif os.path.exists(os.path.join(os.path.dirname(abs_script), arg)) and os.path.isdir(os.path.join(os.path.dirname(abs_script), arg)):
490  modified_argv.append(arg)
491  else:
492  modified_argv.append("@" + arg)
493  else:
494  modified_argv.append(arg)
495 args = parser.parse_args(modified_argv)
496 
497 #The absolute path of current script
498 abs_script = os.path.abspath(sys.argv[0])
499 
500 #The table to be produced
501 table = os.path.abspath(args.table)
502 #Check if output directory exists, if yes abort
503 if os.path.exists(args.output) and not args.restart:
504  logging.error("Output directory " + args.output + " already exists. Please remove it or specify a different one with -o")
505  sys.exit(1)
506 
507 #Check if JUnit dir exist
508 if args.junitdir != "" and not os.path.exists(args.junitdir):
509  os.mkdir(args.junitdir)
510 #Check if PerfPublisher dir exist
511 if args.perfpublisherdir != "" and not os.path.exists(args.perfpublisherdir):
512  os.mkdir(args.perfpublisherdir)
513 #compute JUnit file name
514 junit_file_name = ""
515 if args.junitdir != "":
516  junit_index = 0
517  junit_file_name = os.path.abspath(os.path.join(args.junitdir, "Junit_report"+str(junit_index)+".xml"))
518  while os.path.isfile(junit_file_name) :
519  junit_index = junit_index + 1
520  junit_file_name = os.path.abspath(os.path.join(args.junitdir, "Junit_report"+str(junit_index)+".xml"))
521 #compute PerfPublisher file name
522 perfpublisher_name = ""
523 perfpublisher_file_name = ""
524 if args.perfpublisherdir != "":
525  perfpublisher_index = 0
526  perfpublisher_name = "PerfPublisher_report"+str(perfpublisher_index)
527  perfpublisher_file_name = os.path.abspath(os.path.join(args.perfpublisherdir, perfpublisher_name+".xml"))
528  while os.path.isfile(perfpublisher_file_name) :
529  perfpublisher_index = perfpublisher_index + 1
530  perfpublisher_name = "PerfPublisher_report"+str(perfpublisher_index)
531  perfpublisher_file_name = os.path.abspath(os.path.join(args.perfpublisherdir, perfpublisher_name+".xml"))
532 
533 #Create the folder and enter in it
534 abs_path = os.path.abspath(args.output)
535 
536 if args.restart:
537  if not os.path.exists(abs_path):
538  args.restart = False
539 if not args.restart:
540  os.mkdir(abs_path)
541 os.chdir(abs_path)
542 
543 
544 #Skipping if all benchmarks already pass
545 if args.restart:
546  failed_counter_file_name = os.path.join(abs_path, "failed_counter")
547  if os.path.exists(failed_counter_file_name):
548  failed_counter_file = open(failed_counter_file_name)
549  failed_counter = failed_counter_file.read()
550  if failed_counter == "0" and args.junitdir == "" and args.perfpublisherdir == "":
551  logging.info("Already pass")
552  sys.exit(0)
553 
554 #Check tool executable
555 tool_exe = ""
556 if args.tool == "bambu":
557  if os.path.isfile(args.bambu) and os.access(args.bambu, os.X_OK):
558  tool_exe = args.bambu
559  else:
560  #Check bambu in the path
561  for path in os.environ["PATH"].split(os.pathsep):
562  exe_file = os.path.join(path, "bambu")
563  if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK):
564  tool_exe = exe_file
565  if tool_exe == "":
566  if args.bambu != "opt/panda/bin/bambu":
567  if not os.path.isfile(args.bambu):
568  logging.error(args.bambu + " does not exist")
569  else:
570  logging.error(args.bambu + " is not an executable")
571  else:
572  logging.error("bambu not found")
573  sys.exit(1)
574  logging.info("Bambu found: " + tool_exe)
575 elif args.tool == "zebu":
576  if os.path.isfile(args.zebu) and os.access(args.zebu, os.X_OK):
577  tool_exe = args.zebu
578  else:
579  #Check zebu in the path
580  for path in os.environ["PATH"].split(os.pathsep):
581  exe_file = os.path.join(path, "zebu")
582  if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK):
583  tool_exe = exe_file
584  if tool_exe == "":
585  if args.zebu != "opt/panda/bin/zebu":
586  if not os.path.isfile(args.zebu):
587  logging.error(args.zebu + " does not exist")
588  else:
589  logging.error(args.zebu + " is not an executable")
590  else:
591  logging.error("zebu not found")
592  sys.exit(1)
593  logging.info("Zebu found: " + tool_exe)
594 else:
595  tool_exe = args.tool
596  if distutils.spawn.find_executable(tool_exe) == None:
597  logging.error(tool_exe + " not found")
598  sys.exit(1)
599 
600 if args.benchmarks_root is None:
601  abs_benchmarks_root = abs_configuration_dir
602 else:
603  if os.path.isabs(args.benchmarks_root):
604  abs_benchmarks_root = os.path.abspath(args.benchmarks_root)
605  else:
606  if os.path.exists(os.path.join(os.path.abspath(".."), args.benchmarks_root)):
607  abs_benchmarks_root = os.path.join(os.path.abspath(".."), args.benchmarks_root)
608  else:
609  if os.path.exists(os.path.join(os.path.abspath(os.path.join(os.path.dirname(abs_script), "../../..")), args.benchmarks_root)):
610  abs_benchmarks_root = os.path.join(os.path.abspath(os.path.join(os.path.dirname(abs_script), "../../..")), args.benchmarks_root)
611  else:
612  logging.error(args.benchmarks_root + " not found")
613  sys.exit(1)
614 
615 #Check spider executable
616 spider = ""
617 if os.path.isfile(args.spider) and os.access(args.spider, os.X_OK):
618  spider = args.spider
619 else:
620  #Check spider in the path
621  for path in os.environ["PATH"].split(os.pathsep):
622  exe_file = os.path.join(path, "spider")
623  if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK):
624  spider = exe_file
625 if spider == "":
626  if args.spider != "opt/panda/bin/spider":
627  if not os.path.isfile(args.spider):
628  logging.error(args.spider + " does not exist")
629  else:
630  logging.error(args.spider + " is not an executable")
631  else:
632  logging.error("spider not found")
633  sys.exit(1)
634 
635 if args.benchmarks_root is None:
636  abs_benchmarks_root = abs_configuration_dir
637 else:
638  if os.path.isabs(args.benchmarks_root):
639  abs_benchmarks_root = os.path.abspath(args.benchmarks_root)
640  else:
641  if os.path.exists(os.path.join(os.path.abspath(".."), args.benchmarks_root)):
642  abs_benchmarks_root = os.path.join(os.path.abspath(".."), args.benchmarks_root)
643  else:
644  if os.path.exists(os.path.join(os.path.abspath(os.path.join(os.path.dirname(abs_script), "../../..")), args.benchmarks_root)):
645  abs_benchmarks_root = os.path.join(os.path.abspath(os.path.join(os.path.dirname(abs_script), "../../..")), args.benchmarks_root)
646  else:
647  logging.error(args.benchmarks_root + " not found")
648  sys.exit(1)
649 mutt = None
650 logging.info("Spider found: " + spider)
651 
652 #Check mutt executable
653 if args.mail != None:
654  #Check mutt in the path
655  for path in os.environ["PATH"].split(os.pathsep):
656  exe_file = os.path.join(path, "mutt")
657  if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK):
658  mutt = exe_file
659  if mutt == None:
660  logging.error("mutt not found")
661  sys.exit(1)
662 
663 
664 if not args.restart:
665  #Check if file lists exist
666  abs_lists = []
667  if args.benchmarks_list != None:
668  for relative_list in args.benchmarks_list:
669  #First look in the current directory
670  if os.path.exists(os.path.abspath("../" + relative_list[0])):
671  abs_lists.append(os.path.abspath("../" + relative_list[0]))
672  #Then look in script directory
673  elif os.path.exists(os.path.join(os.path.dirname(abs_script), relative_list[0])):
674  abs_lists.append(os.path.join(os.path.dirname(abs_script), relative_list[0]))
675  #Then look in configuration directory
676  elif os.path.exists(os.path.join(abs_configuration_dir, relative_list[0])):
677  abs_lists.append(os.path.join(abs_configuration_dir, relative_list[0]))
678  #Then look in benchmarks root
679  elif os.path.exists(os.path.join(abs_benchmarks_root, relative_list[0])):
680  abs_lists.append(os.path.join(abs_benchmarks_root, relative_list[0]))
681  else:
682  logging.error(relative_list[0] + " does not exist")
683  sys.exit(1)
684  files_list = []
685  #Create temp list with arg
686  for arg in args.files[0]:
687  #.c/.c++ file
688  if (arg[-2:] == ".c") or (arg[-2:] == ".C") or (arg[-4:] == ".CPP") or (arg[-4:] == ".cpp") or (arg[-4:] == ".cxx") or (arg[-3:] == ".cc") or (arg[-4:] == ".c++"):
689  files_list.append(arg)
690  #Check if it is a directory
691  elif os.path.exists(arg) and os.path.isdir(arg):
692  files_list.append(arg)
693  elif os.path.exists(os.path.join(os.path.dirname(abs_script), arg)) and os.path.isdir(os.path.join(os.path.dirname(abs_script), arg)):
694  files_list.append(arg)
695  elif os.path.exists(os.path.join(abs_benchmarks_root, arg)) and os.path.isdir(os.path.join(abs_benchmarks_root, arg)):
696  files_list.append(os.path.join(abs_benchmarks_root, arg))
697 
698  if args.benchmarks_list == None and len(files_list) == 0 and (args.tool == "bambu" or args.tool == "zebu"):
699  logging.error("Benchmarks not found")
700  sys.exit(2)
701 
702  if len(files_list) > 0:
703  temp_list = open(os.path.join(abs_path, "temp_list"), "w")
704  for element in files_list:
705  temp_list.write(element)
706  temp_list.close()
707  abs_lists.append(os.path.join(abs_path, "temp_list"))
708 
709  #Reordering elements in each row
710  reordered_list_name = os.path.join(abs_path, "reordered_list")
711  reordered_list = open(reordered_list_name, "w")
712 
713  logging.info("Preparing benchmark list")
714  logging.info(" Reordering arguments")
715  for abs_list in abs_lists:
716  list_file = open(abs_list)
717  lines = list_file.readlines()
718  list_file.close()
719  for line in lines:
720  if line.strip() == "":
721  continue
722  if line[0] =='#':
723  continue
724  if args.tool != "bambu" and args.tool != "zebu":
725  reordered_list.write(line)
726  continue
727  tokens = shlex.split(line)
728  parameters = list()
729  #Flag used to ad-hoc manage --param arg
730  follow_param = False
731  for token in tokens:
732  if token[0] == '-':
733  parameters.append(re.escape(token))
734  if token.find("--param") != -1:
735  follow_param = True;
736  else:
737  follow_param = False;
738  else:
739  if follow_param == True:
740  parameters.append(re.escape(token))
741  else:
742  reordered_list.write(token + " ")
743  follow_param = False;
744  for parameter in parameters:
745  reordered_list.write(re.escape(parameter) + " ")
746  reordered_list.write("\n")
747  reordered_list.close()
748 
749  #Expanding directory
750  expanded_list_name = os.path.join(abs_path, "expanded_list")
751  expanded_list = open(expanded_list_name, "w")
752 
753  logging.info(" Expanding directory")
754  lines = open(reordered_list_name).readlines()
755  for line in lines:
756  if line.strip() == "":
757  continue
758  tokens = shlex.split(line)
759  if args.tool == "bambu" or args.tool == "zebu":
760  if(tokens[0][0] != '/'):
761  first_parameter = os.path.join(abs_benchmarks_root, tokens[0])
762  else:
763  first_parameter = tokens[0]
764  else:
765  first_parameter = tokens[0].replace("BENCHMARKS\_ROOT", abs_benchmarks_root)
766  other_parameters = tokens[1:len(tokens)]
767  if not os.path.exists(first_parameter) and (args.tool == "bambu" or args.tool == "zebu"):
768  logging.error(first_parameter + " does not exist")
769  sys.exit(1)
770  #Check if it is a directory or a file
771  if os.path.isdir(first_parameter):
772  logging.info(" " + tokens[0])
773  c_files = SearchCFiles(first_parameter)
774  c_files = sorted(c_files)
775  for c_file in c_files:
776  expanded_list.write(c_file)
777  for other_parameter in other_parameters:
778  expanded_list.write(" " + other_parameter.replace("BENCHMARKS\_ROOT", abs_benchmarks_root))
779  expanded_list.write("\n")
780  else:
781  expanded_list.write(first_parameter)
782  for other_parameter in other_parameters:
783  if ((other_parameter[-2:] == ".c") or (other_parameter[-2:] == ".C") or (other_parameter[-4:] == ".CPP") or (other_parameter[-4:] == ".cpp") or (other_parameter[-4:] == ".cxx") or (other_parameter[-3:] == ".cc") or (other_parameter[-4:] == ".c++") or other_parameter[-4:] == ".xml") and other_parameter[0] != '\\':
784  if other_parameter[0] == '/':
785  expanded_list.write(" " + other_parameter)
786  else:
787  expanded_list.write(" " + os.path.join(abs_benchmarks_root, other_parameter))
788  else:
789  expanded_list.write(" " + other_parameter.replace("BENCHMARKS\_ROOT", abs_benchmarks_root).replace("BENCHMARKS_ROOT", abs_benchmarks_root))
790  expanded_list.write("\n")
791  expanded_list.close()
792 
793  #Adding parameters
794  logging.info(" Considering all tool arguments")
795  arg_lists = args.args
796  if not arg_lists:
797  arg_lists = [("")]
798  arged_list_name = os.path.join(abs_path, "arged_list")
799  arged_list = open(arged_list_name, "w")
800  lines = open(expanded_list_name).readlines()
801  for arg_list in arg_lists:
802  for line in lines:
803  arged_list.write(line.rstrip())
804  if len(arg_list) > 0:
805  arg = arg_list[0]
806  if arg[0] == "\"":
807  arg = arg[1:-1]
808  arged_list.write(" " + arg)
809  if args.commonargs != None and len(args.commonargs) > 0:
810  for commonarg in args.commonargs:
811  arged_list.write(" " + commonarg[0].replace("#", " "))
812  arged_list.write("\n")
813  arged_list.close()
814 
815  #Name of benchmarks
816  full_names = set()
817 
818 
819  #Adding benchmark name
820  logging.info(" Adding benchmark name")
821  named_list_name = os.path.join(abs_path, "named_list")
822  named_list = open(named_list_name, "w")
823  lines = open(arged_list_name).readlines()
824  for line in lines:
825  named_list.write(line.rstrip())
826  #Retrieve configuration name and benchmark name
827  configuration_name = ""
828  benchmark_name = ""
829  tokens = shlex.split(line)
830  for token in tokens:
831  if token.find("--configuration-name") != -1:
832  configuration_name = token[len("--configuration-name="):]
833  if token.find("--benchmark-name") != -1:
834  benchmark_name = token[len("--benchmark-name="):]
835  if benchmark_name == "":
836  if args.tool != "bambu" and args.tool != "zebu":
837  logging.error("Missing benchmark name")
838  sys.exit(1)
839  benchmark_name = os.path.basename(line.split()[0])[:-2]
840  named_list.write(" --benchmark-name=" + benchmark_name)
841  full_name = configuration_name + ":" + benchmark_name
842  logging.info(" " + full_name)
843  if full_name in full_names:
844  logging.error("Duplicated configuration name - benchmark name: " + full_name)
845  sys.exit(1)
846  full_names.add(full_name)
847  named_list.write("\n")
848  named_list.close()
849 
850 
851  #Generating folder
852  logging.info(" Generating output directories")
853  lines = open(named_list_name).readlines()
854  for line in lines:
855  new_dir = ComputeDirectory(line)
856  logging.info(" Creating directory " + new_dir)
857  os.makedirs(new_dir)
858 else:
859  logging.info(" Skipping generation of lists and directories")
860  named_list_name = os.path.join(abs_path, "named_list")
861  if not os.path.exists(named_list_name):
862  logging.error("List of previous run not found")
863  sys.exit(1)
864 
865 #Create threads
866 logging.info(" Launching tool")
867 lock = threading.RLock()
868 lock_creation_destruction = threading.RLock()
869 passed_benchmark = 0
870 total_benchmark = 0
871 threads = []
872 children = [None] * n_jobs
873 for thread_index in range(n_jobs):
874  threads.insert(thread_index, threading.Thread(target=execute_tests, args=(named_list_name, thread_index)))
875  threads[thread_index].daemon=True
876  threads[thread_index].start()
877 
878 try:
879  #Wait threads
880  for thread_index in range(n_jobs):
881  while threads[thread_index].isAlive():
882  threads[thread_index].join(100)
883 except KeyboardInterrupt:
884  logging.error("SIGINT received")
885  failure = True
886  for local_thread_index in range(n_jobs):
887  if children[local_thread_index] != None:
888  if children[local_thread_index].poll() == None:
889  try:
890  kill_proc_tree(children[local_thread_index].pid)
891  except OSError:
892  pass
893  sys.exit(1)
894 
895 #Collect results
896 CollectResults(abs_path)
897 
898 #In case, it create the JUnit file
899 if args.junitdir != "":
900  junit_file = open(junit_file_name, "w")
901  junit_file.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
902  junit_file.write("<testsuites disabled=\"0\" errors=\"0\" failures=\""+str(total_benchmark - passed_benchmark)+"\" name=\"" + abs_path + "\" tests=\"" + str(total_benchmark) + "\">\n")
903  CreateJunitBody(abs_path,junit_file)
904  junit_file.write("</testsuites>\n")
905 
906 #In case, it create the PerfPublisher file
907 if args.perfpublisherdir != "":
908  perfpublisher_file = open(perfpublisher_file_name, "w")
909  perfpublisher_file.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
910  perfpublisher_file.write("<report name=\""+ perfpublisher_name + "\" categ=\"PandA-Report\">\n")
911  CreatePerfPublisherBody(abs_path,perfpublisher_file)
912  perfpublisher_file.write("</report>\n")
913 
914 #Prepare final report
915 if args.tool == "bambu" or args.tool == "zebu":
916  report_file_name = os.path.join(abs_path, "report")
917  report_file = open(report_file_name)
918  lines = report_file.readlines()
919  report_file.close()
920  report_file = open(report_file_name, "w")
921  command = [tool_exe, "--version"]
922  subprocess.call(command, stderr=report_file, stdout=report_file)
923  report_file.write("SYSTEM INFORMATION:\n")
924  report_file.flush()
925  command = ["lsb_release", "-a"]
926  subprocess.call(command, stderr=report_file, stdout=report_file)
927  report_file.write("\n")
928  report_file.write("CURRENT TIME:\n")
929  report_file.write(str(datetime.datetime.now())+ "\n\n")
930  report_file.write("PASSED TESTS:\n")
931  report_file.write(str(passed_benchmark) + "/" + str(total_benchmark) + "\n\n")
932 
933  failed_counter_file_name = os.path.join(abs_path, "failed_counter")
934  failed_counter_file = open(failed_counter_file_name, "w")
935  failed_counter_file.write(str(total_benchmark - passed_benchmark))
936  failed_counter_file.close()
937 
938  for line in lines:
939  report_file.write(line)
940  report_file.close()
941 
942 if args.mail != None:
943  outcome = ""
944  if args.stop:
945  if failure:
946  outcome = "FAILURE"
947  else:
948  outcome = "SUCCESS"
949  else:
950  outcome = str(passed_benchmark) + "/" + str(total_benchmark)
951  full_name = ""
952  if len(args.name):
953  for name in args.name:
954  full_name = full_name + name[0]
955  else:
956  full_name = "Bambu"
957  local_command = "cat " + report_file_name + " | mutt -s \"" + full_name + ": " + outcome + "\" " + args.mail
958  subprocess.call(local_command, shell=True)
959 if failure:
960  sys.exit(1)
961 sys.exit(0)
def CreateJunitBody(directory, ju_file)
Definition: test_panda.py:294
def SearchCFiles(directory)
Definition: test_panda.py:186
unsigned map[NUM_VERTICES]
Definition: bfs.c:12
def GetChildren(parent_pid)
Definition: test_panda.py:38
char str[25]
Definition: fixedptc.c:8
def __call__(self, parser, namespace, values, option_string=None)
Definition: test_panda.py:29
def CreatePerfPublisherBody(directory, pp_file)
Definition: test_panda.py:346
def CollectResults(directory)
Definition: test_panda.py:197
def kill_proc_tree(pid)
Definition: test_panda.py:48
def ComputeDirectory(line)
Definition: test_panda.py:173
short int read(short int *data)
Definition: read.c:3
def execute_tests(named_list, thread_index)
Definition: test_panda.py:54

Generated on Mon Feb 12 2024 13:02:48 for PandA-2024.02 by doxygen 1.8.13