16 from defusedxml
import minidom
18 print>>sys.stderr,
'WARNING: pyhon-defusedxml not available, falling back to unsafe standard libraries...' 19 from xml.dom
import minidom
21 from collections
import deque
30 raise argparse.ArgumentTypeError(
31 "%s must be a positive integer" % value)
35 class StoreOrUpdateMin(argparse.Action):
38 def __call__(self, parser, namespace, values, option_string=None):
41 setattr(namespace, self.dest, values)
43 setattr(namespace, self.dest, min(namespace.j, values))
49 ps_command = subprocess.Popen(
50 "ps -o pid --ppid %d --noheaders" % parent_pid, shell=
True, stdout=subprocess.PIPE)
51 ps_output = ps_command.stdout.read()
53 for pid_str
in ps_output.split(
"\n")[:-1]:
62 os.kill(pid, signal.SIGKILL)
63 for child
in children:
69 global passed_benchmark
70 global total_benchmark
74 lines = open(named_list).readlines()
76 local_index = line_index
78 while local_index < len(lines)
and not (failure
and args.stop):
80 failed_output_file_name = os.path.join(
81 cwd, args.tool +
"_failed_output")
82 if os.path.exists(failed_output_file_name):
83 os.remove(failed_output_file_name)
84 timing_violation_report_file_name = os.path.join(
85 cwd,
"HLS_output/Synthesis/timing_violation_report")
86 if os.path.exists(timing_violation_report_file_name):
87 os.remove(timing_violation_report_file_name)
88 tool_return_value_file_name = os.path.join(
89 cwd, args.tool +
"_return_value")
90 if args.restart
and os.path.exists(os.path.join(cwd, args.tool +
"_return_value")):
91 tool_return_value_file = open(tool_return_value_file_name,
"r") 92 return_value = tool_return_value_file.read() 93 tool_return_value_file.close() 94 if return_value ==
"0":
98 logging.info(
" SKIPPING --- OVERALL: " +
str(passed_benchmark) +
" passed, " +
str(total_benchmark-passed_benchmark) +
99 " failed, " +
str(len(lines)-total_benchmark) +
" queued --- " + lines[local_index].replace(
"\\",
""))
100 local_index = line_index
103 HLS_output_directory = os.path.join(cwd,
"HLS_output")
104 if os.path.exists(HLS_output_directory):
105 shutil.rmtree(HLS_output_directory)
106 output_file_name = os.path.join(cwd, args.tool +
"_execution_output")
107 output_file = open(output_file_name,
"w")
108 local_args = lines[local_index]
109 if local_args[0] ==
"\"":
110 local_args = local_args[1:-1]
111 if args.tool !=
"bambu" and args.tool !=
"zebu":
112 tokens = shlex.split(lines[local_index])
113 args_without_benchmark_name =
"" 115 if token.find(
"--benchmark-name") == -1:
116 args_without_benchmark_name += token +
" " 117 local_args = args_without_benchmark_name
118 local_command =
"ulimit " + args.ulimit + \
119 "; exec timeout " + args.timeout +
" " + tool_exe
120 local_command = local_command +
" " + local_args
121 output_file.write(
"#" * 80 +
"\n")
122 output_file.write(
"cd " + cwd +
"; ")
123 output_file.write(local_command +
"\n")
124 output_file.write(
"#" * 80 +
"\n")
127 with lock_creation_destruction:
128 if not (failure
and args.stop):
129 children[thread_index] = subprocess.Popen(
130 local_command, stderr=output_file, stdout=output_file, cwd=cwd, shell=
True, executable=
"/bin/bash")
132 return_value = children[thread_index].wait()
135 with lock_creation_destruction:
136 if return_value != 0
and (args.stop
or args.returnfail):
138 if failure
and args.stop:
139 for local_thread_index
in range(n_jobs):
140 if children[local_thread_index] !=
None:
141 if children[local_thread_index].poll() ==
None:
144 children[local_thread_index].pid)
147 os.fsync(output_file.fileno())
149 tool_return_value_file = open(tool_return_value_file_name,
"w")
150 tool_return_value_file.write(
str(return_value))
151 tool_return_value_file.close()
152 args_file = open(os.path.join(cwd,
"args"),
"w")
153 args_file.write(lines[local_index])
155 if return_value == 0
and os.path.exists(os.path.join(cwd, args.tool +
"_results_0.xml")):
156 tool_results_file_name = os.path.join(cwd, args.tool +
"_results")
157 tool_results_file = open(tool_results_file_name,
"w")
158 tool_results_string =
"" 159 xml_document = minidom.parse(
160 os.path.join(cwd, args.tool +
"_results_0.xml"))
161 if len(xml_document.getElementsByTagName(
"CYCLES")) > 0:
162 cycles_tag = xml_document.getElementsByTagName(
"CYCLES")[0]
163 tool_results_string = tool_results_string + \
164 cycles_tag.attributes[
"value"].value +
" CYCLES" 165 if len(xml_document.getElementsByTagName(
"CLOCK_SLACK")) > 0:
166 slack_tag = xml_document.getElementsByTagName(
"CLOCK_SLACK")[0]
167 tool_results_string = tool_results_string +
" *** " + \
168 slack_tag.attributes[
"value"].value +
"ns" 169 tool_results_file.write(tool_results_string)
170 tool_results_file.close()
171 if not (failure
and args.stop)
or (return_value != -9
and return_value != 0):
172 if return_value != 0:
173 os.symlink(os.path.basename(output_file_name),
str(os.path.join(
174 os.path.dirname(output_file_name), args.tool +
"_failed_output")))
175 if os.path.exists(timing_violation_report_file_name):
176 annotated_timing_violation_report_file = open(
177 os.path.join(cwd,
"timing_violation_report"),
"w")
178 annotated_timing_violation_report_file.write(
"#" * 80 +
"\n")
179 annotated_timing_violation_report_file.write(
181 annotated_timing_violation_report_file.write(
182 local_command +
"\n")
183 annotated_timing_violation_report_file.write(
"#" * 80 +
"\n")
184 annotated_timing_violation_report_file.flush()
185 timing_violation_report_file = open(
186 timing_violation_report_file_name)
187 annotated_timing_violation_report_file.write(
188 timing_violation_report_file.read())
189 timing_violation_report_file.close()
190 annotated_timing_violation_report_file.close()
193 if return_value == 0:
194 passed_benchmark += 1
195 if not args.no_clean:
196 for sub
in os.listdir(cwd):
197 if os.path.isdir(os.path.join(cwd, sub)):
198 shutil.rmtree(os.path.join(cwd, sub))
200 if sub != args.tool +
"_return_value" and sub != args.tool +
"_execution_output" and sub != args.tool +
"_results_0.xml" and sub !=
"args":
201 os.remove(os.path.join(cwd, sub))
202 if os.path.exists(os.path.join(cwd, args.tool +
"_results_0.xml")):
203 logging.info(
" SUCCESS (" + tool_results_string +
") --- OVERALL: " +
str(passed_benchmark) +
" passed, " +
str(
204 total_benchmark-passed_benchmark) +
" failed, " +
str(len(lines)-total_benchmark) +
" queued --- " + lines[local_index].replace(
"\\",
""))
206 logging.info(
" SUCCESS --- OVERALL: " +
str(passed_benchmark) +
" passed, " +
str(total_benchmark-passed_benchmark) +
207 " failed, " +
str(len(lines)-total_benchmark) +
" queued --- " + lines[local_index].replace(
"\\",
""))
208 elif return_value == 124:
209 logging.info(
" FAILURE (Timeout) --- OVERALL: " +
str(passed_benchmark) +
" passed, " +
str(total_benchmark -
210 passed_benchmark) +
" failed, " +
str(len(lines)-total_benchmark) +
" queued --- " + lines[local_index].replace(
"\\",
""))
211 elif return_value == 153:
212 logging.info(
" FAILURE (File size limit exceeded) --- OVERALL: " +
str(passed_benchmark) +
" passed, " +
str(total_benchmark -
213 passed_benchmark) +
" failed, " +
str(len(lines)-total_benchmark) +
" queued --- " + lines[local_index].replace(
"\\",
""))
215 logging.info(
" FAILURE --- OVERALL: " +
str(passed_benchmark) +
" passed, " +
str(total_benchmark-passed_benchmark) +
216 " failed, " +
str(len(lines)-total_benchmark) +
" queued --- " + lines[local_index].replace(
"\\",
""))
218 local_index = line_index
225 configuration_name =
"" 227 tokens = shlex.split(line)
229 if token.find(
"--configuration-name") != -1:
230 configuration_name = token[len(
"--configuration-name="):]
231 if token.find(
"--benchmark-name") != -1:
232 benchmark_name = token[len(
"--benchmark-name="):]
233 new_dir = os.path.join(abs_path, configuration_name, benchmark_name)
240 logging.info(
" Looking for file in " +
str(directory))
242 for element
in os.listdir(directory):
243 if os.path.isdir(os.path.join(directory, element)):
244 files = files.union(
SearchCFiles(os.path.join(directory, element)))
245 elif (element[-2:] ==
".c")
or (element[-2:] ==
".C")
or (element[-4:] ==
".CPP")
or (element[-4:] ==
".cpp")
or (element[-4:] ==
".cxx")
or (element[-3:] ==
".cc")
or (element[-4:] ==
".c++"):
246 files.add(os.path.join(directory, element))
254 if os.path.exists(os.path.join(directory, args.tool +
"_return_value"))
or os.listdir(directory) == []:
256 subdirs = [s
for s
in sorted(os.listdir(directory))
if os.path.isdir(
257 os.path.join(directory, s))
and s !=
"panda-temp" and s !=
"HLS_output"]
258 for subdir
in subdirs:
260 tool_failed_output = open(os.path.join(
261 directory, args.tool +
"_failed_output"),
"w")
262 for subdir
in subdirs:
263 if os.path.exists(os.path.join(directory, subdir, args.tool +
"_failed_output")):
264 tool_failed_output.write(
265 open(os.path.join(directory, subdir, args.tool +
"_failed_output")).
read())
266 if os.path.exists(os.path.join(directory, subdir, args.tool +
"_execution_output")):
267 tool_failed_output.write(
"\n")
268 tool_failed_output.write(
"\n")
269 tool_failed_output.write(
"\n")
270 tool_failed_output.close()
271 timing_violation_report = open(os.path.join(
272 directory,
"timing_violation_report"),
"w")
273 for subdir
in subdirs:
274 if os.path.exists(os.path.join(directory, subdir,
"timing_violation_report")):
275 timing_violation_report.write(
276 open(os.path.join(directory, subdir,
"timing_violation_report")).
read())
277 if os.path.exists(os.path.join(directory, subdir, args.tool +
"_execution_output")):
278 timing_violation_report.write(
"\n")
279 timing_violation_report.write(
"\n")
280 timing_violation_report.write(
"\n")
281 timing_violation_report.close()
282 report_file = open(os.path.join(directory,
"report"),
"w")
283 for subdir
in subdirs:
284 if os.path.exists(os.path.join(directory, subdir, args.tool +
"_return_value")):
285 return_value_file_name = os.path.join(
286 directory, subdir, args.tool +
"_return_value")
287 return_value_file = open(return_value_file_name)
288 return_value = return_value_file.read()
289 return_value_file.close()
290 args_file = open(os.path.join(directory, subdir,
"args"))
291 command_args = args_file.readlines()[0]
292 command_args = command_args.replace(abs_benchmarks_root +
"/",
"")
294 if return_value ==
"0":
295 tool_results_file_name = os.path.join(
296 directory, subdir, args.tool +
"_results")
297 if os.path.exists(tool_results_file_name):
299 "SUCCESS (" + open(tool_results_file_name).
read() +
" cycles) " + command_args.replace(
"\\",
""))
302 "SUCCESS: " + command_args.replace(
"\\",
""))
304 if return_value ==
"124":
305 report_file.write(
"FAILURE(Timeout): " +
306 command_args.replace(
"\\",
""))
309 "FAILURE: " + command_args.replace(
"\\",
""))
310 report_file.write(
"\n")
311 elif os.path.exists(os.path.join(directory, subdir,
"report")):
312 local_report_file = open(os.path.join(directory, subdir,
"report"))
313 report_file.write(local_report_file.read())
314 local_report_file.close()
316 if args.tool ==
"bambu":
318 named_list_name = os.path.join(abs_path,
"named_list")
319 lines = open(named_list_name).readlines()
322 if os.path.exists(os.path.join(local_dir, args.tool +
"_results_0.xml")):
323 input_files = input_files +
" " + \
324 os.path.join(local_dir, args.tool +
"_results_0.xml")
325 if len(input_files) > 0:
327 experimental_setup_file_name = os.path.join(
328 abs_path,
"experimental_setup.xml")
329 temp_list = open(experimental_setup_file_name,
"w")
330 bambu_version_file_name = os.path.join(abs_path,
"bambu_version")
331 bambu_version_file = open(bambu_version_file_name,
"w")
332 bambu_version_command = [tool_exe]
333 bambu_version_command.extend(shlex.split(
"--version"))
334 subprocess.call(bambu_version_command, stdout=bambu_version_file)
335 bambu_version_file.close()
336 bambu_version_file = open(bambu_version_file_name,
"r") 337 bambu_version = bambu_version_file.readlines()[-2].rstrip() 338 bambu_version_file.close() 339 if args.commonargs !=
None:
340 bambu_arguments =
' '.join(
' '.join(
map(str, l))
341 for l
in args.commonargs)
344 temp_list.write(
"<?xml version=\"1.0\"?>\n")
345 temp_list.write(
"<experimental_setup>\n")
346 temp_list.write(
" <bambu_version value=\"" +
347 bambu_version +
"\"/>\n")
348 temp_list.write(
" <timestamp value=\"" +
349 str(datetime.datetime.now()) +
"\"/>\n")
350 temp_list.write(
" <script value=\"" + args.script +
"\"/>\n")
351 temp_list.write(
" <bambu_arguments value=\"" +
352 bambu_arguments +
"\"/>\n")
353 temp_list.write(
" <benchmarks>\n")
354 reordered_list_name = os.path.join(abs_path,
"reordered_list")
355 reordered_list = open(reordered_list_name,
"r") 356 for line
in reordered_list.readlines():
357 temp_list.write(
" <benchmark value=\"" + line.rstrip().replace(
"\\\\\\-",
"-").replace(
"\\\\\\=",
"=").replace(
358 "\\\\\\_",
"_").replace(
"\\\\\\/",
"/").replace(
"\\\\\\\"",
""").replace(
"\\\\\\.",
".") +
"\"/>\n")
359 temp_list.write(
" </benchmarks>\n")
360 temp_list.write(
"</experimental_setup>\n")
362 local_args = input_files +
" " + experimental_setup_file_name
363 if os.path.exists(args.spider_style):
364 local_args = local_args +
" " + args.spider_style +
" " + table
366 local_args = local_args +
" " + \
367 os.path.join(os.path.dirname(os.path.realpath(__file__)),
"latex_table_format/",
368 args.spider_style) +
" " + table
370 logging.info(
" Creating tex " + spider)
371 local_command = [spider]
372 local_command.extend(shlex.split(local_args))
373 return_value = subprocess.call(local_command)
374 if len(input_files) > 0
and csv !=
None:
375 local_args = input_files +
" " + csv
377 logging.info(
" Creating csv " + spider)
378 local_command = [spider]
379 local_command.extend(shlex.split(local_args))
380 return_value = subprocess.call(local_command)
381 logging.info(
"Collected results of " + directory)
387 if os.path.exists(os.path.join(directory, args.tool +
"_return_value"))
or os.listdir(directory) == []:
389 subdirs = [s
for s
in sorted(os.listdir(directory))
if os.path.isdir(
390 os.path.join(directory, s))
and s !=
"panda-temp" and s !=
"HLS_output"]
391 print_testsuite =
False 392 for subdir
in subdirs:
393 if os.path.exists(os.path.join(directory, subdir, args.tool +
"_return_value")):
394 print_testsuite =
True 396 failed_counter_file_name = os.path.join(abs_path,
"failed_counter")
398 if os.path.exists(failed_counter_file_name):
399 failed_counter_file = open(failed_counter_file_name)
400 failed_counter = failed_counter_file.read()
402 if print_testsuite
and len(subdirs) > 0:
403 ju_file.write(
" <testsuite disabled=\"0\" errors=\"0\" failures=\""+failed_counter+
"\" name=\""+directory +
404 "\" tests=\"" +
str(len(subdirs)) +
"\" timestamp=\"" +
str(datetime.datetime.now()) +
"\">\n")
405 for subdir
in subdirs:
406 if os.path.exists(os.path.join(directory, subdir, args.tool +
"_return_value")):
407 return_value_file_name = os.path.join(
408 directory, subdir, args.tool +
"_return_value")
409 return_value_file = open(return_value_file_name)
410 return_value = return_value_file.read()
411 return_value_file.close()
412 args_file = open(os.path.join(directory, subdir,
"args"))
413 command_args = args_file.readlines()[0]
414 command_args = command_args.replace(abs_benchmarks_root +
"/",
"")
416 if return_value ==
"0":
417 ju_file.write(
" <testcase classname=\"PandA-bambu-tests\" name=\"" +
418 command_args.replace(
"\\",
"").replace(
"\"",
""") +
"\">\n")
420 if return_value ==
"124":
421 ju_file.write(
" <testcase classname=\"PandA-bambu-tests\" name=\"" +
422 command_args.replace(
"\\",
"").replace(
"\"",
""") +
"\">\n")
424 " <failure type=\"FAILURE(Timeout)\"></failure>\n")
425 ju_file.write(
" <system-out>\n")
426 ju_file.write(
"<![CDATA[\n")
428 ju_file.write(
" <testcase classname=\"PandA-bambu-tests\" name=\"" +
429 command_args.replace(
"\\",
"").replace(
"\"",
""") +
"\">\n")
431 " <failure type=\"FAILURE\"></failure>\n")
432 ju_file.write(
" <system-out>\n")
433 ju_file.write(
"<![CDATA[\n")
434 with open(os.path.join(directory, args.tool +
"_failed_output"))
as f:
435 for line
in deque(f, maxlen=args.junit_logtail):
437 ju_file.write(
"]]>\n")
438 ju_file.write(
" </system-out>\n")
439 ju_file.write(
" </testcase>\n")
440 if print_testsuite
and len(subdirs) > 0:
441 ju_file.write(
" </testsuite>\n")
447 if os.path.exists(os.path.join(directory, args.tool +
"_return_value"))
or os.listdir(directory) == []:
449 subdirs = [s
for s
in sorted(os.listdir(directory))
if os.path.isdir(
450 os.path.join(directory, s))
and s !=
"panda-temp" and s !=
"HLS_output"]
451 for subdir
in subdirs:
454 for subdir
in subdirs:
455 if os.path.exists(os.path.join(directory, subdir, args.tool +
"_return_value")):
456 pp_file.write(
" <test name=\"" +
str(directory) +
457 "/" +
str(subdir) +
"\" executed=\"yes\">\n")
458 pp_file.write(
" <result>\n")
459 return_value_file_name = os.path.join(
460 directory, subdir, args.tool +
"_return_value")
461 return_value_file = open(return_value_file_name)
462 return_value = return_value_file.read()
463 return_value_file.close()
464 args_file = open(os.path.join(directory, subdir,
"args"))
465 command_args = args_file.readlines()[0]
466 command_args = command_args.replace(abs_benchmarks_root +
"/",
"")
468 if return_value ==
"0":
470 " <success passed=\"yes\" state=\"100\" hasTimedOut=\"false\"/>\n")
481 HLS_execution_time_tag =
"" 482 if os.path.exists(os.path.join(directory, subdir, args.tool +
"_results_0.xml")):
483 xml_document = minidom.parse(os.path.join(
484 directory, subdir, args.tool +
"_results_0.xml"))
485 if len(xml_document.getElementsByTagName(
"CYCLES")) > 0:
486 cycles_tag =
str(xml_document.getElementsByTagName(
487 "CYCLES")[0].attributes[
"value"].value)
488 if len(xml_document.getElementsByTagName(
"AREAxTIME")) > 0:
489 areatime_tag =
str(xml_document.getElementsByTagName(
490 "AREAxTIME")[0].attributes[
"value"].value)
491 if len(xml_document.getElementsByTagName(
"SLICE")) > 0:
492 slice_tag =
str(xml_document.getElementsByTagName(
493 "SLICE")[0].attributes[
"value"].value)
494 if len(xml_document.getElementsByTagName(
"SLICE_LUTS")) > 0:
495 sliceluts_tag =
str(xml_document.getElementsByTagName(
496 "SLICE_LUTS")[0].attributes[
"value"].value)
497 if len(xml_document.getElementsByTagName(
"REGISTERS")) > 0:
498 registers_tag =
str(xml_document.getElementsByTagName(
499 "REGISTERS")[0].attributes[
"value"].value)
500 if len(xml_document.getElementsByTagName(
"DSPS")) > 0:
501 dsps_tag =
str(xml_document.getElementsByTagName(
502 "DSPS")[0].attributes[
"value"].value)
503 if len(xml_document.getElementsByTagName(
"BRAMS")) > 0:
504 brams_tag =
str(xml_document.getElementsByTagName(
505 "BRAMS")[0].attributes[
"value"].value)
506 if len(xml_document.getElementsByTagName(
"PERIOD")) > 0:
507 period_tag =
str(xml_document.getElementsByTagName(
508 "PERIOD")[0].attributes[
"value"].value)
509 if len(xml_document.getElementsByTagName(
"CLOCK_SLACK")) > 0:
510 slack_tag =
str(xml_document.getElementsByTagName(
511 "CLOCK_SLACK")[0].attributes[
"value"].value)
512 if len(xml_document.getElementsByTagName(
"FREQUENCY")) > 0:
513 frequency_tag =
str(xml_document.getElementsByTagName(
514 "FREQUENCY")[0].attributes[
"value"].value)
515 if len(xml_document.getElementsByTagName(
"HLS_execution_time")) > 0:
516 HLS_execution_time_tag =
str(xml_document.getElementsByTagName(
517 "HLS_execution_time")[0].attributes[
"value"].value)
519 if cycles_tag !=
"" and cycles_tag !=
"inf":
520 pp_file.write(
" <performance unit=\"cycles\" mesure=\"" +
521 cycles_tag +
"\" isRelevant=\"true\"/>\n")
522 if HLS_execution_time_tag !=
"":
523 pp_file.write(
" <executiontime unit=\"s\" mesure=\"" +
524 HLS_execution_time_tag +
"\" isRelevant=\"true\"/>\n")
526 if (areatime_tag !=
"" and areatime_tag !=
"inf")
or (slice_tag !=
"" and slice_tag !=
"inf")
or (sliceluts_tag !=
"" and sliceluts_tag !=
"inf")
or (registers_tag !=
"" and registers_tag !=
"inf")
or (dsps_tag !=
"" and dsps_tag !=
"inf")
or (brams_tag !=
"" and brams_tag !=
"inf")
or (period_tag !=
"" and period_tag !=
"inf")
or (slack_tag !=
"" and slack_tag !=
"inf")
or (frequency_tag !=
"" and frequency_tag !=
"inf"):
527 pp_file.write(
" <metrics>\n")
528 if areatime_tag !=
"" and areatime_tag !=
"inf":
529 pp_file.write(
" <areatime unit=\"lutxns\" mesure=\"" +
530 areatime_tag +
"\" isRelevant=\"true\"/>\n")
531 if slice_tag !=
"" and slice_tag !=
"inf":
532 pp_file.write(
" <slices unit=\"ns\" mesure=\"" +
533 slice_tag +
"\" isRelevant=\"true\"/>\n")
534 if sliceluts_tag !=
"" and sliceluts_tag !=
"inf":
535 pp_file.write(
" <sliceluts unit=\"slice\" mesure=\"" +
536 sliceluts_tag +
"\" isRelevant=\"true\"/>\n")
537 if registers_tag !=
"" and registers_tag !=
"inf":
538 pp_file.write(
" <registers unit=\"registers\" mesure=\"" +
539 registers_tag +
"\" isRelevant=\"true\"/>\n")
540 if dsps_tag !=
"" and dsps_tag !=
"inf":
542 " <dsps unit=\"dsp\" mesure=\"" + dsps_tag +
"\" isRelevant=\"true\"/>\n")
543 if brams_tag !=
"" and brams_tag !=
"inf":
544 pp_file.write(
" <brams unit=\"bram\" mesure=\"" +
545 brams_tag +
"\" isRelevant=\"true\"/>\n")
546 if period_tag !=
"" and period_tag !=
"inf":
547 pp_file.write(
" <period unit=\"ns\" mesure=\"" +
548 period_tag +
"\" isRelevant=\"true\"/>\n")
549 if slack_tag !=
"" and slack_tag !=
"inf":
550 pp_file.write(
" <slack unit=\"ns\" mesure=\"" +
551 slack_tag +
"\" isRelevant=\"true\"/>\n")
552 if frequency_tag !=
"" and frequency_tag !=
"inf":
553 pp_file.write(
" <frequency unit=\"MHz\" mesure=\"" +
554 frequency_tag +
"\" isRelevant=\"true\"/>\n")
555 pp_file.write(
" </metrics>\n")
557 if return_value ==
"124":
559 " <success passed=\"no\" state=\"0\" hasTimedOut=\"true\"/>\n")
562 " <success passed=\"no\" state=\"0\" hasTimedOut=\"false\"/>\n")
563 pp_file.write(
" </result>\n")
564 pp_file.write(
" </test>\n")
567 parser = argparse.ArgumentParser(
568 description=
"Performs panda tests", fromfile_prefix_chars=
'@')
570 "files", help=
"The files to be tested: they can be configuration files, directories containing benchmarks or source code files.", nargs=
'*', action=
"append")
571 parser.add_argument(
'-l',
"--benchmarks_list",
572 help=
"The file containing the list of tests to be performed", nargs=
'*', action=
"append")
573 parser.add_argument(
'-b',
"--benchmarks_root",
574 help=
"The directory containing benchmarks")
576 '-o',
"--output", help=
"The directory where output files we be put (default=\"output\")", default=
"output")
577 parser.add_argument(
'-j', help=
"The number of jobs which execute the benchmarks (default=\"1\")",
578 default=int(os.getenv(
'J',
"1")), type=positive_integer, action=StoreOrUpdateMin)
580 "--bambu", help=
"The bambu executable (default=bambu)", default=
"bambu")
581 parser.add_argument(
"--spider", help=
"The spider executable (default=spider)",
583 parser.add_argument(
"--spider-style", help=
"The spider table style (default=latex_format_bambu_results.xml)",
584 default=
"latex_format_bambu_results.xml")
586 "--zebu", help=
"The zebu executable (default=zebu)", default=
"zebu")
588 '-t',
"--timeout", help=
"Timeout for tool execution (default=75m)", default=
"75m")
590 '-a',
"--args", help=
"A set of arguments to be passed to the tool", nargs=
'*', action=
'append')
591 parser.add_argument(
'-c',
"--commonargs",
592 help=
"A set of arguments to be passed to the tool", nargs=
'*', action=
'append')
594 "--table", help=
"Print the results in tex format", default=
"results.tex")
595 parser.add_argument(
"--csv", help=
"Print the results in csv format")
596 parser.add_argument(
"--tool", help=
"The tool to be tested", default=
"bambu")
597 parser.add_argument(
"--ulimit", help=
"The ulimit options",
598 default=
"-f 8388608 -s 32768 -v 16777216")
599 parser.add_argument(
"--stop", help=
"Stop the execution on first error (default=false)",
600 default=
False, action=
"store_true")
601 parser.add_argument(
"--returnfail", help=
"Return FAILURE in case at least one test fails (default=false)",
602 default=
False, action=
"store_true")
603 parser.add_argument(
"--mail", help=
"Send a mail with the result")
605 "--name", help=
"Set the name of this regression (default=Bambu regression)", nargs=
'*', action=
'append')
606 parser.add_argument(
"--no-clean", help=
"Do not clean produced files",
607 default=
False, action=
"store_true")
608 parser.add_argument(
"--restart", help=
"Restart last execution (default=false)",
609 default=
False, action=
"store_true")
611 "--script", help=
"Set the bash script in the generated tex", default=
"")
612 parser.add_argument(
"--junitdir", help=
"Set the JUnit directory", default=
"")
614 "--junit-logtail", help=
"Set the JUnit log tail length for failed tests (default=300)", default=300, type=positive_integer)
615 parser.add_argument(
"--perfpublisherdir",
616 help=
"Set the PerfPublisher directory", default=
"")
618 args = parser.parse_args()
620 logging.basicConfig(level=logging.INFO, format=
'%(levelname)s: %(message)s')
623 abs_script = os.path.abspath(sys.argv[0])
627 modified_argv.append(sys.argv[0])
628 abs_configuration_dir =
"" 629 for arg
in sys.argv[1:]:
630 if arg
in args.files[0]:
632 if (arg[-2:] ==
".c")
or (arg[-2:] ==
".C")
or (arg[-4:] ==
".CPP")
or (arg[-4:] ==
".cpp")
or (arg[-4:] ==
".cxx")
or (arg[-3:] ==
".cc")
or (arg[-4:] ==
".c++"):
633 modified_argv.append(arg)
635 elif os.path.exists(arg)
and os.path.isdir(arg):
636 modified_argv.append(arg)
637 elif args.benchmarks_root !=
None and os.path.exists(os.path.join(os.path.abspath(args.benchmarks_root), arg))
and os.path.isdir(os.path.join(os.path.abspath(args.benchmarks_root), arg)):
638 modified_argv.append(arg)
639 elif os.path.exists(os.path.join(os.path.dirname(abs_script), arg))
and os.path.isdir(os.path.join(os.path.dirname(abs_script), arg)):
640 modified_argv.append(arg)
642 modified_argv.append(
"@" + arg)
644 modified_argv.append(arg)
645 args = parser.parse_args(modified_argv)
648 abs_script = os.path.abspath(sys.argv[0])
651 table = os.path.abspath(args.table)
655 csv = os.path.abspath(args.csv)
660 if os.path.exists(args.output)
and not args.restart:
661 logging.error(
"Output directory " + args.output +
662 " already exists. Please remove it or specify a different one with -o")
666 if args.junitdir !=
"" and not os.path.exists(args.junitdir):
667 os.mkdir(args.junitdir)
669 if args.perfpublisherdir !=
"" and not os.path.exists(args.perfpublisherdir):
670 os.mkdir(args.perfpublisherdir)
673 if args.junitdir !=
"":
675 junit_file_name = os.path.abspath(os.path.join(
676 args.junitdir,
"Junit_report"+
str(junit_index)+
".xml"))
677 while os.path.isfile(junit_file_name):
678 junit_index = junit_index + 1
679 junit_file_name = os.path.abspath(os.path.join(
680 args.junitdir,
"Junit_report"+
str(junit_index)+
".xml"))
682 perfpublisher_name =
"" 683 perfpublisher_file_name =
"" 684 if args.perfpublisherdir !=
"":
685 perfpublisher_index = 0
686 perfpublisher_name =
"PerfPublisher_report"+
str(perfpublisher_index)
687 perfpublisher_file_name = os.path.abspath(os.path.join(
688 args.perfpublisherdir, perfpublisher_name+
".xml"))
689 while os.path.isfile(perfpublisher_file_name):
690 perfpublisher_index = perfpublisher_index + 1
691 perfpublisher_name =
"PerfPublisher_report"+
str(perfpublisher_index)
692 perfpublisher_file_name = os.path.abspath(os.path.join(
693 args.perfpublisherdir, perfpublisher_name+
".xml"))
696 abs_path = os.path.abspath(args.output)
699 if not os.path.exists(abs_path):
708 failed_counter_file_name = os.path.join(abs_path,
"failed_counter")
709 if os.path.exists(failed_counter_file_name):
710 failed_counter_file = open(failed_counter_file_name)
711 failed_counter = failed_counter_file.read()
712 if failed_counter ==
"0" and args.junitdir ==
"" and args.perfpublisherdir ==
"":
713 logging.info(
"Already pass")
718 if args.tool ==
"bambu":
719 if os.path.isfile(args.bambu)
and os.access(args.bambu, os.X_OK):
720 tool_exe = args.bambu
723 for path
in os.environ[
"PATH"].split(os.pathsep):
724 exe_file = os.path.join(path,
"bambu")
725 if os.path.isfile(exe_file)
and os.access(exe_file, os.X_OK):
728 if args.bambu !=
"opt/panda/bin/bambu":
729 if not os.path.isfile(args.bambu):
730 logging.error(args.bambu +
" does not exist")
732 logging.error(args.bambu +
" is not an executable")
734 logging.error(
"bambu not found")
736 logging.info(
"Bambu found: " + tool_exe)
737 elif args.tool ==
"zebu":
738 if os.path.isfile(args.zebu)
and os.access(args.zebu, os.X_OK):
742 for path
in os.environ[
"PATH"].split(os.pathsep):
743 exe_file = os.path.join(path,
"zebu")
744 if os.path.isfile(exe_file)
and os.access(exe_file, os.X_OK):
747 if args.zebu !=
"opt/panda/bin/zebu":
748 if not os.path.isfile(args.zebu):
749 logging.error(args.zebu +
" does not exist")
751 logging.error(args.zebu +
" is not an executable")
753 logging.error(
"zebu not found")
755 logging.info(
"Zebu found: " + tool_exe)
758 if distutils.spawn.find_executable(tool_exe) ==
None:
759 logging.error(tool_exe +
" not found")
762 if args.benchmarks_root
is None:
763 abs_benchmarks_root = abs_configuration_dir
765 if os.path.isabs(args.benchmarks_root):
766 abs_benchmarks_root = os.path.abspath(args.benchmarks_root)
768 if os.path.exists(os.path.join(os.path.abspath(
".."), args.benchmarks_root)):
769 abs_benchmarks_root = os.path.join(
770 os.path.abspath(
".."), args.benchmarks_root)
772 if os.path.exists(os.path.join(os.path.abspath(os.path.join(os.path.dirname(abs_script),
"../../..")), args.benchmarks_root)):
773 abs_benchmarks_root = os.path.join(os.path.abspath(os.path.join(
774 os.path.dirname(abs_script),
"../../..")), args.benchmarks_root)
776 logging.error(args.benchmarks_root +
" not found")
781 if os.path.isfile(args.spider)
and os.access(args.spider, os.X_OK):
785 for path
in os.environ[
"PATH"].split(os.pathsep):
786 exe_file = os.path.join(path,
"spider")
787 if os.path.isfile(exe_file)
and os.access(exe_file, os.X_OK):
790 if args.spider !=
"opt/panda/bin/spider":
791 if not os.path.isfile(args.spider):
792 logging.error(args.spider +
" does not exist")
794 logging.error(args.spider +
" is not an executable")
796 logging.error(
"spider not found")
799 if args.benchmarks_root
is None:
800 abs_benchmarks_root = abs_configuration_dir
802 if os.path.isabs(args.benchmarks_root):
803 abs_benchmarks_root = os.path.abspath(args.benchmarks_root)
805 if os.path.exists(os.path.join(os.path.abspath(
".."), args.benchmarks_root)):
806 abs_benchmarks_root = os.path.join(
807 os.path.abspath(
".."), args.benchmarks_root)
809 if os.path.exists(os.path.join(os.path.abspath(os.path.join(os.path.dirname(abs_script),
"../../..")), args.benchmarks_root)):
810 abs_benchmarks_root = os.path.join(os.path.abspath(os.path.join(
811 os.path.dirname(abs_script),
"../../..")), args.benchmarks_root)
813 logging.error(args.benchmarks_root +
" not found")
816 logging.info(
"Spider found: " + spider)
819 if args.mail !=
None:
821 for path
in os.environ[
"PATH"].split(os.pathsep):
822 exe_file = os.path.join(path,
"mutt")
823 if os.path.isfile(exe_file)
and os.access(exe_file, os.X_OK):
826 logging.error(
"mutt not found")
833 if args.benchmarks_list !=
None:
834 for relative_list
in args.benchmarks_list:
836 if os.path.exists(os.path.abspath(
"../" + relative_list[0])):
837 abs_lists.append(os.path.abspath(
"../" + relative_list[0]))
839 elif os.path.exists(os.path.join(os.path.dirname(abs_script), relative_list[0])):
840 abs_lists.append(os.path.join(
841 os.path.dirname(abs_script), relative_list[0]))
843 elif os.path.exists(os.path.join(abs_configuration_dir, relative_list[0])):
844 abs_lists.append(os.path.join(
845 abs_configuration_dir, relative_list[0]))
847 elif os.path.exists(os.path.join(abs_benchmarks_root, relative_list[0])):
848 abs_lists.append(os.path.join(
849 abs_benchmarks_root, relative_list[0]))
851 logging.error(relative_list[0] +
" does not exist")
855 for arg
in args.files[0]:
857 if (arg[-2:] ==
".c")
or (arg[-2:] ==
".C")
or (arg[-4:] ==
".CPP")
or (arg[-4:] ==
".cpp")
or (arg[-4:] ==
".cxx")
or (arg[-3:] ==
".cc")
or (arg[-4:] ==
".c++"):
858 files_list.append(arg)
860 elif os.path.exists(arg)
and os.path.isdir(arg):
861 files_list.append(arg)
862 elif os.path.exists(os.path.join(os.path.dirname(abs_script), arg))
and os.path.isdir(os.path.join(os.path.dirname(abs_script), arg)):
863 files_list.append(arg)
864 elif os.path.exists(os.path.join(abs_benchmarks_root, arg))
and os.path.isdir(os.path.join(abs_benchmarks_root, arg)):
865 files_list.append(os.path.join(abs_benchmarks_root, arg))
867 if args.benchmarks_list ==
None and len(files_list) == 0
and (args.tool ==
"bambu" or args.tool ==
"zebu"):
868 logging.error(
"Benchmarks not found")
871 if len(files_list) > 0:
872 temp_list = open(os.path.join(abs_path,
"temp_list"),
"w")
873 for element
in files_list:
874 temp_list.write(element)
876 abs_lists.append(os.path.join(abs_path,
"temp_list"))
879 reordered_list_name = os.path.join(abs_path,
"reordered_list")
880 reordered_list = open(reordered_list_name,
"w")
882 logging.info(
"Preparing benchmark list")
883 logging.info(
" Reordering arguments")
884 for abs_list
in abs_lists:
885 list_file = open(abs_list)
886 lines = list_file.readlines()
889 if line.strip() ==
"":
893 if args.tool !=
"bambu" and args.tool !=
"zebu":
894 reordered_list.write(line)
896 tokens = shlex.split(line)
898 escape_pattern =
"(-|_|\{|\}|=|!|\$|#|&|\"|\'|\(|\)|\||<|>|`|\\\|;|\.|,)" 904 re.sub(escape_pattern,
r"\\\1", token))
905 if token.find(
"--param") != -1:
910 if follow_param ==
True:
912 re.sub(escape_pattern,
r"\\\1", token))
914 reordered_list.write(token +
" ")
916 for parameter
in parameters:
917 reordered_list.write(
918 re.sub(escape_pattern,
r"\\\1", parameter) +
" ")
919 reordered_list.write(
"\n")
920 reordered_list.close()
923 expanded_list_name = os.path.join(abs_path,
"expanded_list")
924 expanded_list = open(expanded_list_name,
"w")
926 logging.info(
" Expanding directory")
927 lines = open(reordered_list_name).readlines()
929 if line.strip() ==
"":
931 tokens = shlex.split(line)
932 if args.tool ==
"bambu" or args.tool ==
"zebu":
933 if(tokens[0][0] !=
'/'):
934 first_parameter = os.path.join(abs_benchmarks_root, tokens[0])
936 first_parameter = tokens[0]
938 first_parameter = tokens[0].replace(
939 "BENCHMARKS\_ROOT", abs_benchmarks_root)
940 other_parameters = tokens[1:len(tokens)]
941 if not os.path.exists(first_parameter)
and (args.tool ==
"bambu" or args.tool ==
"zebu"):
942 logging.error(first_parameter +
" does not exist")
945 if os.path.isdir(first_parameter):
946 logging.info(
" " + tokens[0])
948 c_files = sorted(c_files)
949 for c_file
in c_files:
950 expanded_list.write(c_file)
951 for other_parameter
in other_parameters:
953 " " + other_parameter.replace(
"BENCHMARKS\_ROOT", abs_benchmarks_root))
954 expanded_list.write(
"\n")
956 expanded_list.write(first_parameter)
957 for other_parameter
in other_parameters:
958 if ((other_parameter[-2:] ==
".c")
or (other_parameter[-2:] ==
".C")
or (other_parameter[-4:] ==
".CPP")
or (other_parameter[-4:] ==
".cpp")
or (other_parameter[-4:] ==
".cxx")
or (other_parameter[-3:] ==
".cc")
or (other_parameter[-4:] ==
".c++")
or other_parameter[-4:] ==
".xml")
and other_parameter[0] !=
'\\':
959 if other_parameter[0] ==
'/':
960 expanded_list.write(
" " + other_parameter)
963 " " + os.path.join(abs_benchmarks_root, other_parameter))
965 expanded_list.write(
" " + other_parameter.replace(
"BENCHMARKS\_ROOT",
966 abs_benchmarks_root).replace(
"BENCHMARKS_ROOT", abs_benchmarks_root))
967 expanded_list.write(
"\n")
968 expanded_list.close()
971 logging.info(
" Considering all tool arguments")
972 arg_lists = args.args
975 arged_list_name = os.path.join(abs_path,
"arged_list")
976 arged_list = open(arged_list_name,
"w")
977 lines = open(expanded_list_name).readlines()
978 for arg_list
in arg_lists:
980 arged_list.write(line.rstrip())
981 if len(arg_list) > 0:
985 arged_list.write(
" " + arg)
986 if args.commonargs !=
None and len(args.commonargs) > 0:
987 for commonarg
in args.commonargs:
988 arged_list.write(
" " + commonarg[0].replace(
"#",
" "))
989 arged_list.write(
"\n")
996 logging.info(
" Adding benchmark name")
997 named_list_name = os.path.join(abs_path,
"named_list")
998 named_list = open(named_list_name,
"w")
999 lines = open(arged_list_name).readlines()
1001 named_list.write(line.rstrip())
1003 configuration_name =
"" 1005 tokens = shlex.split(line)
1006 for token
in tokens:
1007 if token.find(
"--configuration-name") != -1:
1008 configuration_name = token[len(
"--configuration-name="):]
1009 if token.find(
"--benchmark-name") != -1:
1010 benchmark_name = token[len(
"--benchmark-name="):]
1011 if benchmark_name ==
"":
1012 if args.tool !=
"bambu" and args.tool !=
"zebu":
1013 logging.error(
"Missing benchmark name")
1015 benchmark_name = os.path.basename(line.split()[0])[:-2]
1016 named_list.write(
" --benchmark-name=" + benchmark_name)
1017 full_name = configuration_name +
":" + benchmark_name
1018 logging.info(
" " + full_name)
1019 if full_name
in full_names:
1021 "Duplicated configuration name - benchmark name: " + full_name)
1023 full_names.add(full_name)
1024 named_list.write(
"\n")
1028 logging.info(
" Generating output directories")
1029 lines = open(named_list_name).readlines()
1032 logging.info(
" Creating directory " + new_dir)
1033 os.makedirs(new_dir)
1035 logging.info(
" Skipping generation of lists and directories")
1036 named_list_name = os.path.join(abs_path,
"named_list")
1037 if not os.path.exists(named_list_name):
1038 logging.error(
"List of previous run not found")
1042 logging.info(
" Launching tool")
1043 lock = threading.RLock()
1044 lock_creation_destruction = threading.RLock()
1045 passed_benchmark = 0
1048 children = [
None] * n_jobs
1049 for thread_index
in range(n_jobs):
1050 threads.insert(thread_index, threading.Thread(
1051 target=execute_tests, args=(named_list_name, thread_index)))
1052 threads[thread_index].daemon =
True 1053 threads[thread_index].start()
1057 for thread_index
in range(n_jobs):
1058 while threads[thread_index].is_alive():
1059 threads[thread_index].join(100)
1060 except KeyboardInterrupt:
1061 logging.error(
"SIGINT received")
1063 for local_thread_index
in range(n_jobs):
1064 if children[local_thread_index] !=
None:
1065 if children[local_thread_index].poll() ==
None:
1076 if args.junitdir !=
"":
1077 junit_file = open(junit_file_name,
"w")
1078 junit_file.write(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
1079 junit_file.write(
"<testsuites disabled=\"0\" errors=\"0\" failures=\""+
str(total_benchmark -
1080 passed_benchmark)+
"\" name=\"" + abs_path +
"\" tests=\"" +
str(total_benchmark) +
"\">\n")
1082 junit_file.write(
"</testsuites>\n")
1085 if args.perfpublisherdir !=
"":
1086 perfpublisher_file = open(perfpublisher_file_name,
"w")
1087 perfpublisher_file.write(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
1088 perfpublisher_file.write(
1089 "<report name=\"" + perfpublisher_name +
"\" categ=\"PandA-Report\">\n")
1091 perfpublisher_file.write(
"</report>\n")
1094 if args.tool ==
"bambu" or args.tool ==
"zebu":
1095 report_file_name = os.path.join(abs_path,
"report")
1096 report_file = open(report_file_name)
1097 lines = report_file.readlines()
1099 report_file = open(report_file_name,
"w")
1100 command = [tool_exe,
"--version"]
1101 subprocess.call(command, stderr=report_file, stdout=report_file)
1102 report_file.write(
"SYSTEM INFORMATION:\n")
1104 command = [
"lsb_release",
"-a"]
1105 subprocess.call(command, stderr=report_file, stdout=report_file)
1106 report_file.write(
"\n")
1107 report_file.write(
"CURRENT TIME:\n")
1108 report_file.write(
str(datetime.datetime.now()) +
"\n\n")
1109 report_file.write(
"PASSED TESTS:\n")
1110 report_file.write(
str(passed_benchmark) +
"/" +
1111 str(total_benchmark) +
"\n\n")
1113 failed_counter_file_name = os.path.join(abs_path,
"failed_counter")
1114 failed_counter_file = open(failed_counter_file_name,
"w")
1115 failed_counter_file.write(
str(total_benchmark - passed_benchmark))
1116 failed_counter_file.close()
1119 report_file.write(line)
1122 if args.mail !=
None:
1130 outcome =
str(passed_benchmark) +
"/" +
str(total_benchmark)
1133 for name
in args.name:
1134 full_name = full_name + name[0]
1137 local_command =
"cat " + report_file_name +
" | mutt -s \"" + \
1138 full_name +
": " + outcome +
"\" " + args.mail
1139 subprocess.call(local_command, shell=
True)
def CreateJunitBody(directory, ju_file)
def SearchCFiles(directory)
unsigned map[NUM_VERTICES]
def GetChildren(parent_pid)
def __call__(self, parser, namespace, values, option_string=None)
def CreatePerfPublisherBody(directory, pp_file)
def CollectResults(directory)
def ComputeDirectory(line)
short int read(short int *data)
def execute_tests(named_list, thread_index)