test_panda.py

Go to the documentation of this file.

#!python3

import argparse
import datetime
import distutils.spawn
import logging
import os
import re
import shlex
import shutil
import signal
import subprocess
import sys
import threading
try:
    from defusedxml import minidom
except ImportError:
    print>>sys.stderr, 'WARNING: pyhon-defusedxml not available, falling back to unsafe standard libraries...'
    from xml.dom import minidom

from collections import deque

line_index = 0
failure = False


def positive_integer(value):
    pos_int = int(value)
    if pos_int <= 0:
        raise argparse.ArgumentTypeError(
            "%s must be a positive integer" % value)
    return pos_int


class StoreOrUpdateMin(argparse.Action):
    first_parsed = True

    def __call__(self, parser, namespace, values, option_string=None):
        if self.first_parsed == True:
            self.first_parsed = False
            setattr(namespace, self.dest, values)
        else:
            setattr(namespace, self.dest, min(namespace.j, values))


# Return children of a process
def GetChildren(parent_pid):
    ret = set()
    ps_command = subprocess.Popen(
        "ps -o pid --ppid %d --noheaders" % parent_pid, shell=True, stdout=subprocess.PIPE)
    ps_output = ps_command.stdout.read()
    ps_command.wait()
    for pid_str in ps_output.split("\n")[:-1]:
        ret.add(int(pid_str))
    return ret

# Kill a process than kill its children


def kill_proc_tree(pid):
    children = GetChildren(pid)
    os.kill(pid, signal.SIGKILL)
    for child in children:
        kill_proc_tree(child)
# Process benchmark in list


def execute_tests(named_list, thread_index):
    global passed_benchmark
    global total_benchmark
    global line_index
    global children
    global failure
    lines = open(named_list).readlines()
    with lock:
        local_index = line_index
        line_index += 1
    while local_index < len(lines) and not (failure and args.stop):
        cwd = ComputeDirectory(lines[local_index])
        failed_output_file_name = os.path.join(
            cwd, args.tool + "_failed_output")
        if os.path.exists(failed_output_file_name):
            os.remove(failed_output_file_name)
        timing_violation_report_file_name = os.path.join(
            cwd, "HLS_output/Synthesis/timing_violation_report")
        if os.path.exists(timing_violation_report_file_name):
            os.remove(timing_violation_report_file_name)
        tool_return_value_file_name = os.path.join(
            cwd, args.tool + "_return_value")
        if args.restart and os.path.exists(os.path.join(cwd, args.tool + "_return_value")):
            tool_return_value_file = open(tool_return_value_file_name, "r")
            return_value = tool_return_value_file.read()
            tool_return_value_file.close()
            if return_value == "0":
                with lock:
                    total_benchmark += 1
                    passed_benchmark += 1
                    logging.info("   SKIPPING --- OVERALL: " + str(passed_benchmark) + " passed, " + str(total_benchmark-passed_benchmark) +
                                 " failed, " + str(len(lines)-total_benchmark) + " queued --- " + lines[local_index].replace("\\", ""))
                    local_index = line_index
                    line_index += 1
                continue
        HLS_output_directory = os.path.join(cwd, "HLS_output")
        if os.path.exists(HLS_output_directory):
            shutil.rmtree(HLS_output_directory)
        output_file_name = os.path.join(cwd, args.tool + "_execution_output")
        output_file = open(output_file_name, "w")
        local_args = lines[local_index]
        if local_args[0] == "\"":
            local_args = local_args[1:-1]
        if args.tool != "bambu" and args.tool != "zebu":
            tokens = shlex.split(lines[local_index])
            args_without_benchmark_name = ""
            for token in tokens:
                if token.find("--benchmark-name") == -1:
                    args_without_benchmark_name += token + " "
            local_args = args_without_benchmark_name
        local_command = "ulimit " + args.ulimit + \
            "; exec timeout " + args.timeout + " " + tool_exe
        local_command = local_command + " " + local_args
        output_file.write("#" * 80 + "\n")
        output_file.write("cd " + cwd + "; ")
        output_file.write(local_command + "\n")
        output_file.write("#" * 80 + "\n")
        output_file.flush()
        return_value = -1
        with lock_creation_destruction:
            if not (failure and args.stop):
                children[thread_index] = subprocess.Popen(
                    local_command, stderr=output_file, stdout=output_file, cwd=cwd, shell=True, executable="/bin/bash")
        try:
            return_value = children[thread_index].wait()
        except:
            pass
        with lock_creation_destruction:
            if return_value != 0 and (args.stop or args.returnfail):
                failure = True
            if failure and args.stop:
                for local_thread_index in range(n_jobs):
                    if children[local_thread_index] != None:
                        if children[local_thread_index].poll() == None:
                            try:
                                kill_proc_tree(
                                    children[local_thread_index].pid)
                            except OSError:
                                pass
        os.fsync(output_file.fileno())
        output_file.close()
        tool_return_value_file = open(tool_return_value_file_name, "w")
        tool_return_value_file.write(str(return_value))
        tool_return_value_file.close()
        args_file = open(os.path.join(cwd, "args"), "w")
        args_file.write(lines[local_index])
        args_file.close()
        if return_value == 0 and os.path.exists(os.path.join(cwd, args.tool + "_results_0.xml")):
            tool_results_file_name = os.path.join(cwd, args.tool + "_results")
            tool_results_file = open(tool_results_file_name, "w")
            tool_results_string = ""
            xml_document = minidom.parse(
                os.path.join(cwd, args.tool + "_results_0.xml"))
            if len(xml_document.getElementsByTagName("CYCLES")) > 0:
                cycles_tag = xml_document.getElementsByTagName("CYCLES")[0]
                tool_results_string = tool_results_string + \
                    cycles_tag.attributes["value"].value + " CYCLES"
            if len(xml_document.getElementsByTagName("CLOCK_SLACK")) > 0:
                slack_tag = xml_document.getElementsByTagName("CLOCK_SLACK")[0]
                tool_results_string = tool_results_string + " *** " + \
                    slack_tag.attributes["value"].value + "ns"
            tool_results_file.write(tool_results_string)
            tool_results_file.close()
        if not (failure and args.stop) or (return_value != -9 and return_value != 0):
            if return_value != 0:
                os.symlink(os.path.basename(output_file_name), str(os.path.join(
                    os.path.dirname(output_file_name), args.tool + "_failed_output")))
            if os.path.exists(timing_violation_report_file_name):
                annotated_timing_violation_report_file = open(
                    os.path.join(cwd, "timing_violation_report"), "w")
                annotated_timing_violation_report_file.write("#" * 80 + "\n")
                annotated_timing_violation_report_file.write(
                    "cd " + cwd + "; ")
                annotated_timing_violation_report_file.write(
                    local_command + "\n")
                annotated_timing_violation_report_file.write("#" * 80 + "\n")
                annotated_timing_violation_report_file.flush()
                timing_violation_report_file = open(
                    timing_violation_report_file_name)
                annotated_timing_violation_report_file.write(
                    timing_violation_report_file.read())
                timing_violation_report_file.close()
                annotated_timing_violation_report_file.close()
            with lock:
                total_benchmark += 1
                if return_value == 0:
                    passed_benchmark += 1
                    if not args.no_clean:
                        for sub in os.listdir(cwd):
                            if os.path.isdir(os.path.join(cwd, sub)):
                                shutil.rmtree(os.path.join(cwd, sub))
                            else:
                                if sub != args.tool + "_return_value" and sub != args.tool + "_execution_output" and sub != args.tool + "_results_0.xml" and sub != "args":
                                    os.remove(os.path.join(cwd, sub))
                    if os.path.exists(os.path.join(cwd, args.tool + "_results_0.xml")):
                        logging.info("   SUCCESS (" + tool_results_string + ") --- OVERALL: " + str(passed_benchmark) + " passed, " + str(
                            total_benchmark-passed_benchmark) + " failed, " + str(len(lines)-total_benchmark) + " queued --- " + lines[local_index].replace("\\", ""))
                    else:
                        logging.info("   SUCCESS --- OVERALL: " + str(passed_benchmark) + " passed, " + str(total_benchmark-passed_benchmark) +
                                     " failed, " + str(len(lines)-total_benchmark) + " queued --- " + lines[local_index].replace("\\", ""))
                elif return_value == 124:
                    logging.info("   FAILURE (Timeout) --- OVERALL: " + str(passed_benchmark) + " passed, " + str(total_benchmark -
                                 passed_benchmark) + " failed, " + str(len(lines)-total_benchmark) + " queued --- " + lines[local_index].replace("\\", ""))
                elif return_value == 153:
                    logging.info("   FAILURE (File size limit exceeded) --- OVERALL: " + str(passed_benchmark) + " passed, " + str(total_benchmark -
                                 passed_benchmark) + " failed, " + str(len(lines)-total_benchmark) + " queued --- " + lines[local_index].replace("\\", ""))
                else:
                    logging.info("   FAILURE --- OVERALL: " + str(passed_benchmark) + " passed, " + str(total_benchmark-passed_benchmark) +
                                 " failed, " + str(len(lines)-total_benchmark) + " queued --- " + lines[local_index].replace("\\", ""))
            with lock:
                local_index = line_index
                line_index += 1

# Computing relative path


def ComputeDirectory(line):
    configuration_name = ""
    benchmark_name = ""
    tokens = shlex.split(line)
    for token in tokens:
        if token.find("--configuration-name") != -1:
            configuration_name = token[len("--configuration-name="):]
        if token.find("--benchmark-name") != -1:
            benchmark_name = token[len("--benchmark-name="):]
    new_dir = os.path.join(abs_path, configuration_name, benchmark_name)
    return new_dir

# Search c files


def SearchCFiles(directory):
    logging.info("       Looking for file in " + str(directory))
    files = set()
    for element in os.listdir(directory):
        if os.path.isdir(os.path.join(directory, element)):
            files = files.union(SearchCFiles(os.path.join(directory, element)))
        elif (element[-2:] == ".c") or (element[-2:] == ".C") or (element[-4:] == ".CPP") or (element[-4:] == ".cpp") or (element[-4:] == ".cxx") or (element[-3:] == ".cc") or (element[-4:] == ".c++"):
            files.add(os.path.join(directory, element))
    return files

# Collecting results


def CollectResults(directory):
    # Skip if this is a leaf directory
    if os.path.exists(os.path.join(directory, args.tool + "_return_value")) or os.listdir(directory) == []:
        return
    subdirs = [s for s in sorted(os.listdir(directory)) if os.path.isdir(
        os.path.join(directory, s)) and s != "panda-temp" and s != "HLS_output"]
    for subdir in subdirs:
        CollectResults(os.path.join(directory, subdir))
    tool_failed_output = open(os.path.join(
        directory, args.tool + "_failed_output"), "w")
    for subdir in subdirs:
        if os.path.exists(os.path.join(directory, subdir, args.tool + "_failed_output")):
            tool_failed_output.write(
                open(os.path.join(directory, subdir, args.tool + "_failed_output")).read())
            if os.path.exists(os.path.join(directory, subdir, args.tool + "_execution_output")):
                tool_failed_output.write("\n")
                tool_failed_output.write("\n")
                tool_failed_output.write("\n")
    tool_failed_output.close()
    timing_violation_report = open(os.path.join(
        directory, "timing_violation_report"), "w")
    for subdir in subdirs:
        if os.path.exists(os.path.join(directory, subdir, "timing_violation_report")):
            timing_violation_report.write(
                open(os.path.join(directory, subdir, "timing_violation_report")).read())
            if os.path.exists(os.path.join(directory, subdir, args.tool + "_execution_output")):
                timing_violation_report.write("\n")
                timing_violation_report.write("\n")
                timing_violation_report.write("\n")
    timing_violation_report.close()
    report_file = open(os.path.join(directory, "report"), "w")
    for subdir in subdirs:
        if os.path.exists(os.path.join(directory, subdir, args.tool + "_return_value")):
            return_value_file_name = os.path.join(
                directory, subdir, args.tool + "_return_value")
            return_value_file = open(return_value_file_name)
            return_value = return_value_file.read()
            return_value_file.close()
            args_file = open(os.path.join(directory, subdir, "args"))
            command_args = args_file.readlines()[0]
            command_args = command_args.replace(abs_benchmarks_root + "/", "")
            args_file.close()
            if return_value == "0":
                tool_results_file_name = os.path.join(
                    directory, subdir, args.tool + "_results")
                if os.path.exists(tool_results_file_name):
                    report_file.write(
                        "SUCCESS (" + open(tool_results_file_name).read() + " cycles) " + command_args.replace("\\", ""))
                else:
                    report_file.write(
                        "SUCCESS: " + command_args.replace("\\", ""))
            else:
                if return_value == "124":
                    report_file.write("FAILURE(Timeout): " +
                                      command_args.replace("\\", ""))
                else:
                    report_file.write(
                        "FAILURE: " + command_args.replace("\\", ""))
            report_file.write("\n")
        elif os.path.exists(os.path.join(directory, subdir, "report")):
            local_report_file = open(os.path.join(directory, subdir, "report"))
            report_file.write(local_report_file.read())
            local_report_file.close()
    report_file.close()
    if args.tool == "bambu":
        input_files = ""
        named_list_name = os.path.join(abs_path, "named_list")
        lines = open(named_list_name).readlines()
        for line in lines:
            local_dir = ComputeDirectory(line)
            if os.path.exists(os.path.join(local_dir, args.tool + "_results_0.xml")):
                input_files = input_files + " " + \
                    os.path.join(local_dir, args.tool + "_results_0.xml")
        if len(input_files) > 0:
            # Generate experimental setup xml
            experimental_setup_file_name = os.path.join(
                abs_path, "experimental_setup.xml")
            temp_list = open(experimental_setup_file_name, "w")
            bambu_version_file_name = os.path.join(abs_path, "bambu_version")
            bambu_version_file = open(bambu_version_file_name, "w")
            bambu_version_command = [tool_exe]
            bambu_version_command.extend(shlex.split("--version"))
            subprocess.call(bambu_version_command, stdout=bambu_version_file)
            bambu_version_file.close()
            bambu_version_file = open(bambu_version_file_name, "r")
            bambu_version = bambu_version_file.readlines()[-2].rstrip()
            bambu_version_file.close()
            if args.commonargs != None:
                bambu_arguments = ' '.join(' '.join(map(str, l))
                                           for l in args.commonargs)
            else:
                bambu_arguments = ""
            temp_list.write("<?xml version=\"1.0\"?>\n")
            temp_list.write("<experimental_setup>\n")
            temp_list.write("   <bambu_version value=\"" +
                            bambu_version + "\"/>\n")
            temp_list.write("   <timestamp value=\"" +
                            str(datetime.datetime.now()) + "\"/>\n")
            temp_list.write("   <script value=\"" + args.script + "\"/>\n")
            temp_list.write("   <bambu_arguments value=\"" +
                            bambu_arguments + "\"/>\n")
            temp_list.write("   <benchmarks>\n")
            reordered_list_name = os.path.join(abs_path, "reordered_list")
            reordered_list = open(reordered_list_name, "r")
            for line in reordered_list.readlines():
                temp_list.write("      <benchmark value=\"" + line.rstrip().replace("\\\\\\-", "-").replace("\\\\\\=", "=").replace(
                    "\\\\\\_", "_").replace("\\\\\\/", "/").replace("\\\\\\\"", "&quot;").replace("\\\\\\.", ".") + "\"/>\n")
            temp_list.write("   </benchmarks>\n")
            temp_list.write("</experimental_setup>\n")
            temp_list.close()
            local_args = input_files + " " + experimental_setup_file_name
            if os.path.exists(args.spider_style):
                local_args = local_args + " " + args.spider_style + " " + table
            else:
                local_args = local_args + " " + \
                    os.path.join(os.path.dirname(os.path.realpath(__file__)), "latex_table_format/",
                                 args.spider_style) + " " + table
#            logging.info("   Creating tex " + spider + " " + local_args)
            logging.info("   Creating tex " + spider)
            local_command = [spider]
            local_command.extend(shlex.split(local_args))
            return_value = subprocess.call(local_command)
        if len(input_files) > 0 and csv != None:
            local_args = input_files + " " + csv
#            logging.info("   Creating csv " + spider + " " + local_args)
            logging.info("   Creating csv " + spider)
            local_command = [spider]
            local_command.extend(shlex.split(local_args))
            return_value = subprocess.call(local_command)
        logging.info("Collected results of " + directory)


# Create Junit Body
def CreateJunitBody(directory, ju_file):
    # Skip if this is a leaf directory
    if os.path.exists(os.path.join(directory, args.tool + "_return_value")) or os.listdir(directory) == []:
        return
    subdirs = [s for s in sorted(os.listdir(directory)) if os.path.isdir(
        os.path.join(directory, s)) and s != "panda-temp" and s != "HLS_output"]
    print_testsuite = False
    for subdir in subdirs:
        if os.path.exists(os.path.join(directory, subdir, args.tool + "_return_value")):
            print_testsuite = True
        CreateJunitBody(os.path.join(directory, subdir), ju_file)
    failed_counter_file_name = os.path.join(abs_path, "failed_counter")
    failed_counter = "0"
    if os.path.exists(failed_counter_file_name):
        failed_counter_file = open(failed_counter_file_name)
        failed_counter = failed_counter_file.read()

    if print_testsuite and len(subdirs) > 0:
        ju_file.write("  <testsuite disabled=\"0\" errors=\"0\" failures=\""+failed_counter+"\" name=\""+directory +
                      "\" tests=\"" + str(len(subdirs)) + "\" timestamp=\"" + str(datetime.datetime.now()) + "\">\n")
    for subdir in subdirs:
        if os.path.exists(os.path.join(directory, subdir, args.tool + "_return_value")):
            return_value_file_name = os.path.join(
                directory, subdir, args.tool + "_return_value")
            return_value_file = open(return_value_file_name)
            return_value = return_value_file.read()
            return_value_file.close()
            args_file = open(os.path.join(directory, subdir, "args"))
            command_args = args_file.readlines()[0]
            command_args = command_args.replace(abs_benchmarks_root + "/", "")
            args_file.close()
            if return_value == "0":
                ju_file.write("    <testcase classname=\"PandA-bambu-tests\" name=\"" +
                              command_args.replace("\\", "").replace("\"", "&quot;") + "\">\n")
            else:
                if return_value == "124":
                    ju_file.write("    <testcase classname=\"PandA-bambu-tests\" name=\"" +
                                  command_args.replace("\\", "").replace("\"", "&quot;") + "\">\n")
                    ju_file.write(
                        "      <failure type=\"FAILURE(Timeout)\"></failure>\n")
                    ju_file.write("      <system-out>\n")
                    ju_file.write("<![CDATA[\n")
                else:
                    ju_file.write("    <testcase classname=\"PandA-bambu-tests\" name=\"" +
                                  command_args.replace("\\", "").replace("\"", "&quot;") + "\">\n")
                    ju_file.write(
                        "      <failure type=\"FAILURE\"></failure>\n")
                    ju_file.write("      <system-out>\n")
                    ju_file.write("<![CDATA[\n")
                with open(os.path.join(directory, args.tool + "_failed_output")) as f:
                    for line in deque(f, maxlen=args.junit_logtail):
                        ju_file.write(line)
                ju_file.write("]]>\n")
                ju_file.write("      </system-out>\n")
            ju_file.write("    </testcase>\n")
    if print_testsuite and len(subdirs) > 0:
        ju_file.write("  </testsuite>\n")


# Create PerfPublisher Body
def CreatePerfPublisherBody(directory, pp_file):
    # Skip if this is a leaf directory
    if os.path.exists(os.path.join(directory, args.tool + "_return_value")) or os.listdir(directory) == []:
        return
    subdirs = [s for s in sorted(os.listdir(directory)) if os.path.isdir(
        os.path.join(directory, s)) and s != "panda-temp" and s != "HLS_output"]
    for subdir in subdirs:
        CreatePerfPublisherBody(os.path.join(directory, subdir), pp_file)

    for subdir in subdirs:
        if os.path.exists(os.path.join(directory, subdir, args.tool + "_return_value")):
            pp_file.write("  <test name=\"" + str(directory) +
                          "/" + str(subdir) + "\" executed=\"yes\">\n")
            pp_file.write("    <result>\n")
            return_value_file_name = os.path.join(
                directory, subdir, args.tool + "_return_value")
            return_value_file = open(return_value_file_name)
            return_value = return_value_file.read()
            return_value_file.close()
            args_file = open(os.path.join(directory, subdir, "args"))
            command_args = args_file.readlines()[0]
            command_args = command_args.replace(abs_benchmarks_root + "/", "")
            args_file.close()
            if return_value == "0":
                pp_file.write(
                    "      <success passed=\"yes\" state=\"100\" hasTimedOut=\"false\"/>\n")
                cycles_tag = ""
                areatime_tag = ""
                slice_tag = ""
                sliceluts_tag = ""
                registers_tag = ""
                dsps_tag = ""
                brams_tag = ""
                period_tag = ""
                slack_tag = ""
                frequency_tag = ""
                HLS_execution_time_tag = ""
                if os.path.exists(os.path.join(directory, subdir, args.tool + "_results_0.xml")):
                    xml_document = minidom.parse(os.path.join(
                        directory, subdir, args.tool + "_results_0.xml"))
                    if len(xml_document.getElementsByTagName("CYCLES")) > 0:
                        cycles_tag = str(xml_document.getElementsByTagName(
                            "CYCLES")[0].attributes["value"].value)
                    if len(xml_document.getElementsByTagName("AREAxTIME")) > 0:
                        areatime_tag = str(xml_document.getElementsByTagName(
                            "AREAxTIME")[0].attributes["value"].value)
                    if len(xml_document.getElementsByTagName("SLICE")) > 0:
                        slice_tag = str(xml_document.getElementsByTagName(
                            "SLICE")[0].attributes["value"].value)
                    if len(xml_document.getElementsByTagName("SLICE_LUTS")) > 0:
                        sliceluts_tag = str(xml_document.getElementsByTagName(
                            "SLICE_LUTS")[0].attributes["value"].value)
                    if len(xml_document.getElementsByTagName("REGISTERS")) > 0:
                        registers_tag = str(xml_document.getElementsByTagName(
                            "REGISTERS")[0].attributes["value"].value)
                    if len(xml_document.getElementsByTagName("DSPS")) > 0:
                        dsps_tag = str(xml_document.getElementsByTagName(
                            "DSPS")[0].attributes["value"].value)
                    if len(xml_document.getElementsByTagName("BRAMS")) > 0:
                        brams_tag = str(xml_document.getElementsByTagName(
                            "BRAMS")[0].attributes["value"].value)
                    if len(xml_document.getElementsByTagName("PERIOD")) > 0:
                        period_tag = str(xml_document.getElementsByTagName(
                            "PERIOD")[0].attributes["value"].value)
                    if len(xml_document.getElementsByTagName("CLOCK_SLACK")) > 0:
                        slack_tag = str(xml_document.getElementsByTagName(
                            "CLOCK_SLACK")[0].attributes["value"].value)
                    if len(xml_document.getElementsByTagName("FREQUENCY")) > 0:
                        frequency_tag = str(xml_document.getElementsByTagName(
                            "FREQUENCY")[0].attributes["value"].value)
                    if len(xml_document.getElementsByTagName("HLS_execution_time")) > 0:
                        HLS_execution_time_tag = str(xml_document.getElementsByTagName(
                            "HLS_execution_time")[0].attributes["value"].value)

                if cycles_tag != "" and cycles_tag != "inf":
                    pp_file.write("      <performance unit=\"cycles\" mesure=\"" +
                                  cycles_tag + "\" isRelevant=\"true\"/>\n")
                if HLS_execution_time_tag != "":
                    pp_file.write("      <executiontime unit=\"s\" mesure=\"" +
                                  HLS_execution_time_tag + "\" isRelevant=\"true\"/>\n")

                if (areatime_tag != "" and areatime_tag != "inf") or (slice_tag != "" and slice_tag != "inf") or (sliceluts_tag != "" and sliceluts_tag != "inf") or (registers_tag != "" and registers_tag != "inf") or (dsps_tag != "" and dsps_tag != "inf") or (brams_tag != "" and brams_tag != "inf") or (period_tag != "" and period_tag != "inf") or (slack_tag != "" and slack_tag != "inf") or (frequency_tag != "" and frequency_tag != "inf"):
                    pp_file.write("      <metrics>\n")
                    if areatime_tag != "" and areatime_tag != "inf":
                        pp_file.write("        <areatime unit=\"lutxns\" mesure=\"" +
                                      areatime_tag + "\" isRelevant=\"true\"/>\n")
                    if slice_tag != "" and slice_tag != "inf":
                        pp_file.write("        <slices unit=\"ns\" mesure=\"" +
                                      slice_tag + "\" isRelevant=\"true\"/>\n")
                    if sliceluts_tag != "" and sliceluts_tag != "inf":
                        pp_file.write("        <sliceluts unit=\"slice\" mesure=\"" +
                                      sliceluts_tag + "\" isRelevant=\"true\"/>\n")
                    if registers_tag != "" and registers_tag != "inf":
                        pp_file.write("        <registers unit=\"registers\" mesure=\"" +
                                      registers_tag + "\" isRelevant=\"true\"/>\n")
                    if dsps_tag != "" and dsps_tag != "inf":
                        pp_file.write(
                            "        <dsps unit=\"dsp\" mesure=\"" + dsps_tag + "\" isRelevant=\"true\"/>\n")
                    if brams_tag != "" and brams_tag != "inf":
                        pp_file.write("        <brams unit=\"bram\" mesure=\"" +
                                      brams_tag + "\" isRelevant=\"true\"/>\n")
                    if period_tag != "" and period_tag != "inf":
                        pp_file.write("        <period unit=\"ns\" mesure=\"" +
                                      period_tag + "\" isRelevant=\"true\"/>\n")
                    if slack_tag != "" and slack_tag != "inf":
                        pp_file.write("        <slack unit=\"ns\" mesure=\"" +
                                      slack_tag + "\" isRelevant=\"true\"/>\n")
                    if frequency_tag != "" and frequency_tag != "inf":
                        pp_file.write("        <frequency unit=\"MHz\" mesure=\"" +
                                      frequency_tag + "\" isRelevant=\"true\"/>\n")
                    pp_file.write("      </metrics>\n")
            else:
                if return_value == "124":
                    pp_file.write(
                        "      <success passed=\"no\" state=\"0\" hasTimedOut=\"true\"/>\n")
                else:
                    pp_file.write(
                        "      <success passed=\"no\" state=\"0\" hasTimedOut=\"false\"/>\n")
            pp_file.write("    </result>\n")
            pp_file.write("  </test>\n")


parser = argparse.ArgumentParser(
    description="Performs panda tests", fromfile_prefix_chars='@')
parser.add_argument(
    "files", help="The files to be tested: they can be configuration files, directories containing benchmarks or source code files.", nargs='*', action="append")
parser.add_argument('-l', "--benchmarks_list",
                    help="The file containing the list of tests to be performed", nargs='*', action="append")
parser.add_argument('-b', "--benchmarks_root",
                    help="The directory containing benchmarks")
parser.add_argument(
    '-o', "--output", help="The directory where output files we be put (default=\"output\")", default="output")
parser.add_argument('-j', help="The number of jobs which execute the benchmarks (default=\"1\")",
                    default=int(os.getenv('J', "1")), type=positive_integer, action=StoreOrUpdateMin)
parser.add_argument(
    "--bambu", help="The bambu executable (default=bambu)", default="bambu")
parser.add_argument("--spider", help="The spider executable (default=spider)",
                    default="spider")
parser.add_argument("--spider-style", help="The spider table style (default=latex_format_bambu_results.xml)",
                    default="latex_format_bambu_results.xml")
parser.add_argument(
    "--zebu", help="The zebu executable (default=zebu)", default="zebu")
parser.add_argument(
    '-t', "--timeout", help="Timeout for tool execution (default=75m)", default="75m")
parser.add_argument(
    '-a', "--args", help="A set of arguments to be passed to the tool", nargs='*', action='append')
parser.add_argument('-c', "--commonargs",
                    help="A set of arguments to be passed to the tool", nargs='*', action='append')
parser.add_argument(
    "--table", help="Print the results in tex format", default="results.tex")
parser.add_argument("--csv", help="Print the results in csv format")
parser.add_argument("--tool", help="The tool to be tested", default="bambu")
parser.add_argument("--ulimit", help="The ulimit options",
                    default="-f 8388608 -s 32768 -v 16777216")
parser.add_argument("--stop", help="Stop the execution on first error (default=false)",
                    default=False, action="store_true")
parser.add_argument("--returnfail", help="Return FAILURE in case at least one test fails (default=false)",
                    default=False, action="store_true")
parser.add_argument("--mail", help="Send a mail with the result")
parser.add_argument(
    "--name", help="Set the name of this regression (default=Bambu regression)", nargs='*', action='append')
parser.add_argument("--no-clean", help="Do not clean produced files",
                    default=False, action="store_true")
parser.add_argument("--restart", help="Restart last execution (default=false)",
                    default=False, action="store_true")
parser.add_argument(
    "--script", help="Set the bash script in the generated tex", default="")
parser.add_argument("--junitdir", help="Set the JUnit directory", default="")
parser.add_argument(
    "--junit-logtail", help="Set the JUnit log tail length for failed tests (default=300)", default=300, type=positive_integer)
parser.add_argument("--perfpublisherdir",
                    help="Set the PerfPublisher directory", default="")

args = parser.parse_args()
n_jobs = args.j  # set this, because it will be overwritten by the parse of modified_argv
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')

# The absolute path of current script
abs_script = os.path.abspath(sys.argv[0])

# Expand configuration files
modified_argv = []
modified_argv.append(sys.argv[0])
abs_configuration_dir = ""
for arg in sys.argv[1:]:
    if arg in args.files[0]:
        # .c/c++ file
        if (arg[-2:] == ".c") or (arg[-2:] == ".C") or (arg[-4:] == ".CPP") or (arg[-4:] == ".cpp") or (arg[-4:] == ".cxx") or (arg[-3:] == ".cc") or (arg[-4:] == ".c++"):
            modified_argv.append(arg)
        # Check if it is a directory
        elif os.path.exists(arg) and os.path.isdir(arg):
            modified_argv.append(arg)
        elif args.benchmarks_root != None and os.path.exists(os.path.join(os.path.abspath(args.benchmarks_root), arg)) and os.path.isdir(os.path.join(os.path.abspath(args.benchmarks_root), arg)):
            modified_argv.append(arg)
        elif os.path.exists(os.path.join(os.path.dirname(abs_script), arg)) and os.path.isdir(os.path.join(os.path.dirname(abs_script), arg)):
            modified_argv.append(arg)
        else:
            modified_argv.append("@" + arg)
    else:
        modified_argv.append(arg)
args = parser.parse_args(modified_argv)

# The absolute path of current script
abs_script = os.path.abspath(sys.argv[0])

# The table to be produced
table = os.path.abspath(args.table)

# The csv to be produced
if args.csv != None:
    csv = os.path.abspath(args.csv)
else:
    csv = None

# Check if output directory exists, if yes abort
if os.path.exists(args.output) and not args.restart:
    logging.error("Output directory " + args.output +
                  " already exists. Please remove it or specify a different one with -o")
    sys.exit(1)

# Check if JUnit dir exist
if args.junitdir != "" and not os.path.exists(args.junitdir):
    os.mkdir(args.junitdir)
# Check if PerfPublisher dir exist
if args.perfpublisherdir != "" and not os.path.exists(args.perfpublisherdir):
    os.mkdir(args.perfpublisherdir)
# compute JUnit file name
junit_file_name = ""
if args.junitdir != "":
    junit_index = 0
    junit_file_name = os.path.abspath(os.path.join(
        args.junitdir, "Junit_report"+str(junit_index)+".xml"))
    while os.path.isfile(junit_file_name):
        junit_index = junit_index + 1
        junit_file_name = os.path.abspath(os.path.join(
            args.junitdir, "Junit_report"+str(junit_index)+".xml"))
# compute PerfPublisher file name
perfpublisher_name = ""
perfpublisher_file_name = ""
if args.perfpublisherdir != "":
    perfpublisher_index = 0
    perfpublisher_name = "PerfPublisher_report"+str(perfpublisher_index)
    perfpublisher_file_name = os.path.abspath(os.path.join(
        args.perfpublisherdir, perfpublisher_name+".xml"))
    while os.path.isfile(perfpublisher_file_name):
        perfpublisher_index = perfpublisher_index + 1
        perfpublisher_name = "PerfPublisher_report"+str(perfpublisher_index)
        perfpublisher_file_name = os.path.abspath(os.path.join(
            args.perfpublisherdir, perfpublisher_name+".xml"))

# Create the folder and enter in it
abs_path = os.path.abspath(args.output)

if args.restart:
    if not os.path.exists(abs_path):
        args.restart = False
if not args.restart:
    os.mkdir(abs_path)
os.chdir(abs_path)


# Skipping if all benchmarks already pass
if args.restart:
    failed_counter_file_name = os.path.join(abs_path, "failed_counter")
    if os.path.exists(failed_counter_file_name):
        failed_counter_file = open(failed_counter_file_name)
        failed_counter = failed_counter_file.read()
        if failed_counter == "0" and args.junitdir == "" and args.perfpublisherdir == "":
            logging.info("Already pass")
            sys.exit(0)

# Check tool executable
tool_exe = ""
if args.tool == "bambu":
    if os.path.isfile(args.bambu) and os.access(args.bambu, os.X_OK):
        tool_exe = args.bambu
    else:
        # Check bambu in the path
        for path in os.environ["PATH"].split(os.pathsep):
            exe_file = os.path.join(path, "bambu")
            if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK):
                tool_exe = exe_file
    if tool_exe == "":
        if args.bambu != "opt/panda/bin/bambu":
            if not os.path.isfile(args.bambu):
                logging.error(args.bambu + " does not exist")
            else:
                logging.error(args.bambu + " is not an executable")
        else:
            logging.error("bambu not found")
        sys.exit(1)
    logging.info("Bambu found: " + tool_exe)
elif args.tool == "zebu":
    if os.path.isfile(args.zebu) and os.access(args.zebu, os.X_OK):
        tool_exe = args.zebu
    else:
        # Check zebu in the path
        for path in os.environ["PATH"].split(os.pathsep):
            exe_file = os.path.join(path, "zebu")
            if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK):
                tool_exe = exe_file
    if tool_exe == "":
        if args.zebu != "opt/panda/bin/zebu":
            if not os.path.isfile(args.zebu):
                logging.error(args.zebu + " does not exist")
            else:
                logging.error(args.zebu + " is not an executable")
        else:
            logging.error("zebu not found")
        sys.exit(1)
    logging.info("Zebu found: " + tool_exe)
else:
    tool_exe = args.tool
    if distutils.spawn.find_executable(tool_exe) == None:
        logging.error(tool_exe + " not found")
        sys.exit(1)

if args.benchmarks_root is None:
    abs_benchmarks_root = abs_configuration_dir
else:
    if os.path.isabs(args.benchmarks_root):
        abs_benchmarks_root = os.path.abspath(args.benchmarks_root)
    else:
        if os.path.exists(os.path.join(os.path.abspath(".."), args.benchmarks_root)):
            abs_benchmarks_root = os.path.join(
                os.path.abspath(".."), args.benchmarks_root)
        else:
            if os.path.exists(os.path.join(os.path.abspath(os.path.join(os.path.dirname(abs_script), "../../..")), args.benchmarks_root)):
                abs_benchmarks_root = os.path.join(os.path.abspath(os.path.join(
                    os.path.dirname(abs_script), "../../..")), args.benchmarks_root)
            else:
                logging.error(args.benchmarks_root + " not found")
                sys.exit(1)

# Check spider executable
spider = ""
if os.path.isfile(args.spider) and os.access(args.spider, os.X_OK):
    spider = args.spider
else:
    # Check spider in the path
    for path in os.environ["PATH"].split(os.pathsep):
        exe_file = os.path.join(path, "spider")
        if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK):
            spider = exe_file
if spider == "":
    if args.spider != "opt/panda/bin/spider":
        if not os.path.isfile(args.spider):
            logging.error(args.spider + " does not exist")
        else:
            logging.error(args.spider + " is not an executable")
    else:
        logging.error("spider not found")
    sys.exit(1)

if args.benchmarks_root is None:
    abs_benchmarks_root = abs_configuration_dir
else:
    if os.path.isabs(args.benchmarks_root):
        abs_benchmarks_root = os.path.abspath(args.benchmarks_root)
    else:
        if os.path.exists(os.path.join(os.path.abspath(".."), args.benchmarks_root)):
            abs_benchmarks_root = os.path.join(
                os.path.abspath(".."), args.benchmarks_root)
        else:
            if os.path.exists(os.path.join(os.path.abspath(os.path.join(os.path.dirname(abs_script), "../../..")), args.benchmarks_root)):
                abs_benchmarks_root = os.path.join(os.path.abspath(os.path.join(
                    os.path.dirname(abs_script), "../../..")), args.benchmarks_root)
            else:
                logging.error(args.benchmarks_root + " not found")
                sys.exit(1)
mutt = None
logging.info("Spider found: " + spider)

# Check mutt executable
if args.mail != None:
    # Check mutt in the path
    for path in os.environ["PATH"].split(os.pathsep):
        exe_file = os.path.join(path, "mutt")
        if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK):
            mutt = exe_file
    if mutt == None:
        logging.error("mutt not found")
        sys.exit(1)


if not args.restart:
    # Check if file lists exist
    abs_lists = []
    if args.benchmarks_list != None:
        for relative_list in args.benchmarks_list:
            # First look in the current directory
            if os.path.exists(os.path.abspath("../" + relative_list[0])):
                abs_lists.append(os.path.abspath("../" + relative_list[0]))
            # Then look in script directory
            elif os.path.exists(os.path.join(os.path.dirname(abs_script), relative_list[0])):
                abs_lists.append(os.path.join(
                    os.path.dirname(abs_script), relative_list[0]))
            # Then look in configuration directory
            elif os.path.exists(os.path.join(abs_configuration_dir, relative_list[0])):
                abs_lists.append(os.path.join(
                    abs_configuration_dir, relative_list[0]))
            # Then look in benchmarks root
            elif os.path.exists(os.path.join(abs_benchmarks_root, relative_list[0])):
                abs_lists.append(os.path.join(
                    abs_benchmarks_root, relative_list[0]))
            else:
                logging.error(relative_list[0] + " does not exist")
                sys.exit(1)
    files_list = []
    # Create temp list with arg
    for arg in args.files[0]:
        # .c/.c++ file
        if (arg[-2:] == ".c") or (arg[-2:] == ".C") or (arg[-4:] == ".CPP") or (arg[-4:] == ".cpp") or (arg[-4:] == ".cxx") or (arg[-3:] == ".cc") or (arg[-4:] == ".c++"):
            files_list.append(arg)
        # Check if it is a directory
        elif os.path.exists(arg) and os.path.isdir(arg):
            files_list.append(arg)
        elif os.path.exists(os.path.join(os.path.dirname(abs_script), arg)) and os.path.isdir(os.path.join(os.path.dirname(abs_script), arg)):
            files_list.append(arg)
        elif os.path.exists(os.path.join(abs_benchmarks_root, arg)) and os.path.isdir(os.path.join(abs_benchmarks_root, arg)):
            files_list.append(os.path.join(abs_benchmarks_root, arg))

    if args.benchmarks_list == None and len(files_list) == 0 and (args.tool == "bambu" or args.tool == "zebu"):
        logging.error("Benchmarks not found")
        sys.exit(2)

    if len(files_list) > 0:
        temp_list = open(os.path.join(abs_path, "temp_list"), "w")
        for element in files_list:
            temp_list.write(element)
        temp_list.close()
        abs_lists.append(os.path.join(abs_path, "temp_list"))

    # Reordering elements in each row
    reordered_list_name = os.path.join(abs_path, "reordered_list")
    reordered_list = open(reordered_list_name, "w")

    logging.info("Preparing benchmark list")
    logging.info("   Reordering arguments")
    for abs_list in abs_lists:
        list_file = open(abs_list)
        lines = list_file.readlines()
        list_file.close()
        for line in lines:
            if line.strip() == "":
                continue
            if line[0] == '#':
                continue
            if args.tool != "bambu" and args.tool != "zebu":
                reordered_list.write(line)
                continue
            tokens = shlex.split(line)
            parameters = list()
            escape_pattern = "(-|_|\{|\}|=|!|\$|#|&|\"|\'|\(|\)|\||<|>|`|\\\|;|\.|,)"
            # Flag used to ad-hoc manage --param arg
            follow_param = False
            for token in tokens:
                if token[0] == '-':
                    parameters.append(
                        re.sub(escape_pattern, r"\\\1", token))
                    if token.find("--param") != -1:
                        follow_param = True
                    else:
                        follow_param = False
                else:
                    if follow_param == True:
                        parameters.append(
                            re.sub(escape_pattern, r"\\\1", token))
                    else:
                        reordered_list.write(token + " ")
                    follow_param = False
            for parameter in parameters:
                reordered_list.write(
                    re.sub(escape_pattern, r"\\\1", parameter) + " ")
            reordered_list.write("\n")
    reordered_list.close()

    # Expanding directory
    expanded_list_name = os.path.join(abs_path, "expanded_list")
    expanded_list = open(expanded_list_name, "w")

    logging.info("   Expanding directory")
    lines = open(reordered_list_name).readlines()
    for line in lines:
        if line.strip() == "":
            continue
        tokens = shlex.split(line)
        if args.tool == "bambu" or args.tool == "zebu":
            if(tokens[0][0] != '/'):
                first_parameter = os.path.join(abs_benchmarks_root, tokens[0])
            else:
                first_parameter = tokens[0]
        else:
            first_parameter = tokens[0].replace(
                "BENCHMARKS\_ROOT", abs_benchmarks_root)
        other_parameters = tokens[1:len(tokens)]
        if not os.path.exists(first_parameter) and (args.tool == "bambu" or args.tool == "zebu"):
            logging.error(first_parameter + " does not exist")
            sys.exit(1)
        # Check if it is a directory or a file
        if os.path.isdir(first_parameter):
            logging.info("       " + tokens[0])
            c_files = SearchCFiles(first_parameter)
            c_files = sorted(c_files)
            for c_file in c_files:
                expanded_list.write(c_file)
                for other_parameter in other_parameters:
                    expanded_list.write(
                        " " + other_parameter.replace("BENCHMARKS\_ROOT", abs_benchmarks_root))
                expanded_list.write("\n")
        else:
            expanded_list.write(first_parameter)
            for other_parameter in other_parameters:
                if ((other_parameter[-2:] == ".c") or (other_parameter[-2:] == ".C") or (other_parameter[-4:] == ".CPP") or (other_parameter[-4:] == ".cpp") or (other_parameter[-4:] == ".cxx") or (other_parameter[-3:] == ".cc") or (other_parameter[-4:] == ".c++") or other_parameter[-4:] == ".xml") and other_parameter[0] != '\\':
                    if other_parameter[0] == '/':
                        expanded_list.write(" " + other_parameter)
                    else:
                        expanded_list.write(
                            " " + os.path.join(abs_benchmarks_root, other_parameter))
                else:
                    expanded_list.write(" " + other_parameter.replace("BENCHMARKS\_ROOT",
                                        abs_benchmarks_root).replace("BENCHMARKS_ROOT", abs_benchmarks_root))
            expanded_list.write("\n")
    expanded_list.close()

    # Adding parameters
    logging.info("   Considering all tool arguments")
    arg_lists = args.args
    if not arg_lists:
        arg_lists = [("")]
    arged_list_name = os.path.join(abs_path, "arged_list")
    arged_list = open(arged_list_name, "w")
    lines = open(expanded_list_name).readlines()
    for arg_list in arg_lists:
        for line in lines:
            arged_list.write(line.rstrip())
            if len(arg_list) > 0:
                arg = arg_list[0]
                if arg[0] == "\"":
                    arg = arg[1:-1]
                arged_list.write(" " + arg)
            if args.commonargs != None and len(args.commonargs) > 0:
                for commonarg in args.commonargs:
                    arged_list.write(" " + commonarg[0].replace("#", " "))
            arged_list.write("\n")
    arged_list.close()

    # Name of benchmarks
    full_names = set()

    # Adding benchmark name
    logging.info("   Adding benchmark name")
    named_list_name = os.path.join(abs_path, "named_list")
    named_list = open(named_list_name, "w")
    lines = open(arged_list_name).readlines()
    for line in lines:
        named_list.write(line.rstrip())
        # Retrieve configuration name and benchmark name
        configuration_name = ""
        benchmark_name = ""
        tokens = shlex.split(line)
        for token in tokens:
            if token.find("--configuration-name") != -1:
                configuration_name = token[len("--configuration-name="):]
            if token.find("--benchmark-name") != -1:
                benchmark_name = token[len("--benchmark-name="):]
        if benchmark_name == "":
            if args.tool != "bambu" and args.tool != "zebu":
                logging.error("Missing benchmark name")
                sys.exit(1)
            benchmark_name = os.path.basename(line.split()[0])[:-2]
            named_list.write(" --benchmark-name=" + benchmark_name)
        full_name = configuration_name + ":" + benchmark_name
        logging.info("     " + full_name)
        if full_name in full_names:
            logging.error(
                "Duplicated configuration name - benchmark name: " + full_name)
            sys.exit(1)
        full_names.add(full_name)
        named_list.write("\n")
    named_list.close()

    # Generating folder
    logging.info("   Generating output directories")
    lines = open(named_list_name).readlines()
    for line in lines:
        new_dir = ComputeDirectory(line)
        logging.info("      Creating directory " + new_dir)
        os.makedirs(new_dir)
else:
    logging.info("   Skipping generation of lists and directories")
    named_list_name = os.path.join(abs_path, "named_list")
    if not os.path.exists(named_list_name):
        logging.error("List of previous run not found")
        sys.exit(1)

# Create threads
logging.info("   Launching tool")
lock = threading.RLock()
lock_creation_destruction = threading.RLock()
passed_benchmark = 0
total_benchmark = 0
threads = []
children = [None] * n_jobs
for thread_index in range(n_jobs):
    threads.insert(thread_index, threading.Thread(
        target=execute_tests, args=(named_list_name, thread_index)))
    threads[thread_index].daemon = True
    threads[thread_index].start()

try:
    # Wait threads
    for thread_index in range(n_jobs):
        while threads[thread_index].is_alive():
            threads[thread_index].join(100)
except KeyboardInterrupt:
    logging.error("SIGINT received")
    failure = True
    for local_thread_index in range(n_jobs):
        if children[local_thread_index] != None:
            if children[local_thread_index].poll() == None:
                try:
                    kill_proc_tree(children[local_thread_index].pid)
                except OSError:
                    pass
    sys.exit(1)

# Collect results
CollectResults(abs_path)

# In case, it create the JUnit file
if args.junitdir != "":
    junit_file = open(junit_file_name, "w")
    junit_file.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
    junit_file.write("<testsuites disabled=\"0\" errors=\"0\" failures=\""+str(total_benchmark -
                     passed_benchmark)+"\" name=\"" + abs_path + "\" tests=\"" + str(total_benchmark) + "\">\n")
    CreateJunitBody(abs_path, junit_file)
    junit_file.write("</testsuites>\n")

# In case, it create the PerfPublisher file
if args.perfpublisherdir != "":
    perfpublisher_file = open(perfpublisher_file_name, "w")
    perfpublisher_file.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
    perfpublisher_file.write(
        "<report name=\"" + perfpublisher_name + "\" categ=\"PandA-Report\">\n")
    CreatePerfPublisherBody(abs_path, perfpublisher_file)
    perfpublisher_file.write("</report>\n")

# Prepare final report
if args.tool == "bambu" or args.tool == "zebu":
    report_file_name = os.path.join(abs_path, "report")
    report_file = open(report_file_name)
    lines = report_file.readlines()
    report_file.close()
    report_file = open(report_file_name, "w")
    command = [tool_exe, "--version"]
    subprocess.call(command, stderr=report_file, stdout=report_file)
    report_file.write("SYSTEM INFORMATION:\n")
    report_file.flush()
    command = ["lsb_release", "-a"]
    subprocess.call(command, stderr=report_file, stdout=report_file)
    report_file.write("\n")
    report_file.write("CURRENT TIME:\n")
    report_file.write(str(datetime.datetime.now()) + "\n\n")
    report_file.write("PASSED TESTS:\n")
    report_file.write(str(passed_benchmark) + "/" +
                      str(total_benchmark) + "\n\n")

    failed_counter_file_name = os.path.join(abs_path, "failed_counter")
    failed_counter_file = open(failed_counter_file_name, "w")
    failed_counter_file.write(str(total_benchmark - passed_benchmark))
    failed_counter_file.close()

    for line in lines:
        report_file.write(line)
    report_file.close()

if args.mail != None:
    outcome = ""
    if args.stop:
        if failure:
            outcome = "FAILURE"
        else:
            outcome = "SUCCESS"
    else:
        outcome = str(passed_benchmark) + "/" + str(total_benchmark)
    full_name = ""
    if len(args.name):
        for name in args.name:
            full_name = full_name + name[0]
    else:
        full_name = "Bambu"
    local_command = "cat " + report_file_name + " | mutt -s \"" + \
        full_name + ": " + outcome + "\" " + args.mail
    subprocess.call(local_command, shell=True)
if failure:
    sys.exit(1)
sys.exit(0)