How to display the amount of errors that occured in a self-verifying testbench?

Question

Below is my testbench code for a simple (unclocked) 4 bit Adder. My simulation currently will display any errors that occur along with a "Test Completed" at the end. If there are no errors, the simulation will simply return "Test Completed".
My question is: Is there a way to somehow include an "if" statement so as to display a "Test Completed, no errors" when no errors are detected in the simulation, and a "Test Completed, [x] errors found" when errors are detected in the simulation (where x is the variable amount of errors returned when the simulation is finished.)?

library IEEE;
use IEEE.STD_LOGIC_1164.all;
use IEEE.NUMERIC_STD.ALL;
use IEEE.STD_LOGIC_UNSIGNED.ALL;

ENTITY adder_4bit_TB IS
END adder_4bit_TB;

ARCHITECTURE behavior OF adder_4bit_TB IS 

    -- Component Declaration for the Unit Under Test (UUT)

    COMPONENT adder_4bit
    PORT(
         a : IN  std_logic_vector(3 downto 0);
         b : IN  std_logic_vector(3 downto 0);
         carry : OUT  std_logic;
         sum   : OUT  std_logic_vector(3 downto 0)
        );
    END COMPONENT;


   --Inputs
   signal a : std_logic_vector(3 downto 0) := (others => '0');
   signal b : std_logic_vector(3 downto 0) := (others => '0');

    --Outputs
   signal carry : std_logic;
   signal sum   : std_logic_vector(3 downto 0);


BEGIN

    -- Instantiate the Unit Under Test (UUT)
   uut: adder_4bit PORT MAP (
          a => a,
          b => b,
          carry => carry,
          sum => sum
        );


   -- Stimulus process
   stim_proc: process    -- No CLK
   begin        

       -- Initialize Input values
        a <= "0000";
        b <= "0000";

        --Loop over all values of "a" and check sum
        for I in 0 to 15 loop
            --Loop over all values of "b" and check sum
            for J in 0 to 15 loop
                -- Wait for output to update (10 ns)
                wait for 10ns;

                -- Below is the self-verification routune being implemented for the 4 bit Adder.
                -- The routine checks the sum of "a" and "b" at the end of every loop, and 
                -- reports any Errors that may have occured. If no errors occur, simulation
                -- will return "Test Completed" (line109) in Command Window.

                assert (sum = a + b) report "Expected sum of " &
                    integer'image(to_integer(unsigned((a + b)))) & ". For a = " & 
                    integer'image(to_integer(unsigned((a)))) & " and b = " & 
                    integer'image(to_integer(unsigned((b)))) & ", but returned sum was " & 
                    integer'image(to_integer(unsigned((sum)))) severity ERROR;  -- severity level can be NOTE, WARNING, ERROR, or FAILURE

                -- Increment to next value of four bit vector "b"
                b <= b + "0001";
            end loop;   

            -- Increment to next value of four bit vector "a"
            a <= a + "0001";            
        end loop;

        --Echo to user that report has finished
        report "Test completed";

      wait; --will wait forever
   end process;

END;

---

Using the answer below, here is the resulting working code:

library IEEE;
use IEEE.STD_LOGIC_1164.all;
use IEEE.NUMERIC_STD.ALL;
use IEEE.STD_LOGIC_UNSIGNED.ALL;

ENTITY adder_4bit_TB IS
END adder_4bit_TB;

ARCHITECTURE behavior OF adder_4bit_TB IS 

    -- Component Declaration for the Unit Under Test (UUT)

    COMPONENT adder_4bit
    PORT(
         a : IN  std_logic_vector(3 downto 0);
         b : IN  std_logic_vector(3 downto 0);
         carry : OUT  std_logic;
         sum   : OUT  std_logic_vector(3 downto 0)
        );
    END COMPONENT;


   --Inputs
   signal a : std_logic_vector(3 downto 0) := (others => '0');
   signal b : std_logic_vector(3 downto 0) := (others => '0');

    --Outputs
   signal carry : std_logic;
   signal sum   : std_logic_vector(3 downto 0);


    --Outputs (Testbench only)
    signal Errors : boolean;            -- Boolean value.  True if error detected. False if no error detected.     
    signal ErrorCount : integer := 0;   -- Integer value to store the qty of errors.  Intitialized to zero

BEGIN

    -- Instantiate the Unit Under Test (UUT)
   uut: adder_4bit PORT MAP (
          a => a,
          b => b,
          carry => carry,
          sum => sum
        );


   -- Stimulus process
   stim_proc: process    -- No CLK
   begin        

       -- Initialize Input values
        a <= "0000";
        b <= "0000";

        --Loop over all values of "a" and check sum
        for I in 0 to 15 loop
            --Loop over all values of "b" and check sum
            for J in 0 to 15 loop
                -- Wait for output to update (10 ns)
                wait for 10ns;

                -- Below is the self-verification routune being implemented for the 4 bit Adder.
                -- The routine checks the sum of "a" and "b" at the end of every loop, and 
                -- reports any Errors that may have occured.

                if (sum /= a + b) then  ---- "/="  syntax:  test for inequality, result is boolean

                    Errors <= true;
                    ErrorCount <= ErrorCount + 1;
                else
                    Errors <= false;
                end if;

                assert (Errors = false) report "Expected sum of " &

                    integer'image(to_integer(unsigned((a + b)))) & ". For a = " & 
                    integer'image(to_integer(unsigned((a)))) & " and b = " & 
                    integer'image(to_integer(unsigned((b)))) & ", but returned sum was " & 
                    integer'image(to_integer(unsigned((sum)))) severity ERROR;  -- severity level can be NOTE, WARNING, ERROR, or FAILURE

                -- Increment to next value of four bit vector "b"
                b <= b + "0001";
            end loop;   

            -- Increment to next value of four bit vector "a"
            a <= a + "0001";            
        end loop;

        --Echo to user that report has finished
        report "Test completed with " & integer'image(ErrorCount) & " errors";

      wait; --will wait forever
   end process;

END;

Answer 1

I recommend that you have a look at the open source test framework VUnit (https://github.com/LarsAsplund/vunit). With that you can do

check_equal(sum, a + b);

which in case of an error will give you an error message like this

ERROR: Equality check failed! Got 1111 (15). Expected 1110 (14).

To output the error statistics you can use the get_checker_stat function. For example

info("Test Summary" & LF & to_string(get_checker_stat));

which gives you something like this

INFO: Test Summary
Checks: 6
Passed: 1
Failed: 5

Answer 2

You can use a simple report statement instead of assert and wrap it in an if..then..end if block. For example:

if (error_count = 0) then
  report "Test completed." severity NOTE;
else
  report "Test completed with " & INTEGER'image(error_count) & " errors." severity ERROR;
end if;

Here is an more advanced way:

You can built a helper package for simulations that hides some internal code, so the simulation uses a much clearer interface. The following example declares a shared variable pass to track if an error occurred.

Moreover, it declares three procedures to offer an assert 'statement' and a 'print simulation result' method:

• tbFail writes a message to the simulator log and sets the tracking variable to false.
• tbAssert tests a condition and if it fails, it calls tbFail to generate a log message
• tbPrintResult writes the overall result to stdout.
  (This can also be seen in the simulator log, but especially it can be parsed by other command line tools, if the simulation runs in batch mode.)

Here is an example helper package:

use  std.TextIO.all;

package body simulation is
  -- Test Bench Status Management
  -- =============================================
  --  * Internal state variable to log a failure condition for final reporting.
  --  * Once de-asserted, this variable will never return to a value of true.
  shared variable pass : boolean := true;

  procedure tbFail(msg : in string := "") is
  begin
    if msg'length > 0 then
      report msg severity error;
    end if;
    pass := false;
  end;

  procedure tbAssert(cond : in boolean; msg : in string := "") is
  begin
    if not cond then
      tbFail(msg);
    end if;
  end;

  procedure tbPrintResult is
    variable l : line;
  begin
    write(l, string'("SIMULATION RESULT = "));
    if pass then
      write(l, string'("PASSED"));
    else
      write(l, string'("FAILED"));
    end if;
    writeline(output, l);
  end procedure;
end package;

This code can be used in a testbench as follows:

architecture test of arith_prng_tb is
  constant CLOCK_PERIOD_100MHZ  : TIME                := 10 ns;
  constant COMPARE_LIST_8_BITS  : T_SLVV_8(0 TO 15)  := (
    x"12", x"24", x"48", x"90", x"21", x"42", x"85", x"0A",
    x"14", x"28", x"51", x"A2", x"45", x"8B", x"17", x"2E"
  );

  signal SimStop      : std_logic   := '0';
  signal Clock        : STD_LOGIC   := '1';
  signal Test_got     : STD_LOGIC   := '0';
  signal PRNG_Value   : T_SLV_8;
begin
  Clock <= Clock xnor SimStop after CLOCK_PERIOD_100MHZ / 2.0;

  process
  begin
    for i in 0 to 255 loop
      Test_got        <= '1';
      wait until rising_edge(Clock);
      tbAssert(
        (PRNG_Value = COMPARE_LIST_8_BITS(I)),
        "I=" & INTEGER'image(I) &  " Value=" & raw_format_slv_hex(PRNG_Value) & " Expected=" & raw_format_slv_hex(COMPARE_LIST_8_BITS(I))
      );
    end loop;

    Test_got        <= '0';

    -- Report overall simulation result
    tbPrintResult;
    SimStop  <= '1';
    wait;
  end process;

  -- ...
end architecture;

Sources:
- PoC.simulation a helper package for simulations (VHDL-2008 version)
- Testbench for PoC.arith.prng - a pseudo random number generator

Answer 3

This would be pretty straight forward to add to what you have. Instead of using an assertion to directly test the result of the sum, use an if statement to set a boolean if the sum is not correct, then assert/count errors based on this. Something like:

variable Error : boolean;
variable ErrorCount : integer := 0;

...

if (sum /= a + b) then
    Error := true;
    ErrorCount := ErrorCount + 1;
else
    Error := false;
end if;

assert (Error = false) report "Expected sum of " &
                integer'image(to_integer(unsigned((a + b)))) & ". For a = " & 
                integer'image(to_integer(unsigned((a)))) & " and b = " & 
                integer'image(to_integer(unsigned((b)))) & ", but returned sum was " & 
                integer'image(to_integer(unsigned((sum)))) severity ERROR;

...

report "Test completed with " & integer'image(ErrorCount) & " errors";