Unit test runner in bash for C and CPP libraries

Early 2009, I wrote a parsing expression grammar library natively in C and C++.

I had already become accustomed to ReSharper’s and MSTest test runner and wanted a test report with a similar style.

ReSharper’s Test Runner

ReSharper

MSTest

Since I need to support C and C++ versions of the tests, I have decided to use a consistent interface/script, and build a bash script to compile every unit test into its own program. The program’s exit code determines a pass or fail. Should the program fail, a standard error will be displayed. Should the program pass, a different text will appear on the screen, as long as the verbose flag has been activated.

Results of my labor

The output of the bash script running all the tests with no verbose flag set.

           sh run_tests.sh

        

The output of the bash script running all the tests with verbose flag set.

           sh run_tests.sh --verbose

        

I’m already past the “Red, Green, and Refactor” stages, and this project’s status is ‘Done’. I have added some dummy tests that purposely fail, to demonstrate what failing asserts look like. Note that when a test fails you always get standard output, with standard error streams written to the standard output, regardless of the verbose flag.

           sh run_tests.sh

        

The bash scripts that make this unit test runner

lib_tests.sh

The following shell script is a library file that includes the functions that will be used by the consuming test runner script.

          #!/bin/bash

fn_test_status()
{
	case $1 in
		0)
			echo -n "Passed"
		;;
		*)
			echo -n "Failed"
			echo
		;;
	esac
}


fn_test_run()
{
	pattern=$1
	isverbose=$2
	for TEST in ${pattern}*; do
		if [ -x ${TEST} ]; then
			./${TEST} >out.teststdin 2>out.teststderr
			status=$?
			printf "%-50.43s  %-22s\n" $TEST `fn_test_status $status`

			if [ $status -ne 0 ] || [ $isverbose = 1 ]; then
				cat out.teststdin
				cat out.teststderr
				echo
			fi
		fi
	done
}

        

run_tests.sh

          #!/bin/bash
# Format of test app names: test_<structure>_<tested routines>

echo "Building tests...."
make clean all
status=$?
if [ "$status" != "0" ]
then
	echo "Build error status code: $status";
	exit $status
fi

echo
echo


. ./lib_tests.sh


isverbose=0
[ "$1" = "--verbose" ] && isverbose=1


echo
echo "Testing hashmap implementation:"
fn_test_run "bin/test_hashmap_" $isverbose

echo
echo "Testing stack implementation:"
fn_test_run "bin/test_stack_" $isverbose

echo
echo "Testing stackstack implementation:"
fn_test_run "bin/test_stackstack_" $isverbose

echo
echo "Testing list implementation:"
fn_test_run "bin/test_list_" $isverbose

echo
echo "Testing AST implementation:"
fn_test_run "bin/test_ast_" $isverbose

echo
echo "Testing input iterator implementation:"
fn_test_run "bin/test_inputiterator_" $isverbose

echo
echo "Testing npeg terminals:"
fn_test_run "bin/test_terminal_" $isverbose

echo
echo "Testing npeg non-terminals:"
fn_test_run "bin/test_nonterminal_" $isverbose

echo
echo "Testing ParserTest:"
fn_test_run "ParserTest_NpegNode/program" $isverbose
fn_test_run "ParserTest_PhoneNumber/program" $isverbose
fn_test_run "ParserTest_SimpleSentence/program" $isverbose
fn_test_run "ParserTest_MathematicalFormula/program" $isverbose
fn_test_run "ParserTest_SimpleXml/program" $isverbose

        

Example Tests

I recommend that you include the assert.h header file if you need to write the unit tests in C and the cassert header file if you need to write the unit tests in C++.

If all your asserts pass (non zero), then the program exist code should be zero, to signal the bash script that everything is okay. An assert equivalent to 0, false, generates a SIGABRT signal and provides an unsuccessful termination error code to the host environment. For example, here is a C and C++ version of a test we have in our test suite.

In C: test_nonterminal_npeg_AndPredicate.c

          #include <assert.h>
#include <stdio.h>
#include <string.h>
#include "robusthaven/text/npeg.h"
#include "robusthaven/text/npeg_inputiterator.h"

#define TESTSTRING1 "test"
#define TESTSTRING2 "TEST"
#define OTHERSTRING "somthing else"

static int _sub_expression1(npeg_inputiterator *iterator, npeg_context *context) {
  return npeg_Literal(iterator, context, TESTSTRING1, strlen(TESTSTRING1), 1);
}

static int _sub_expression2(npeg_inputiterator *iterator, npeg_context *context) {
  return npeg_Literal(iterator, context, TESTSTRING1, strlen(TESTSTRING1), 0);
}

static int _expression1(npeg_inputiterator *iterator, npeg_context *context) {
  return npeg_AndPredicate(iterator, context, _sub_expression1)
    && _sub_expression2(iterator, context);
}

static int _expression2(npeg_inputiterator *iterator, npeg_context *context) {
  return npeg_AndPredicate(iterator, context, _sub_expression1)
    && _sub_expression2(iterator, context);
}

/*
 * With all tests, also test commutativity.
 * - Does AND handle the empty string properly?
 * - Does AND only consume the string once?
 * - Does and consume nothing if one predicate is false?
 * - Does AND still work properly in the middle of a string?
 */
int main(int argc, char *argv[])
{
  const char emptystring[] = "";
  const char test1string[] = TESTSTRING1"blah";
  const char not_test1string[] = TESTSTRING2"blah";
  const char test2string[] = TESTSTRING1 TESTSTRING1 TESTSTRING2"blah";
  const char middle_test2string[] = OTHERSTRING TESTSTRING1 TESTSTRING1 TESTSTRING2"blah";
  const char errmsg[] = "some kind of error";

  npeg_inputiterator iterator;
  rh_stack_instance disable_back_reference, errors;
  rh_list_instance warnings;
  rh_stackstack_instance ast_stack;
  rh_hashmap_instance lookup;
  npeg_context context;
  npeg_error error;

  context.disableBackReferenceStack = &disable_back_reference;
  context.sandbox = &ast_stack;
  context.backReferenceLookup = &lookup;
  context.warnings = &warnings;
  context.errors = &errors;

  npeg_constructor(&context, NULL);

  npeg_inputiterator_constructor(&iterator, emptystring, 0);
  assert(_expression1(&iterator, &context) == 0 && iterator.index == 0);
  assert(_expression2(&iterator, &context) == 0 && iterator.index == 0);
  printf("\tVerified: Handling of empty string.\n");
  npeg_inputiterator_destructor(&iterator);

  npeg_inputiterator_constructor(&iterator, test1string, strlen(test1string));
  assert(_expression1(&iterator, &context) == 1 && iterator.index == strlen(TESTSTRING1));
  iterator.index = 0;
  assert(_expression2(&iterator, &context) == 1 && iterator.index == strlen(TESTSTRING1));
  printf("\tVerified: Handling of single occurence string.\n");
  npeg_inputiterator_destructor(&iterator);

  npeg_inputiterator_constructor(&iterator, test1string, strlen(test1string));
  error.message = (char*)errmsg, error.iteratorIndex = 0;
  rh_stack_push(context.errors, &error);
  assert(_expression1(&iterator, &context) == 0 && iterator.index == 0);
  rh_stack_pop(context.errors);
  printf("\tVerified: Abortion on error without modification of state.\n");
  npeg_inputiterator_destructor(&iterator);

  npeg_inputiterator_constructor(&iterator, not_test1string, strlen(not_test1string));
  assert(_expression1(&iterator, &context) == 0 && iterator.index == 0);
  assert(_expression2(&iterator, &context) == 0 && iterator.index == 0);
  printf("\tVerified: Handling of single occurence string.\n");
  npeg_inputiterator_destructor(&iterator);

  npeg_inputiterator_constructor(&iterator, test2string, strlen(test2string));
  assert(_expression1(&iterator, &context) == 1 && iterator.index == strlen(TESTSTRING1));
  iterator.index = 0;
  assert(_expression2(&iterator, &context) == 1 && iterator.index == strlen(TESTSTRING1));
  printf("\tVerified: Handling of double occurence string.\n");
  npeg_inputiterator_destructor(&iterator);

  npeg_inputiterator_constructor(&iterator, middle_test2string, strlen(middle_test2string));
  iterator.index = strlen(OTHERSTRING);
  assert(_expression1(&iterator, &context) == 1 && iterator.index
 == strlen(OTHERSTRING) + strlen(TESTSTRING1));
  iterator.index = strlen(OTHERSTRING);
  assert(_expression2(&iterator, &context) == 1 && iterator.index
 == strlen(OTHERSTRING) + strlen(TESTSTRING1));
  printf("\tVerified: Handling of double occurence at center of string.\n");
  npeg_inputiterator_destructor(&iterator);

  npeg_destructor(&context);

  return 0;
}

        

In C++: test_nonterminal_npeg_AndPredicate.cpp

          #include <cassert>
#include <cstdio>
#include <cstring>

#include "robusthaven/text/StringInputIterator.h"
#include "robusthaven/text/Npeg.h"

using namespace RobustHaven::Text;

#define TESTSTRING1 "test"
#define TESTSTRING2 "TEST"
#define OTHERSTRING "somthing else"

class _AndTest : public Npeg {
public:
  int sub_expression1() throw (ParsingFatalTerminalException) {
    return literal(TESTSTRING1, strlen(TESTSTRING1), 1);
  }

  int sub_expression2() throw (ParsingFatalTerminalException) {
    return literal(TESTSTRING1, strlen(TESTSTRING1), 0);
  }

  int expression1() throw (ParsingFatalTerminalException) {
    return andPredicate((Npeg::IsMatchPredicate)&_AndTest::sub_expression1) && sub_expression2();
  }

  int expression2() throw (ParsingFatalTerminalException) {
    return andPredicate((Npeg::IsMatchPredicate)&_AndTest::sub_expression2) && sub_expression1();
  }

  int isMatch() throw (ParsingFatalTerminalException) {
    return 1;
  }

public:
  _AndTest(InputIterator *iterator) : Npeg(iterator) {}
};

/*
 * With all tests, also test commutativity.
 * - Does AND handle the empty string properly?
 * - Does AND only consume the string once?
 * - Does and consume nothing if one predicate is false?
 * - Does AND still work properly in the middle of a string?
 */
int main(int argc, char *argv[])
{
  const char emptystring[] = "";
  const char test1string[] = TESTSTRING1"blah";
  const char not_test1string[] = TESTSTRING2"blah";
  const char test2string[] = TESTSTRING1 TESTSTRING1 TESTSTRING2"blah";
  const char middle_test2string[] = OTHERSTRING TESTSTRING1 TESTSTRING1 TESTSTRING2"blah";
  const char errmsg[] = "some kind of error";

  StringInputIterator *p_iterator;
  _AndTest *p_context;

  p_iterator = new StringInputIterator(emptystring, 0);
  p_context = new _AndTest(p_iterator);
  assert(p_context->expression1() == 0 && p_iterator->getIndex() == 0);
  assert(p_context->expression2() == 0 && p_iterator->getIndex() == 0);
  printf("\tVerified: Handling of empty string.\n");
  delete p_context;
  delete p_iterator;

  p_iterator = new StringInputIterator(test1string, strlen(test1string));
  p_context = new _AndTest(p_iterator);
  assert(p_context->expression1() == 1 && p_iterator->getIndex() == strlen(TESTSTRING1));
  p_iterator->setIndex(0);
  assert(p_context->expression2() == 1 && p_iterator->getIndex() == strlen(TESTSTRING1));
  printf("\tVerified: Handling of single occurence string.\n");
  delete p_context;
  delete p_iterator;

  p_iterator = new StringInputIterator(not_test1string, strlen(not_test1string));
  p_context = new _AndTest(p_iterator);
  assert(p_context->expression1() == 0 && p_iterator->getIndex() == 0);
  assert(p_context->expression2() == 0 && p_iterator->getIndex() == 0);
  printf("\tVerified: Handling of single occurence string.\n");
  delete p_context;
  delete p_iterator;

  p_iterator = new StringInputIterator(test2string, strlen(test2string));
  p_context = new _AndTest(p_iterator);
  assert(p_context->expression1() == 1 && p_iterator->getIndex() == strlen(TESTSTRING1));
  p_iterator->setIndex(0);
  assert(p_context->expression2() == 1 && p_iterator->getIndex() == strlen(TESTSTRING1));
  printf("\tVerified: Handling of double occurence string.\n");
  delete p_context;
  delete p_iterator;

  p_iterator = new StringInputIterator(middle_test2string, strlen(middle_test2string));
  p_context = new _AndTest(p_iterator);
  p_iterator->setIndex(strlen(OTHERSTRING));
  assert(p_context->expression1() == 1
 && p_iterator->getIndex() == strlen(OTHERSTRING) + strlen(TESTSTRING1));
  p_iterator->setIndex(strlen(OTHERSTRING));
  assert(p_context->expression2() == 1
 && p_iterator->getIndex() == strlen(OTHERSTRING) + strlen(TESTSTRING1));
  printf("\tVerified: Handling of double occurence at center of string.\n");
  delete p_context;
  delete p_iterator;

  return 0;
}

        

Workflow with MonoDevelop

When a test fails, I generally set MonoDevelop as my GUI for GDB.

In the screenshot above:

References