Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add JUnit XML support #337

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 58 additions & 0 deletions doc/check.texi
Original file line number Diff line number Diff line change
Expand Up @@ -1930,13 +1930,17 @@ of to a file.
@findex srunner_set_xml
@findex srunner_has_xml
@findex srunner_xml_fname
@findex srunner_xml_format
@findex srunner_set_xml_format
The log can also be written in XML. The following functions define
the interface for XML logs:
@example
@verbatim
void srunner_set_xml (SRunner *sr, const char *fname);
int srunner_has_xml (SRunner *sr);
const char *srunner_xml_fname (SRunner *sr);
enum xml_format srunner_xml_format(SRunner * sr);
void srunner_set_xml_format(SRunner * sr, enum xml_format format);
@end verbatim
@end example

Expand Down Expand Up @@ -2048,6 +2052,58 @@ If both plain text and XML log files are specified, by any of above methods,
then check will log to both files. In other words logging in plain text and XML
format simultaneously is supported.

JUnit XML Support is also available. It is enabled by a call to
@code{srunner_set_xml_format(CK_XML_FORMAT_JUNIT)} before the tests are run.
It can also be enabled by setting the environment variable @code{CK_XML_FORMAT_NAME} to @code{junit}.

Here is an example of the JUnit XML format:
@example
@verbatim
<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="8" errors="1" failures="4">
<testsuite name="S1" tests="3" errors="1" failures="1">
<testcase classname="Core" name="test_pass">
</testcase>
<testcase classname="Core" name="test_fail">
<failure message="Failure">
Core:test_fail:0
ex_output.c:37
Failure
</failure>
</testcase>
<testcase classname="Core" name="test_exit">
<error message="Early exit with return value 1">
Core:test_exit:0
ex_output.c:46
Early exit with return value 1
</error>
</testcase>
</testsuite>
<testsuite name="S2" tests="4" errors="0" failures="2">
<testcase classname="Core" name="test_pass2">
</testcase>
<testcase classname="Core" name="test_loop">
<failure message="Iteration 0 failed">
Core:test_loop:0
ex_output.c:72
Iteration 0 failed
</failure>
</testcase>
<testcase classname="Core" name="test_loop">
</testcase>
<testcase classname="Core" name="test_loop">
<failure message="Iteration 2 failed">
Core:test_loop:2
ex_output.c:72
Iteration 2 failed
</failure>
</testcase>
</testsuite>
</testsuites>
@end verbatim
@end example


@node TAP Logging, , Test Logging, Test Logging
@subsection TAP Logging

Expand Down Expand Up @@ -2307,6 +2363,8 @@ CK_LOG_FILE_NAME: Filename to write logs to. See section @ref{Test Logging}.

CK_XML_LOG_FILE_NAME: Filename to write XML log to. See section @ref{XML Logging}.

CK_XML_FORMAT_NAME: Name of the XML format to use. ``junit'' will output in JUnit XML format, all other values will yield the default XML format. @ref{XML Logging}.

CK_TAP_LOG_FILE_NAME: Filename to write TAP (Test Anything Protocol) output to. See section @ref{TAP Logging}.

CK_MAX_MSG_SIZE: Maximal assertion message size.
Expand Down
7 changes: 5 additions & 2 deletions src/check.c
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ TCase *tcase_create(const char *name)
tc->unch_tflst = check_list_create();
tc->ch_tflst = check_list_create();
tc->tags = check_list_create();
tc->s = NULL;

return tc;
}
Expand Down Expand Up @@ -246,6 +247,7 @@ void suite_add_tcase(Suite * s, TCase * tc)
{
return;
}
tc->s = s;

check_list_add_end(s->tclst, tc);
}
Expand Down Expand Up @@ -412,6 +414,7 @@ SRunner *srunner_create(Suite * s)
sr->resultlst = check_list_create();
sr->log_fname = NULL;
sr->xml_fname = NULL;
sr->xml_format = CK_XML_FORMAT_UNSPECIFIED;
sr->tap_fname = NULL;
sr->loglst = NULL;

Expand Down Expand Up @@ -531,7 +534,7 @@ static void tr_init(TestResult * tr)
tr->rtype = CK_TEST_RESULT_INVALID;
tr->msg = NULL;
tr->file = NULL;
tr->tcname = NULL;
tr->tc = NULL;
tr->tname = NULL;
tr->duration = -1;
}
Expand Down Expand Up @@ -571,7 +574,7 @@ enum ck_result_ctx tr_ctx(TestResult * tr)

const char *tr_tcname(TestResult * tr)
{
return tr->tcname;
return tr->tc->name;
}

static enum fork_status _fstat = CK_FORK;
Expand Down
46 changes: 46 additions & 0 deletions src/check.h.in
Original file line number Diff line number Diff line change
Expand Up @@ -2272,6 +2272,52 @@ CK_DLL_EXP int CK_EXPORT srunner_has_tap(SRunner * sr);
*/
CK_DLL_EXP const char *CK_EXPORT srunner_tap_fname(SRunner * sr);

/**
* enum describing the specific XML format used for XML logging
*/
enum xml_format {
CK_XML_FORMAT_UNSPECIFIED,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should the unspecified state be the default state? What is the motivation for having an unspecified state? And, if it were unspecified what XML stlye ends up being used?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

CK_XML_FORMAT_UNSPECIFIED is the default value when srunner_create is called. This is in fact what I was missing and why my tests were failing.

srunner_xml_format is used to determine what format to use.
srunner_xml_format will never yield CK_XML_FORMAT_UNSPECIFIED, it will yield only CK_XML_FORMAT_DEFAULT or CK_XML_FORMAT_JUNIT.

The motivation for the unspecified state is that I needed an initial state which was unspecified since the environment variable to set the format only overrides the selected format if it's unspecified. If it were set to 'default', then I couldn't tell if someone had set it explicitly or not.

CK_XML_FORMAT_DEFAULT, // the default (original) format
CK_XML_FORMAT_JUNIT, // output in JUnit compatible XML
};

/**
* Returns the XML format used if XML is to be logged.
*
* This value can be explicitly set via `srunner_set_xml_format` or can
* be set via the CK_XML_FORMAT_NAME environment variable.
*
* This setting does not conflict with the other log output types;
* all logging types can occur concurrently if configured.
* @return CK_XML_FORMAT_DEFAULT unless the format is explicitly set via
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If another XML style were introduced in the future, the docs here would need to be updated to remove the JUNIT reference (as there would be other possibilities).

Perhaps mention how the environment variable and value set in the code interact instead, so that is more clear. For example, mention that if the environment variable is set then its value is used, otherwise whatever is set in srunner_set_xml_format is used, and the default is CK_XML_FORMAT_DEFAULT.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

addressed in 2e7563c

* `srunner_set_xml_format(sr, CK_XML_FORMAT_JUNIT)` or
* `getenv("CK_XML_FORMAT_NAME")` is set to a known value.
* Any value set explicitly via `srunner_set_xml_format` will take
* precedence over the environment variable.
*
* @param sr suite runner to check
*
* @since 0.15.3.
*/
CK_DLL_EXP enum xml_format CK_EXPORT srunner_xml_format(SRunner * sr);

/**
* Set the suite runner to output the result in an XML format compatible
* with JUnit's XML format.
*
* Note: XML format setting is an initialize only operation -- it should
* be done immediately after SRunner creation, and the XML format can't be
* changed after being set.
*
* This setting does not afffect the other log output types.
*
* @param sr suite runner to log results of in XML format
* @param format the xml_format to use
*
* @since 0.15.3
*/
CK_DLL_EXP void CK_EXPORT srunner_set_xml_format(SRunner * sr, enum xml_format format);

/**
* Enum describing the current fork usage.
*/
Expand Down
4 changes: 3 additions & 1 deletion src/check_impl.h
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ struct TCase
{
const char *name;
struct timespec timeout;
struct Suite *s;
List *tflst; /* list of test functions */
List *unch_sflst;
List *unch_tflst;
Expand All @@ -82,7 +83,7 @@ struct TestResult
int line; /* Line number where the test occurred */
int iter; /* The iteration value for looping tests */
int duration; /* duration of this test in microseconds */
const char *tcname; /* Test case that generated the result */
TCase *tc; /* Test case that generated the result */
const char *tname; /* Test that generated the result */
char *msg; /* Failure message */
};
Expand Down Expand Up @@ -121,6 +122,7 @@ struct SRunner
List *resultlst; /* List of unit test results */
const char *log_fname; /* name of log file */
const char *xml_fname; /* name of xml output file */
enum xml_format xml_format; /* the xml format to use */
const char *tap_fname; /* name of tap output file */
List *loglst; /* list of Log objects */
enum fork_status fstat; /* controls if suites are forked or not
Expand Down
88 changes: 86 additions & 2 deletions src/check_log.c
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <check.h>
#if ENABLE_SUBUNIT
#include <subunit/child.h>
Expand Down Expand Up @@ -63,6 +64,27 @@ const char *srunner_log_fname(SRunner * sr)
return getenv("CK_LOG_FILE_NAME");
}

enum xml_format srunner_xml_format(SRunner * sr)
{
// if the format as been explicitly set already via
// `srunner_set_xml_format`, then use that value
if (sr->xml_format != CK_XML_FORMAT_UNSPECIFIED)
return sr->xml_format;

// junit is the only value of CK_XML_FORMAT_NAME that will
// return something other than CK_XML_FORMAT_DEFAULT
const char *format_name = getenv("CK_XML_FORMAT_NAME");
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think in other places in the code the environment variable takes precedence. Example:

https://github.com/libcheck/check/blob/master/src/check.c#L317

Should that be the case here as well?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since it's logging, I made the precedence the same as srunner_xml_fname. This seems sane to me even if it's different than other items. These two should at least agree probably.

if (format_name && strcmp(format_name, "junit") == 0)
return CK_XML_FORMAT_JUNIT;

return CK_XML_FORMAT_DEFAULT;
}

void srunner_set_xml_format(SRunner * sr, enum xml_format format)
{
sr->xml_format = format;
}


void srunner_set_xml(SRunner * sr, const char *fname)
{
Expand Down Expand Up @@ -337,6 +359,65 @@ void xml_lfun(SRunner * sr CK_ATTRIBUTE_UNUSED, FILE * file,

}

void junit_lfun(SRunner * sr CK_ATTRIBUTE_UNUSED, FILE * file,
enum print_output printmode CK_ATTRIBUTE_UNUSED, void *obj,
enum cl_event evt)
{
// we're only interested in the end of the full run.
if (evt != CLEND_SR) return;

TestResult *tr;
Suite *s;
TestStats stats;

fprintf(file, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
fprintf(file,
"<testsuites"
" tests=\"%d\""
" errors=\"%d\""
" failures=\"%d\""
">\n",
sr->stats->n_checked, sr->stats->n_errors, sr->stats->n_failed);

// iterate over the suites
for (check_list_front(sr->slst); !check_list_at_end(sr->slst); check_list_advance(sr->slst)) {
s = (Suite*) check_list_val(sr->slst);

// calculate the stats
stats.n_checked = stats.n_errors = stats.n_failed = 0;
for (check_list_front(sr->resultlst); !check_list_at_end(sr->resultlst);
check_list_advance(sr->resultlst)) {
tr = (TestResult *)check_list_val(sr->resultlst);
if (tr->tc->s != s)
continue;
stats.n_checked++;
if (tr->rtype == CK_FAILURE)
stats.n_failed++;
else if (tr->rtype == CK_ERROR)
stats.n_errors++;
}

fprintf(file, " <testsuite name=\"");
fprint_xml_esc(file, s->name);
fprintf(file,
"\""
" tests=\"%d\""
" errors=\"%d\""
" failures=\"%d\""
">\n",
stats.n_checked, stats.n_errors, stats.n_failed);
for (check_list_front(sr->resultlst); !check_list_at_end(sr->resultlst);
check_list_advance(sr->resultlst)) {
tr = (TestResult *)check_list_val(sr->resultlst);
if (tr->tc->s != s)
continue;
tr_junitprint(file, tr, CK_VERBOSE);
}
fprintf(file, " </testsuite>\n");
}
fprintf(file, "</testsuites>\n");
}

void tap_lfun(SRunner * sr CK_ATTRIBUTE_UNUSED, FILE * file,
enum print_output printmode CK_ATTRIBUTE_UNUSED, void *obj,
enum cl_event evt)
Expand Down Expand Up @@ -372,7 +453,7 @@ void tap_lfun(SRunner * sr CK_ATTRIBUTE_UNUSED, FILE * file,
tr = (TestResult *)obj;
fprintf(file, "%s %d - %s:%s:%s: %s\n",
tr->rtype == CK_PASS ? "ok" : "not ok", num_tests_run,
tr->file, tr->tcname, tr->tname, tr->msg);
tr->file, tr->tc->name, tr->tname, tr->msg);
fflush(file);
break;
default:
Expand Down Expand Up @@ -520,7 +601,10 @@ void srunner_init_logging(SRunner * sr, enum print_output print_mode)
f = srunner_open_xmlfile(sr);
if(f)
{
srunner_register_lfun(sr, f, f != stdout, xml_lfun, print_mode);
if (srunner_xml_format(sr) == CK_XML_FORMAT_JUNIT)
srunner_register_lfun(sr, f, f != stdout, junit_lfun, print_mode);
else
srunner_register_lfun(sr, f, f != stdout, xml_lfun, print_mode);
}
f = srunner_open_tapfile(sr);
if(f)
Expand Down
3 changes: 3 additions & 0 deletions src/check_log.h
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ void lfile_lfun(SRunner * sr, FILE * file, enum print_output,
void xml_lfun(SRunner * sr, FILE * file, enum print_output,
void *obj, enum cl_event evt);

void junit_lfun(SRunner * sr, FILE * file, enum print_output,
void *obj, enum cl_event evt);

void tap_lfun(SRunner * sr, FILE * file, enum print_output,
void *obj, enum cl_event evt);

Expand Down
Loading