On Thu, Aug 24, 2006 at 02:07:51PM +0100, Daniel P. Berrange wrote:
> Quite a critical bit of the libvirt code is that which converts between
> SEXPR and XML, and vica-verca. I've broken this code several times when
> making changes, so it is way overdue to get some unit test coverage in
> this area.
Having added some unit test coverage, I was wondering just how *much*
coverage... So I've attached another patch which adds a new flag to
configure - --enable-test-coverage, which causes everything to be
compiled & linked with -fprofile-arcs -ftest-coverage
There is then a 'make cov' target in the top level directory which
genrates an HTML report showing code coverage:
http://people.redhat.com/berrange/libvirt-coverage/
Obviously it is mostly red - my goal is not to get 100% coverage - clearly
thats impossible in unit tests, since most of the XenD interaction can
only be tested through integration tests. It is however useful to be able
to view how much of the XML<->SEXPR routines are covered, since they're
easily unit testable. Also coverage on 'virsh' is useful since we can
run it with the test backend instead of xen.
The reports are generated using Perl & xsltproc - yes, this adds Perl as
a dependancy, but I don't think its a worry, because this is totally a
no-op unless you explicitly pass --enable-test-coverage to configure,
so won't impact normal 'configure; make ; make install' cycle done by
most developers.
Regards,
Dan.
--
|=- Red Hat, Engineering, Emerging Technologies, Boston. +1 978 392 2496 -=|
|=- Perl modules: http://search.cpan.org/~danberr/ -=|
|=- Projects: http://freshmeat.net/~danielpb/ -=|
|=- GnuPG: 7D3B9505 F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 -=|
diff -ruN libvirt-new/configure.in libvirt/configure.in
--- libvirt-new/configure.in 2006-08-24 10:26:46.000000000 -0400
+++ libvirt/configure.in 2006-08-24 10:21:47.000000000 -0400
@@ -246,6 +246,34 @@
AC_SUBST(PYTHON_INCLUDES)
AC_SUBST(PYTHON_SITE_PACKAGES)
+AC_ARG_ENABLE(test-coverage,
+[ --enable-test-coverage turn on code coverage instrumentation],
+[case "${enableval}" in
+ yes|no) ;;
+ *) AC_MSG_ERROR([bad value ${enableval} for test-coverage option]) ;;
+ esac],
+ [enableval=no])
+
+if test "${enableval}" = yes; then
+ COV_CFLAGS="-fprofile-arcs -ftest-coverage"
+ COV_LDFLAGS="-Wc,-fprofile-arcs -Wc,-ftest-coverage"
+ AC_MSG_CHECKING(whether compiler accepts $COV_CFLAGS)
+ ac_save_CFLAGS="$CFLAGS"
+ ac_save_LDFLAGS="$LDFLAGS"
+ CFLAGS="$CFLAGS $COV_CFLAGS"
+ LDFLAGS="$LDFLAGS $COV_LDFLAGS"
+ AC_TRY_COMPILE(,
+ [int x;],
+ AC_MSG_RESULT(yes),
+ COV_CFLAGS=""
+ COV_LDFLAGS=""
+ AC_MSG_RESULT(no))
+ CFLAGS="$ac_save_CFLAGS"
+ LDFLAGS="$ac_save_LDFLAGS"
+ AC_SUBST([COVERAGE_CFLAGS], [$COV_CFLAGS])
+ AC_SUBST([COVERAGE_LDFLAGS], [$COV_LDFLAGS])
+fi
+
# very annoying
rm -f COPYING
cp COPYING.LIB COPYING
@@ -256,4 +284,5 @@
libvirt.pc libvirt.spec \
include/libvirt/Makefile include/libvirt/libvirt.h \
python/Makefile python/tests/Makefile \
- tests/Makefile proxy/Makefile)
+ tests/Makefile proxy/Makefile \
+ utils/Makefile)
diff -ruN libvirt-new/Makefile.am libvirt/Makefile.am
--- libvirt-new/Makefile.am 2006-07-07 08:30:27.000000000 -0400
+++ libvirt/Makefile.am 2006-08-24 10:21:10.000000000 -0400
@@ -1,6 +1,6 @@
## Process this file with automake to produce Makefile.in
-SUBDIRS = src include docs @PYTHON_SUBDIR@ tests proxy
+SUBDIRS = src include docs @PYTHON_SUBDIR@ tests proxy utils
EXTRA_DIST = libvirt.spec.in libvirt.spec COPYING.LIB \
libvirt.pc.in libvirt.pc TODO AUTHORS ChangeLog \
@@ -11,6 +11,25 @@
pkgconfigdir = $(libdir)/pkgconfig
pkgconfig_DATA = libvirt.pc
+cov: cov-recursive cov-am
+
+clean-cov:
+ rm -rf coverage
+ cd src && $(MAKE) $(AM_MAKEFLAGS) clean-cov
+
+cov-recursive:
+ cd src && $(MAKE) $(AM_MAKEFLAGS) cov
+
+cov-am:
+ rm -rf coverage
+ mkdir coverage
+ perl $(srcdir)/utils/coverage-report.pl src/*.cov > coverage/index.xml
+ xsltproc $(srcdir)/utils/coverage-report.xsl coverage/index.xml \
+ > coverage/index.html
+ for i in src/*.gcov ; do o=`echo $$i | sed -e 's/src/coverage/'` ; \
+ perl $(srcdir)/utils/coverage-report-entry.pl $$i > $$o.html ; done
+
rpm: clean
@(unset CDPATH ; $(MAKE) dist && rpmbuild -ta $(distdir).tar.gz)
diff -ruN libvirt-new/src/.cvsignore libvirt/src/.cvsignore
--- libvirt-new/src/.cvsignore 2006-07-07 08:30:32.000000000 -0400
+++ libvirt/src/.cvsignore 2006-08-24 10:19:50.000000000 -0400
@@ -5,3 +5,7 @@
*.lo
*.la
virsh
+*.gcov
+*.cov
+*.gcno
+*.gcda
diff -ruN libvirt-new/src/Makefile.am libvirt/src/Makefile.am
--- libvirt-new/src/Makefile.am 2006-07-07 08:30:32.000000000 -0400
+++ libvirt/src/Makefile.am 2006-08-24 09:55:24.000000000 -0400
@@ -9,10 +9,11 @@
EXTRA_DIST = libvirt_sym.version
lib_LTLIBRARIES = libvirt.la
+libvirt_la_CFLAGS = $(COVERAGE_CFLAGS)
libvirt_la_LIBADD = @LIBXML_LIBS@
-
libvirt_la_LDFLAGS = -Wl,--version-script=$(srcdir)/libvirt_sym.version \
- -version-info @LIBVIRT_VERSION_INFO@
+ -version-info @LIBVIRT_VERSION_INFO@ \
+ $(COVERAGE_LDFLAGS)
libvirt_la_SOURCES = \
libvirt.c internal.h \
@@ -30,7 +31,22 @@
bin_PROGRAMS = virsh
virsh_SOURCES = virsh.c
-virsh_LDFLAGS =
+virsh_CFLAGS = $(COVERAGE_CFLAGS)
+virsh_LDFLAGS = $(COVERAGE_CFLAGS)
virsh_DEPENDENCIES = $(DEPS)
virsh_LDADD = $(LDADDS) $(VIRSH_LIBS)
+COVERAGE_FILES = $(libvirt_la_SOURCES:%.c=libvirt_la-%.cov) $(virsh_SOURCES:%.c=virsh-%.cov)
+
+cov: clean-cov $(COVERAGE_FILES)
+
+clean-cov:
+ rm -f *.cov *.gcov
+
+libvirt_la-%.cov: libvirt_la-%.o
+ gcov -b -f -o . $< > $@
+
+virsh-%.cov: virsh-%.o
+ gcov -b -f -o . $< > $@
+
+CLEANFILES = *.cov *.gcov .libs/*.gcda
diff -ruN libvirt-new/tests/Makefile.am libvirt/tests/Makefile.am
--- libvirt-new/tests/Makefile.am 2006-08-24 10:27:02.000000000 -0400
+++ libvirt/tests/Makefile.am 2006-08-24 10:10:39.000000000 -0400
@@ -27,19 +27,19 @@
$(top_builddir)/src/xmlrpc.c \
$(top_builddir)/src/xmlrpc.h
-xmlrpctest_LDFLAGS =
+xmlrpctest_LDFLAGS = $(COVERAGE_CFLAGS)
xmlrpctest_LDADD = $(LDADDS)
xml2sexprtest_SOURCES = \
xml2sexprtest.c \
testutils.c testutils.h
-xml2sexprtest_LDFLAGS =
+xml2sexprtest_LDFLAGS = $(COVERAGE_CFLAGS)
xml2sexprtest_LDADD = $(LDADDS)
sexpr2xmltest_SOURCES = \
sexpr2xmltest.c \
testutils.c testutils.h
-sexpr2xmltest_LDFLAGS =
+sexpr2xmltest_LDFLAGS = $(COVERAGE_CFLAGS)
sexpr2xmltest_LDADD = $(LDADDS)
$(LIBVIRT):
diff -ruN libvirt-new/utils/coverage-report-entry.pl libvirt/utils/coverage-report-entry.pl
--- libvirt-new/utils/coverage-report-entry.pl 1969-12-31 19:00:00.000000000 -0500
+++ libvirt/utils/coverage-report-entry.pl 2006-08-24 10:25:21.000000000 -0400
@@ -0,0 +1,57 @@
+#!/usr/bin/perl
+#
+# Copyright (C) 2006 Daniel P. Berrange
+#
+# See COPYING.LIB for the License of this software
+
+print <<EOF;
+<html>
+<head>
+<title>Coverage report for $ARGV[0]</title>
+<style type="text/css">
+ span.perfect {
+ background: rgb(0,255,0);
+ }
+ span.terrible {
+ background: rgb(255,0,0);
+ }
+</style>
+</head>
+<body>
+<h1>Coverage report for $ARGV[0]</h1>
+
+<pre>
+EOF
+
+
+while (<>) {
+ s/&/&/g;
+ s/</</g;
+ s/>/>/g;
+
+ if (/^\s*function (\S+) called (\d+) returned \d+% blocks executed \d+%/) {
+ my $class = $2 > 0 ? "perfect" : "terrible";
+ $_ = "<span class=\"$class\" id=\"" . $1 . "\">$_</span>";
+ } elsif (/^\s*branch\s+\d+\s+taken\s+(\d+)%\s+.*$/) {
+ my $class = $1 > 0 ? "perfect" : "terrible";
+ $_ = "<span class=\"$class\">$_</span>";
+ } elsif (/^\s*branch\s+\d+\s+never executed.*$/) {
+ my $class = "terrible";
+ $_ = "<span class=\"$class\">$_</span>";
+ } elsif (/^\s*call\s+\d+\s+never executed.*$/) {
+ my $class = "terrible";
+ $_ = "<span class=\"$class\">$_</span>";
+ } elsif (/^\s*call\s+\d+\s+returned\s+(\d+)%.*$/) {
+ my $class = $1 > 0 ? "perfect" : "terrible";
+ $_ = "<span class=\"$class\">$_</span>";
+ }
+
+
+ print;
+}
+
+print <<EOF;
+</pre>
+</body>
+</html>
+EOF
diff -ruN libvirt-new/utils/coverage-report.pl libvirt/utils/coverage-report.pl
--- libvirt-new/utils/coverage-report.pl 1969-12-31 19:00:00.000000000 -0500
+++ libvirt/utils/coverage-report.pl 2006-08-24 10:25:35.000000000 -0400
@@ -0,0 +1,113 @@
+#!/usr/bin/perl
+#
+# Copyright (C) 2006 Daniel P. Berrange
+#
+# See COPYING.LIB for the License of this software
+
+use warnings;
+use strict;
+
+my %coverage = ( functions => {}, files => {} );
+
+my %filemap;
+
+my $type;
+my $name;
+
+my @functions;
+
+while (<>) {
+ if (/^Function '(.*)'\s*$/) {
+ $type = "function";
+ $name = $1;
+ $coverage{$type}->{$name} = {};
+ push @functions, $name;
+ } elsif (/^File '(.*?)'\s*$/) {
+ $type = "file";
+ $name = $1;
+ $coverage{$type}->{$name} = {};
+
+ foreach my $func (@functions) {
+ $coverage{"function"}->{$func}->{file} = $name;
+ }
+ @functions = ();
+ } elsif (/^Lines executed:(.*)%\s*of\s*(\d+)\s*$/) {
+ $coverage{$type}->{$name}->{lines} = $2;
+ $coverage{$type}->{$name}->{linesCoverage} = $1;
+ } elsif (/^Branches executed:(.*)%\s*of\s*(\d+)\s*$/) {
+ $coverage{$type}->{$name}->{branches} = $2;
+ $coverage{$type}->{$name}->{branchesCoverage} = $1;
+ } elsif (/^Taken at least once:(.*)%\s*of\s*(\d+)\s*$/) {
+ $coverage{$type}->{$name}->{conds} = $2;
+ $coverage{$type}->{$name}->{condsCoverage} = $1;
+ } elsif (/^Calls executed:(.*)%\s*of\s*(\d+)\s*$/) {
+ $coverage{$type}->{$name}->{calls} = $2;
+ $coverage{$type}->{$name}->{callsCoverage} = $1;
+ } elsif (/^No branches$/) {
+ $coverage{$type}->{$name}->{branches} = 0;
+ $coverage{$type}->{$name}->{branchesCoverage} = "100.00";
+ $coverage{$type}->{$name}->{conds} = 0;
+ $coverage{$type}->{$name}->{condsCoverage} = "100.00";
+ } elsif (/^No calls$/) {
+ $coverage{$type}->{$name}->{calls} = 0;
+ $coverage{$type}->{$name}->{callsCoverage} = "100.00";
+ } elsif (/^\s*(.*):creating '(.*)'\s*$/) {
+ $filemap{$1} = $2;
+ } elsif (/^\s*$/) {
+ # nada
+ } else {
+ warn "Shit [$_]\n";
+ }
+}
+
+my %summary;
+foreach my $type ("function", "file") {
+ $summary{$type} = {};
+ foreach my $m ("lines", "branches", "conds", "calls") {
+ my $totalGot = 0;
+ my $totalMiss = 0;
+ my $count = 0;
+ foreach my $func (keys %{$coverage{function}}) {
+ $count++;
+ my $got = $coverage{function}->{$func}->{$m};
+ $totalGot += $got;
+ my $miss = $got * $coverage{function}->{$func}->{$m ."Coverage"} / 100;
+ $totalMiss += $miss;
+ }
+ $summary{$type}->{$m} = sprintf("%d", $totalGot);
+ $summary{$type}->{$m . "Coverage"} = sprintf("%.2f", $totalMiss / $totalGot * 100);
+ }
+}
+
+
+
+print "<coverage>\n";
+
+foreach my $type ("function", "file") {
+ printf "<%ss>\n", $type;
+ foreach my $name (sort { $a cmp $b } keys %{$coverage{$type}}) {
+ my $rec = $coverage{$type}->{$name};
+ printf " <entry name=\"%s\" details=\"%s\">\n", $name, ($type eq "file" ? $filemap{$name} : $filemap{$rec->{file}});
+ printf " <lines count=\"%s\" coverage=\"%s\"/>\n", $rec->{lines}, $rec->{linesCoverage};
+ if (exists $rec->{branches}) {
+ printf " <branches count=\"%s\" coverage=\"%s\"/>\n", $rec->{branches}, $rec->{branchesCoverage};
+ }
+ if (exists $rec->{conds}) {
+ printf " <conditions count=\"%s\" coverage=\"%s\"/>\n", $rec->{conds}, $rec->{condsCoverage};
+ }
+ if (exists $rec->{calls}) {
+ printf " <calls count=\"%s\" coverage=\"%s\"/>\n", $rec->{calls}, $rec->{callsCoverage};
+ }
+ print " </entry>\n";
+ }
+
+ printf " <summary>\n";
+ printf " <lines count=\"%s\" coverage=\"%s\"/>\n", $summary{$type}->{lines}, $summary{$type}->{linesCoverage};
+ printf " <branches count=\"%s\" coverage=\"%s\"/>\n", $summary{$type}->{branches}, $summary{$type}->{branchesCoverage};
+ printf " <conditions count=\"%s\" coverage=\"%s\"/>\n", $summary{$type}->{conds}, $summary{$type}->{condsCoverage};
+ printf " <calls count=\"%s\" coverage=\"%s\"/>\n", $summary{$type}->{calls}, $summary{$type}->{callsCoverage};
+ printf " </summary>\n";
+ printf "</%ss>\n", $type;
+}
+
+print "</coverage>\n";
diff -ruN libvirt-new/utils/coverage-report.xsl libvirt/utils/coverage-report.xsl
--- libvirt-new/utils/coverage-report.xsl 1969-12-31 19:00:00.000000000 -0500
+++ libvirt/utils/coverage-report.xsl 2006-08-24 09:10:46.000000000 -0400
@@ -0,0 +1,217 @@
+<?xml version="1.0" encoding="utf-8"?>
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform";
+ version="1.0">
+
+ <xsl:output method="html"/>
+
+ <xsl:template match="coverage">
+ <html>
+ <head>
+ <title>Coverage report</title>
+ <style type="text/css">
+ tbody tr.odd td.label {
+ border-top: 1px solid rgb(128,128,128);
+ border-bottom: 1px solid rgb(128,128,128);
+ }
+ tbody tr.odd td.label {
+ background: rgb(200,200,200);
+ }
+
+ thead, tfoot {
+ background: rgb(60,60,60);
+ color: white;
+ font-weight: bold;
+ }
+
+ tr td.perfect {
+ background: rgb(0,255,0);
+ color: black;
+ }
+ tr td.excellant {
+ background: rgb(140,255,140);
+ color: black;
+ }
+ tr td.good {
+ background: rgb(160,255,0);
+ color: black;
+ }
+ tr td.poor {
+ background: rgb(255,160,0);
+ color: black;
+ }
+ tr td.bad {
+ background: rgb(255,140,140);
+ color: black;
+ }
+ tr td.terrible {
+ background: rgb(255,0,0);
+ color: black;
+ }
+ </style>
+ </head>
+ <body>
+ <h1>Coverage report</h1>
+ <xsl:apply-templates/>
+ </body>
+ </html>
+ </xsl:template>
+
+ <xsl:template match="functions">
+ <h2>Function coverage</h2>
+ <xsl:call-template name="content">
+ <xsl:with-param name="type" select="'function'"/>
+ </xsl:call-template>
+ </xsl:template>
+
+
+ <xsl:template match="files">
+ <h2>File coverage</h2>
+ <xsl:call-template name="content">
+ <xsl:with-param name="type" select="'file'"/>
+ </xsl:call-template>
+ </xsl:template>
+
+ <xsl:template name="content">
+ <xsl:param name="type"/>
+ <table>
+ <thead>
+ <tr>
+ <th>Name</th>
+ <th>Lines</th>
+ <th>Branches</th>
+ <th>Conditions</th>
+ <th>Calls</th>
+ </tr>
+ </thead>
+ <tbody>
+ <xsl:for-each select="entry">
+ <xsl:call-template name="entry">
+ <xsl:with-param name="type" select="$type"/>
+ <xsl:with-param name="class">
+ <xsl:choose>
+ <xsl:when test="position() mod 2">
+ <xsl:text>odd</xsl:text>
+ </xsl:when>
+ <xsl:otherwise>
+ <xsl:text>even</xsl:text>
+ </xsl:otherwise>
+ </xsl:choose>
+ </xsl:with-param>
+ </xsl:call-template>
+ </xsl:for-each>
+ </tbody>
+ <tfoot>
+ <xsl:for-each select="summary">
+ <xsl:call-template name="entry">
+ <xsl:with-param name="type" select="'summary'"/>
+ <xsl:with-param name="class">
+ <xsl:choose>
+ <xsl:when test="position() mod 2">
+ <xsl:text>odd</xsl:text>
+ </xsl:when>
+ <xsl:otherwise>
+ <xsl:text>even</xsl:text>
+ </xsl:otherwise>
+ </xsl:choose>
+ </xsl:with-param>
+ </xsl:call-template>
+ </xsl:for-each>
+ </tfoot>
+ </table>
+ </xsl:template>
+
+ <xsl:template name="entry">
+ <xsl:param name="type"/>
+ <xsl:param name="class"/>
+ <tr class="{$class}">
+ <xsl:choose>
+ <xsl:when test="$type = 'function'">
+ <td class="label"><a href="{@details}.html#{@name}"><xsl:value-of select="@name"/></a></td>
+ </xsl:when>
+ <xsl:when test="$type = 'file'">
+ <td class="label"><a href="{@details}.html"><xsl:value-of select="@name"/></a></td>
+ </xsl:when>
+ <xsl:otherwise>
+ <td class="label">Summary</td>
+ </xsl:otherwise>
+ </xsl:choose>
+
+ <xsl:if test="count(lines)">
+ <xsl:apply-templates select="lines"/>
+ </xsl:if>
+ <xsl:if test="not(count(lines))">
+ <xsl:call-template name="missing"/>
+ </xsl:if>
+
+ <xsl:if test="count(branches)">
+ <xsl:apply-templates select="branches"/>
+ </xsl:if>
+ <xsl:if test="not(count(branches))">
+ <xsl:call-template name="missing"/>
+ </xsl:if>
+
+ <xsl:if test="count(conditions)">
+ <xsl:apply-templates select="conditions"/>
+ </xsl:if>
+ <xsl:if test="not(count(conditions))">
+ <xsl:call-template name="missing"/>
+ </xsl:if>
+
+ <xsl:if test="count(calls)">
+ <xsl:apply-templates select="calls"/>
+ </xsl:if>
+ <xsl:if test="not(count(calls))">
+ <xsl:call-template name="missing"/>
+ </xsl:if>
+
+ </tr>
+ </xsl:template>
+
+ <xsl:template match="lines">
+ <xsl:call-template name="row"/>
+ </xsl:template>
+
+ <xsl:template match="branches">
+ <xsl:call-template name="row"/>
+ </xsl:template>
+
+ <xsl:template match="conditions">
+ <xsl:call-template name="row"/>
+ </xsl:template>
+
+ <xsl:template match="calls">
+ <xsl:call-template name="row"/>
+ </xsl:template>
+
+ <xsl:template name="missing">
+ <td></td>
+ </xsl:template>
+
+ <xsl:template name="row">
+ <xsl:variable name="quality">
+ <xsl:choose>
+ <xsl:when test="@coverage = 100">
+ <xsl:text>perfect</xsl:text>
+ </xsl:when>
+ <xsl:when test="@coverage >= 80.0">
+ <xsl:text>excellant</xsl:text>
+ </xsl:when>
+ <xsl:when test="@coverage >= 60.0">
+ <xsl:text>good</xsl:text>
+ </xsl:when>
+ <xsl:when test="@coverage >= 40.0">
+ <xsl:text>poor</xsl:text>
+ </xsl:when>
+ <xsl:when test="@coverage >= 20.0">
+ <xsl:text>bad</xsl:text>
+ </xsl:when>
+ <xsl:otherwise>
+ <xsl:text>terrible</xsl:text>
+ </xsl:otherwise>
+ </xsl:choose>
+ </xsl:variable>
+
+ <td class="{$quality}"><xsl:value-of select="@coverage"/>% of <xsl:value-of select="@count"/></td>
+ </xsl:template>
+
+</xsl:stylesheet>
diff -ruN libvirt-new/utils/Makefile.am libvirt/utils/Makefile.am
--- libvirt-new/utils/Makefile.am 1969-12-31 19:00:00.000000000 -0500
+++ libvirt/utils/Makefile.am 2006-08-24 10:22:52.000000000 -0400
@@ -0,0 +1,3 @@
+
+EXTRA_DIST = $(wildcard *.xsl) $(wildcard *.pl)
+