Hi,
(I'm CC'ing classpath@xxxxxxx as this might be of more general interest)
This is a first implementation for visual interactive tests in Mauve.
This allows to write testcases that require human interaction, for
example:
- to test specific rendering issues of Swing components
- for complex issues that are not easily transformed in automated
testcases, but are quite easy to check interactivly
- quickly transform Swing testprograms from the bugdatabase into
regression tests
- and many more
In order to write such an interactive test, subclass from
gnu.testlet.VisualTestlet and implement the abstract methods.
To run interactive tests:
java Harness -interactive <tests>
The -interactive flag tells the TestRunner to only perform interactive
tests, leaving this flag away only performs only non-interactive tests.
The test asks the tester on the console if the test passed or failed. I
think that this is better than additional GUIness, because we are
testing the GUI here, and need to run with a minimum of additional GUI
stuff. This is slightly inconvenient, as it requires the tester to
change between a Terminal and the test window.
This adds an example test that I've written for a bug that I fixed this
morning.
Comments and improvements are welcome.
2006-10-03 Roman Kennke <kennke@xxxxxxxxx>
* Harness.java
(InputPiperThread): New inner class. Forwards the input from
the outside process to the inside (test) process.
(initProcess): Set up piping.
(printHelpMessage): Added description of -interactive option.
* RunnerProcess.java
(interactive): New field. This is set to true when we are running
interactive tests only.
(main): Interpret -interactive option.
(runtest): Filter tests based on the -interactive flag.
* gnu/testlet/VisualTestlet.java: New class. This is the
base class for visual (interactive) tests.
* gnu/testlet/javax/swing/JMenuItem/DragSelectTest.java: New
interactive test.
/Roman
Index: Harness.java
===================================================================
RCS file: /cvs/mauve/mauve/Harness.java,v
retrieving revision 1.25
diff -u -1 -5 -r1.25 Harness.java
--- Harness.java 16 Aug 2006 19:00:18 -0000 1.25
+++ Harness.java 13 Oct 2006 11:44:53 -0000
@@ -571,32 +571,33 @@
"tests before running them. This" + align + "overrides the configure" +
"option --disable-auto-compilation but requires an ecj jar" + align +
"to be in /usr/share/java/eclipse-ecj.jar or specified via the " +
"--with-ecj-jar" + align + "option to configure. See the README" +
" file for more details.\n" +
" -timeout [millis]: specifies a timeout value for the tests " +
"(default is 60000 milliseconds)" +
// Testcase Selection Options.
"\n\nTestcase Selection Options:\n" +
" -exclude [test|folder]: specifies a test or a folder to exclude " +
"from the run\n" +
" -norecursion: if a folder is specified to be run, don't " +
"run the tests in its subfolders\n" +
" -file [filename]: specifies a file that contains the names " +
- "of tests to be run (one per line)" +
-
+ "of tests to be run (one per line)\n" +
+ " -interactive: only run interavtice tests, if not set, " +
+ "only run non-interactive tests\n" +
// Output Options.
"\n\nOutput Options:\n" +
" -showpasses: display passing tests as well as failing " +
"ones\n" +
" -hidecompilefails: hide errors from the compiler when " +
"tests fail to compile\n" +
" -noexceptions: suppress stack traces for uncaught " +
"exceptions\n" +
" -verbose: run in noisy mode, displaying extra " +
"information\n" +
" -debug: displays some extra information for " +
"failing tests that " +
"use the" + align + "harness.check(Object, Object) method\n" +
" -xmlout [filename]: specifies a file to use for xml output\n" +
"\nOther Options:\n" +
@@ -632,30 +633,33 @@
{
StringBuffer sb = new StringBuffer(" RunnerProcess");
for (int i = 0; i < args.length; i++)
sb.append(" " + args[i]);
sb.insert(0, vmCommand + vmArgs);
try
{
// Exec the process and set up in/out communications with it.
runnerProcess = Runtime.getRuntime().exec(sb.toString());
runner_out = new PrintWriter(runnerProcess.getOutputStream(), true);
runner_in =
new BufferedReader
(new InputStreamReader(runnerProcess.getInputStream()));
runner_esp = new ErrorStreamPrinter(runnerProcess.getErrorStream());
+ InputPiperThread pipe = new InputPiperThread(System.in,
+ runnerProcess.getOutputStream());
+ pipe.start();
runner_esp.start();
}
catch (IOException e)
{
System.err.println("Problems invoking RunnerProcess: " + e);
System.exit(1);
}
// Create a timer to watch this new process. After confirming the
// RunnerProcess started properly, we create a new runner_watcher
// because it may be a while before we run the next test (due to
// preprocessing and compilation) and we don't want the runner_watcher
// to time out.
if (runner_watcher != null)
@@ -1563,16 +1567,59 @@
* printed out and also so that if the output is verbose we print
* our own summary.
*/
public void print(String x)
{
if (isCompileSummary(x))
{
if (verbose)
super.println("TEST FAILED: compile failed for "
+ lastFailingCompile);
}
else
super.print(x);
}
}
+
+ /**
+ * Reads from one stream and writes this to another. This is used to pipe
+ * the input (System.in) from the outside process to the test process.
+ */
+ private static class InputPiperThread
+ extends Thread
+ {
+ InputStream in;
+ OutputStream out;
+ InputPiperThread(InputStream i, OutputStream o)
+ {
+ in = i;
+ out = o;
+ }
+ public void run()
+ {
+ int ch = 0;
+ do
+ {
+ try
+ {
+ if (in.available() > 0)
+ {
+ ch = in.read();
+ if (ch != '\n') // Skip the trailing newline.
+ out.write(ch);
+ out.flush();
+ }
+ else
+ Thread.sleep(200);
+ }
+ catch (IOException ex)
+ {
+ ex.printStackTrace();
+ }
+ catch (InterruptedException ex)
+ {
+ ch = -1; // Jump outside.
+ }
+ } while (ch != -1);
+ }
+ }
}
Index: RunnerProcess.java
===================================================================
RCS file: /cvs/mauve/mauve/RunnerProcess.java,v
retrieving revision 1.14
diff -u -1 -5 -r1.14 RunnerProcess.java
--- RunnerProcess.java 4 Aug 2006 20:30:09 -0000 1.14
+++ RunnerProcess.java 13 Oct 2006 11:44:53 -0000
@@ -17,53 +17,54 @@
// You should have received a copy of the GNU General Public License
// along with Mauve; see the file COPYING. If not, write to
// the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
// Boston, MA 02110-1301 USA.
// This file is used by Harness.java to run the tests in a separate process
// so that the process can be killed by the Harness if it is hung.
import gnu.testlet.ResourceNotFoundException;
import gnu.testlet.TestHarness;
import gnu.testlet.TestReport;
import gnu.testlet.TestResult;
import gnu.testlet.TestSecurityManager;
import gnu.testlet.Testlet;
+import gnu.testlet.VisualTestlet;
import gnu.testlet.config;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.Vector;
public class RunnerProcess
extends TestHarness
{
// A description of files that are not tests
private static final String NOT_A_TEST_DESCRIPTION = "not-a-test";
-
+
// A description of files that fail to load
private static final String FAIL_TO_LOAD_DESCRIPTION = "failed-to-load";
// A description of a test that throws an uncaught exception
private static final String UNCAUGHT_EXCEPTION_DESCRIPTION = "uncaught-exception";
// Total number of harness.check calls since the last checkpoint
private int count = 0;
// The location of the emma.jar file
private static String emmaJarLocation = null;
// Whether or not to use EMMA
private static boolean useEMMA = true;
@@ -100,31 +101,36 @@
// The TestReport if a report is necessary
private static TestReport report = null;
// The xmlfile for the report
private static String xmlfile = null;
// The result of the current test
private TestResult currentResult = null;
// The EMMA forced data dump method
private static Method emmaMethod = null;
// The failure message for the last failing check()
private String lastFailureMessage = null;
-
+
+ /**
+ * Should we run interactive or non-interactive tests ?
+ */
+ private static boolean interactive;
+
protected RunnerProcess()
{
try
{
BufferedReader xfile = new BufferedReader(new FileReader("xfails"));
String str;
while ((str = xfile.readLine()) != null)
{
expected_xfails.addElement(str);
}
}
catch (FileNotFoundException ex)
{
// Nothing.
}
@@ -159,30 +165,32 @@
{
// User wants a report.
if (++i >= args.length)
throw new RuntimeException("No file path after '-xmlout'.");
xmlfile = args[i];
}
else if (args[i].equalsIgnoreCase("-emma"))
{
// User is specifying the location of the eclipse-ecj.jar file
// to use for compilation.
if (++i >= args.length)
throw new RuntimeException("No file path " +
"after '-emma'. Exit");
emmaJarLocation = args[i];
}
+ else if (args[i].equals("-interactive"))
+ interactive = true;
}
// If the user wants an xml report, create a new TestReport.
if (xmlfile != null)
{
report = new TestReport(System.getProperties());
}
// Setup the data coverage dumping mechanism. The default configuration
// is to auto-detect EMMA, meaning if the emma classes are found on the
// classpath then we should force a dump of coverage data. Also, the user
// can configure with -with-emma=JARLOCATION or can specify -emma
// JARLOCATION on the command line to explicitly specify an emma.jar to use
// to dump coverage data.
if (emmaJarLocation == null)
emmaJarLocation = config.emmaString;
@@ -256,31 +264,46 @@
{
Class k = Class.forName(name);
int mods = k.getModifiers();
if (Modifier.isAbstract(mods))
{
description = NOT_A_TEST_DESCRIPTION;
return;
}
Object o = k.newInstance();
if (! (o instanceof Testlet))
{
description = NOT_A_TEST_DESCRIPTION;
return;
}
-
+ if (o instanceof VisualTestlet)
+ {
+ if (! interactive)
+ {
+ description = NOT_A_TEST_DESCRIPTION;
+ return;
+ }
+ }
+ else
+ {
+ if (interactive)
+ {
+ description = NOT_A_TEST_DESCRIPTION;
+ return;
+ }
+ }
t = (Testlet) o;
}
catch (Throwable ex)
{
description = FAIL_TO_LOAD_DESCRIPTION;
// Maybe the file was marked not-a-test, check that before we report
// it as an error
try
{
File f = new File(name.replace('.', File.separatorChar) + ".java");
BufferedReader r = new BufferedReader(new FileReader(f));
String firstLine = r.readLine();
// Since some people mistakenly put not-a-test not as the first line
// we have to check through the file.
while (firstLine != null)
Index: gnu/testlet/VisualTestlet.java
===================================================================
RCS file: gnu/testlet/VisualTestlet.java
diff -N gnu/testlet/VisualTestlet.java
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ gnu/testlet/VisualTestlet.java 13 Oct 2006 11:44:54 -0000
@@ -0,0 +1,131 @@
+/* VisualTestlet.java -- Abstract superclass for visual tests
+ Copyright (C) 2006 Roman Kennke (kennke@xxxxxxxxx)
+This file is part of Mauve.
+
+Mauve is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2, or (at your option)
+any later version.
+
+Mauve is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with Mauve; see the file COPYING. If not, write to the
+Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+02110-1301 USA.
+
+*/
+
+package gnu.testlet;
+
+import java.awt.Component;
+import java.awt.Frame;
+import java.io.IOException;
+
+import javax.swing.JComponent;
+import javax.swing.JFrame;
+
+/**
+ * Provides an environment for visual tests. Visual tests must provide a
+ * component, instructions and the expected results. The harness provides
+ * all three to the tester and ask if the test passed or not.
+ *
+ * The test component is displayed inside a AWT Frame or a Swing JFrame
+ * (depending on the type of the component). This means that the tested
+ * Java environment needs to have some basic AWT or Swing functionality. This
+ * should be covered by other tests (possibly by java.awt.Robot or so).
+ */
+public abstract class VisualTestlet
+ implements Testlet
+{
+
+ /**
+ * Starts the test.
+ *
+ * @param h the test harness
+ */
+ public void test(TestHarness h)
+ {
+ // Initialize and show test component.
+ Component c = getTestComponent();
+ Frame f;
+ if (c instanceof JComponent)
+ {
+ JFrame jFrame = new JFrame();
+ jFrame.setContentPane((JComponent) c);
+ f = jFrame;
+ }
+ else
+ {
+ f = new Frame();
+ f.add(c);
+ }
+ f.pack();
+ f.setVisible(true);
+
+ // Print instructions and expected results on console.
+ System.out.println("====================================================");
+ System.out.print("This is a test that needs human interaction. Please ");
+ System.out.print("read the instructions carefully and follow them. ");
+ System.out.print("Then check if your results match the expected results. ");
+ System.out.print("Type p <ENTER> if the test showed the expected results,");
+ System.out.println(" f <ENTER> otherwise.");
+ System.out.println("====================================================");
+ System.out.println("INSTRUCTIONS:");
+ System.out.println(getInstructions());
+ System.out.println("====================================================");
+ System.out.println("EXPECTED RESULTS:");
+ System.out.println(getExpectedResults());
+ System.out.println("====================================================");
+
+ // Ask the tester whether the test passes or fails.
+ System.out.println("(P)ASS or (F)AIL ?");
+ while (true)
+ {
+ int ch;
+ try
+ {
+ ch = System.in.read();
+ if (ch == 'P' || ch == 'p')
+ {
+ h.check(true);
+ break;
+ }
+ else if (ch == 'f' || ch == 'F')
+ {
+ h.check(false);
+ break;
+ }
+ }
+ catch (IOException ex)
+ {
+ h.debug(ex);
+ h.fail("Unexpected IO problem on console");
+ }
+ }
+ }
+
+ /**
+ * Provides the instructions for the test.
+ *
+ * @return the instructions for the test
+ */
+ public abstract String getInstructions();
+
+ /**
+ * Describes the expected results.
+ *
+ * @return the expected results
+ */
+ public abstract String getExpectedResults();
+
+ /**
+ * Provides the test component.
+ *
+ * @return the test component
+ */
+ public abstract Component getTestComponent();
+}
Index: gnu/testlet/javax/swing/JMenuItem/DragSelectTest.java
===================================================================
RCS file: gnu/testlet/javax/swing/JMenuItem/DragSelectTest.java
diff -N gnu/testlet/javax/swing/JMenuItem/DragSelectTest.java
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ gnu/testlet/javax/swing/JMenuItem/DragSelectTest.java 13 Oct 2006 11:44:54 -0000
@@ -0,0 +1,84 @@
+/* DragSelectTest.java -- Tests if drag selection works
+ Copyright (C) 2006 Roman Kennke (kennke@xxxxxxxxx)
+This file is part of Mauve.
+
+Mauve is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2, or (at your option)
+any later version.
+
+Mauve is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with Mauve; see the file COPYING. If not, write to the
+Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+02110-1301 USA.
+
+*/
+
+// Tags: JDK1.2 manual
+
+package gnu.testlet.javax.swing.JMenuItem;
+
+import java.awt.Component;
+import java.awt.event.ActionEvent;
+import java.awt.event.ActionListener;
+
+import javax.swing.JLabel;
+import javax.swing.JMenu;
+import javax.swing.JMenuBar;
+import javax.swing.JMenuItem;
+import javax.swing.JRootPane;
+
+import gnu.testlet.VisualTestlet;
+
+public class DragSelectTest extends VisualTestlet
+{
+ public String getInstructions()
+ {
+ return "Press the mouse on the 'Menu' menu, hold the button pressed and "
+ + "drag it to one of the menu items. Then release the mouse "
+ + "button";
+ }
+
+ public String getExpectedResults()
+ {
+ return "The menu should be closed and the name of the menu item shown in "
+ + "the panel below";
+ }
+
+ public Component getTestComponent()
+ {
+ JRootPane rp = new JRootPane();
+ JMenuBar mb = new JMenuBar();
+ JMenu menu = new JMenu("Menu");
+ final JLabel label =
+ new JLabel("The selected menu item should show up here");
+ ActionListener l = new ActionListener()
+ {
+ public void actionPerformed(ActionEvent ev)
+ {
+ JMenuItem i = (JMenuItem) ev.getSource();
+ label.setText(i.getText());
+ }
+ };
+
+ JMenuItem item1 = new JMenuItem("MenuItem 1");
+ item1.addActionListener(l);
+ JMenuItem item2 = new JMenuItem("MenuItem 2");
+ item2.addActionListener(l);
+ JMenuItem item3 = new JMenuItem("MenuItem 3");
+ item3.addActionListener(l);
+ menu.add(item1);
+ menu.add(item2);
+ menu.add(item3);
+ mb.add(menu);
+ rp.setJMenuBar(mb);
+ rp.getContentPane().add(label);
+ return rp;
+ }
+
+}
