jjg@1643: /*
jjg@1643:  * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
jjg@1643:  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
jjg@1643:  *
jjg@1643:  * This code is free software; you can redistribute it and/or modify it
jjg@1643:  * under the terms of the GNU General Public License version 2 only, as
jjg@1643:  * published by the Free Software Foundation.
jjg@1643:  *
jjg@1643:  * This code is distributed in the hope that it will be useful, but WITHOUT
jjg@1643:  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
jjg@1643:  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
jjg@1643:  * version 2 for more details (a copy is included in the LICENSE file that
jjg@1643:  * accompanied this code).
jjg@1643:  *
jjg@1643:  * You should have received a copy of the GNU General Public License version
jjg@1643:  * 2 along with this work; if not, write to the Free Software Foundation,
jjg@1643:  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
jjg@1643:  *
jjg@1643:  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
jjg@1643:  * or visit www.oracle.com if you need additional information or have any
jjg@1643:  * questions.
jjg@1643:  */
jjg@1643: 
jjg@1643: import java.io.*;
jjg@1643: import java.util.*;
jjg@1643: import java.lang.annotation.*;
jjg@1643: import java.lang.reflect.InvocationTargetException;
jjg@1643: 
jjg@1643: /**
jjg@1643:  * {@code Tester} is an abstract test-driver that provides the logic
jjg@1643:  * to execute test-cases, grouped by test classes.
jjg@1643:  * A test class is a main class extending this class, that instantiate
jjg@1643:  * itself, and calls the {@link run} method, passing any command line
jjg@1643:  * arguments.
jjg@1643:  * <p>
jjg@1643:  * The {@code run} method, expects arguments to identify test-case classes.
jjg@1643:  * A test-case class is a class extending the test class, and annotated
jjg@1643:  * with {@code TestCase}.
jjg@1643:  * <p>
jjg@1643:  * If no test-cases are specified, the test class directory is searched for
jjg@1643:  * co-located test-case classes (i.e. any class extending the test class,
jjg@1643:  * annotated with  {@code TestCase}).
jjg@1643:  * <p>
jjg@1643:  * Besides serving to group test-cases, extending the driver allow
jjg@1643:  * setting up a test-case template, and possibly overwrite default
jjg@1643:  * test-driver behaviour.
jjg@1643:  */
jjg@1643: public abstract class Tester {
jjg@1643: 
jjg@1643:     private static boolean debug = false;
jjg@1643:     private static final PrintStream out = System.err;
jjg@1643:     private static final PrintStream err = System.err;
jjg@1643: 
jjg@1643: 
jjg@1643:     protected void run(String... args) throws Exception {
jjg@1643: 
jjg@1643:         final File classesdir = new File(System.getProperty("test.classes", "."));
jjg@1643: 
jjg@1643:         String[] classNames = args;
jjg@1643: 
jjg@1643:         // If no test-cases are specified, we regard all co-located classes
jjg@1643:         // as potential test-cases.
jjg@1643:         if (args.length == 0) {
jjg@1643:             final String pattern =  ".*\\.class";
jjg@1643:             final File classFiles[] = classesdir.listFiles(new FileFilter() {
jjg@1643:                     public boolean accept(File f) {
jjg@1643:                         return f.getName().matches(pattern);
jjg@1643:                     }
jjg@1643:                 });
jjg@1643:             ArrayList<String> names = new ArrayList<String>(classFiles.length);
jjg@1643:             for (File f : classFiles) {
jjg@1643:                 String fname = f.getName();
jjg@1643:                 names.add(fname.substring(0, fname.length() -6));
jjg@1643:             }
jjg@1643:             classNames = names.toArray(new String[names.size()]);
jjg@1643:         } else {
jjg@1643:             debug = true;
jjg@1643:         }
jjg@1643:         // Test-cases must extend the driver type, and be marked
jjg@1643:         // @TestCase. Other arguments (classes) are ignored.
jjg@1643:         // Test-cases are instantiated, and thereby executed.
jjg@1643:         for (String clname : classNames) {
jjg@1643:             try {
jjg@1643:                 final Class tclass = Class.forName(clname);
jjg@1643:                 if  (!getClass().isAssignableFrom(tclass)) continue;
jjg@1643:                 TestCase anno = (TestCase) tclass.getAnnotation(TestCase.class);
jjg@1643:                 if (anno == null) continue;
jjg@1643:                 if (!debug) {
jjg@1643:                     ignore i = (ignore) tclass.getAnnotation(ignore.class);
jjg@1643:                     if (i != null) {
jjg@1643:                         out.println("Ignore: " + clname);
jjg@1643:                         ignored++;
jjg@1643:                         continue;
jjg@1643:                     }
jjg@1643:                 }
jjg@1643:                 out.println("TestCase: " + clname);
jjg@1643:                 cases++;
jjg@1643:                 Tester tc = (Tester) tclass.getConstructor().newInstance();
jjg@1643:                 if (tc.errors > 0) {
jjg@1643:                     error("" + tc.errors + " test points failed in " + clname);
jjg@1643:                     errors += tc.errors - 1;
jjg@1643:                     fcases++;
jjg@1643:                 }
jjg@1643:             } catch(ReflectiveOperationException roe) {
jjg@1643:                 error("Warning: " + clname + " - ReflectiveOperationException");
jjg@1643:                 roe.printStackTrace(err);
jjg@1643:             } catch(Exception unknown) {
jjg@1643:                 error("Warning: " + clname + " - uncaught exception");
jjg@1643:                 unknown.printStackTrace(err);
jjg@1643:             }
jjg@1643:         }
jjg@1643: 
jjg@1643:         String imsg = ignored > 0 ? " (" +  ignored + " ignored)" : "";
jjg@1643:         if (errors > 0)
jjg@1643:             throw new Error(errors + " error, in " + fcases + " of " + cases + " test-cases" + imsg);
jjg@1643:         else
jjg@1643:             err.println("" + cases + " test-cases executed" + imsg + ", no errors");
jjg@1643:     }
jjg@1643: 
jjg@1643: 
jjg@1643:     /**
jjg@1643:      * Test-cases must be marked with the {@code TestCase} annotation,
jjg@1643:      * as well as extend {@code Tester} (or an driver extension
jjg@1643:      * specified as the first argument to the {@code main()} method.
jjg@1643:      */
jjg@1643:     @Retention(RetentionPolicy.RUNTIME)
jjg@1643:     @interface TestCase { }
jjg@1643: 
jjg@1643:     /**
jjg@1643:      * Individual test-cases failing due to product bugs, may temporarily
jjg@1721:      * be excluded by marking them like this, (where "at-" is replaced by "@")
jjg@1721:      * at-ignore // 1234567: bug synopsis
jjg@1643:      */
jjg@1643:     @Retention(RetentionPolicy.RUNTIME)
jjg@1643:     @interface ignore { }
jjg@1643: 
jjg@1643:     /**
jjg@1643:      * Test-cases are classes extending {@code Tester}, and
jjg@1643:      * calling {@link setSrc}, followed by one or more invocations
jjg@1643:      * of {@link verify} in the body of the constructor.
jjg@1643:      * <p>
jjg@1643:      * Sets a default test-case template, which is empty except
jjg@1643:      * for a key of {@code "TESTCASE"}.
jjg@1643:      * Subclasses will typically call {@code setSrc(TestSource)}
jjg@1643:      * to setup a useful test-case template.
jjg@1643:      */
jjg@1643:     public Tester() {
jjg@1643:         this.testCase = this.getClass().getName();
jjg@1643:         src = new TestSource("TESTCASE");
jjg@1643:     }
jjg@1643: 
jjg@1643:     /**
jjg@1643:      * Set the top-level source template.
jjg@1643:      */
jjg@1643:     protected Tester setSrc(TestSource src) {
jjg@1643:         this.src = src;
jjg@1643:         return this;
jjg@1643:     }
jjg@1643: 
jjg@1643:     /**
jjg@1643:      * Convenience method for calling {@code innerSrc("TESTCASE", ...)}.
jjg@1643:      */
jjg@1643:     protected Tester setSrc(String... lines) {
jjg@1643:         return innerSrc("TESTCASE", lines);
jjg@1643:     }
jjg@1643: 
jjg@1643:     /**
jjg@1643:      * Convenience method for calling {@code innerSrc(key, new TestSource(...))}.
jjg@1643:      */
jjg@1643:     protected Tester innerSrc(String key, String... lines) {
jjg@1643:         return innerSrc(key, new TestSource(lines));
jjg@1643:     }
jjg@1643: 
jjg@1643:     /**
jjg@1643:      * Specialize the testcase template, setting replacement content
jjg@1643:      * for the specified key.
jjg@1643:      */
jjg@1643:     protected Tester innerSrc(String key, TestSource content) {
jjg@1643:         if (src == null) {
jjg@1643:             src = new TestSource(key);
jjg@1643:         }
jjg@1643:         src.setInner(key, content);
jjg@1643:         return this;
jjg@1643:     }
jjg@1643: 
jjg@1643:     /**
jjg@1643:      * On the first invocation, call {@code execute()} to compile
jjg@1643:      * the test-case source and process the resulting class(se)
jjg@1643:      * into verifiable output.
jjg@1643:      * <p>
jjg@1643:      * Verify that the output matches each of the regular expressions
jjg@1643:      * given as argument.
jjg@1643:      * <p>
jjg@1643:      * Any failure to match constitutes a test failure, but doesn't
jjg@1643:      * abort the test-case.
jjg@1643:      * <p>
jjg@1643:      * Any exception (e.g. bad regular expression syntax) results in
jjg@1643:      * a test failure, and aborts the test-case.
jjg@1643:      */
jjg@1643:     protected void verify(String... expect) {
jjg@1643:         if (!didExecute) {
jjg@1643:             try {
jjg@1643:                 execute();
jjg@1643:             } catch(Exception ue) {
jjg@1643:                 throw new Error(ue);
jjg@1643:             } finally {
jjg@1643:                 didExecute = true;
jjg@1643:             }
jjg@1643:         }
jjg@1643:         if (output == null) {
jjg@1643:             error("output is null");
jjg@1643:             return;
jjg@1643:         }
jjg@1643:         for (String e: expect) {
jjg@1643:             // Escape regular expressions (to allow input to be literals).
jjg@1643:             // Notice, characters to be escaped are themselves identified
jjg@1643:             // using regular expressions
jjg@1643:             String rc[] = { "(", ")", "[", "]", "{", "}", "$" };
jjg@1643:             for (String c : rc) {
jjg@1643:                 e = e.replace(c, "\\" + c);
jjg@1643:             }
jjg@1643:             // DEBUG: Uncomment this to test modulo constant pool index.
jjg@1643:             // e = e.replaceAll("#[0-9]{2}", "#[0-9]{2}");
jjg@1643:             if (!output.matches("(?s).*" + e + ".*")) {
jjg@1643:                 if (!didPrint) {
jjg@1643:                     out.println(output);
jjg@1643:                     didPrint = true;
jjg@1643:                 }
jjg@1643:                 error("not matched: '" + e + "'");
jjg@1643:             } else if(debug) {
jjg@1643:                 out.println("matched: '" + e + "'");
jjg@1643:             }
jjg@1643:         }
jjg@1643:     }
jjg@1643: 
jjg@1643:     /**
jjg@1643:      * Calls {@code writeTestFile()} to write out the test-case source
jjg@1643:      * content to a file, then call {@code compileTestFile()} to
jjg@1643:      * compile it, and finally run the {@link process} method to produce
jjg@1643:      * verifiable output. The default {@code process} method runs javap.
jjg@1643:      * <p>
jjg@1643:      * If an exception occurs, it results in a test failure, and
jjg@1643:      * aborts the test-case.
jjg@1643:      */
jjg@1643:     protected void execute() throws IOException {
jjg@1643:         err.println("TestCase: " + testCase);
jjg@1643:         writeTestFile();
jjg@1643:         compileTestFile();
jjg@1643:         process();
jjg@1643:     }
jjg@1643: 
jjg@1643:     /**
jjg@1643:      * Generate java source from test-case.
jjg@1643:      * TBD: change to use javaFileObject, possibly make
jjg@1643:      * this class extend JavaFileObject.
jjg@1643:      */
jjg@1643:     protected void writeTestFile() throws IOException {
jjg@1643:         javaFile = new File("Test.java");
jjg@1643:         FileWriter fw = new FileWriter(javaFile);
jjg@1643:         BufferedWriter bw = new BufferedWriter(fw);
jjg@1643:         PrintWriter pw = new PrintWriter(bw);
jjg@1643:         for (String line : src) {
jjg@1643:             pw.println(line);
jjg@1643:             if (debug) out.println(line);
jjg@1643:         }
jjg@1643:         pw.close();
jjg@1643:     }
jjg@1643: 
jjg@1643:     /**
jjg@1643:      * Compile the Java source code.
jjg@1643:      */
jjg@1643:     protected void compileTestFile() {
jjg@1643:         String path = javaFile.getPath();
jjg@1643:         String params[] =  { "-source", "1.8", "-g", path };
jjg@1643:         int rc = com.sun.tools.javac.Main.compile(params);
jjg@1643:         if (rc != 0)
jjg@1643:             throw new Error("compilation failed. rc=" + rc);
jjg@1643:         classFile = new File(path.substring(0, path.length() - 5) + ".class");
jjg@1643:     }
jjg@1643: 
jjg@1643: 
jjg@1643:     /**
jjg@1643:      * Process class file to generate output for verification.
jjg@1643:      * The default implementation simply runs javap. This might be
jjg@1643:      * overwritten to generate output in a different manner.
jjg@1643:      */
jjg@1643:     protected void process() {
jjg@1643:         String testClasses = "."; //System.getProperty("test.classes", ".");
jjg@1643:         StringWriter sw = new StringWriter();
jjg@1643:         PrintWriter pw = new PrintWriter(sw);
jjg@1643:         String[] args = { "-v", "-classpath", testClasses, "Test" };
jjg@1643:         int rc = com.sun.tools.javap.Main.run(args, pw);
jjg@1643:         if (rc != 0)
jjg@1643:             throw new Error("javap failed. rc=" + rc);
jjg@1643:         pw.close();
jjg@1643:         output = sw.toString();
jjg@1643:         if (debug) {
jjg@1643:             out.println(output);
jjg@1643:             didPrint = true;
jjg@1643:         }
jjg@1643: 
jjg@1643:     }
jjg@1643: 
jjg@1643: 
jjg@1643:     private String testCase;
jjg@1643:     private TestSource src;
jjg@1643:     private File javaFile = null;
jjg@1643:     private File classFile = null;
jjg@1643:     private String output = null;
jjg@1643:     private boolean didExecute = false;
jjg@1643:     private boolean didPrint = false;
jjg@1643: 
jjg@1643: 
jjg@1643:     protected void error(String msg) {
jjg@1643:         err.println("Error: " + msg);
jjg@1643:         errors++;
jjg@1643:     }
jjg@1643: 
jjg@1643:     private int cases;
jjg@1643:     private int fcases;
jjg@1643:     private int errors;
jjg@1643:     private int ignored;
jjg@1643: 
jjg@1643:     /**
jjg@1643:      * The TestSource class provides a simple container for
jjg@1643:      * test cases. It contains an array of source code lines,
jjg@1643:      * where zero or more lines may be markers for nested lines.
jjg@1643:      * This allows representing templates, with specialization.
jjg@1643:      * <P>
jjg@1643:      * This may be generalized to support more advance combo
jjg@1643:      * tests, but presently it's only used with a static template,
jjg@1643:      * and one level of specialization.
jjg@1643:      */
jjg@1643:     public class TestSource implements Iterable<String> {
jjg@1643: 
jjg@1643:         private String[] lines;
jjg@1643:         private Hashtable<String, TestSource> innerSrc;
jjg@1643: 
jjg@1643:         public TestSource(String... lines) {
jjg@1643:             this.lines = lines;
jjg@1643:             innerSrc = new Hashtable<String, TestSource>();
jjg@1643:         }
jjg@1643: 
jjg@1643:         public void setInner(String key, TestSource inner) {
jjg@1643:             innerSrc.put(key, inner);
jjg@1643:         }
jjg@1643: 
jjg@1643:         public void setInner(String key, String... lines) {
jjg@1643:             innerSrc.put(key, new TestSource(lines));
jjg@1643:         }
jjg@1643: 
jjg@1643:         public Iterator<String> iterator() {
jjg@1643:             return new LineIterator();
jjg@1643:         }
jjg@1643: 
jjg@1643:         private class LineIterator implements Iterator<String> {
jjg@1643: 
jjg@1643:             int nextLine = 0;
jjg@1643:             Iterator<String> innerIt = null;
jjg@1643: 
jjg@1643:             public  boolean hasNext() {
jjg@1643:                 return nextLine < lines.length;
jjg@1643:             }
jjg@1643: 
jjg@1643:             public String next() {
jjg@1643:                 if (!hasNext()) throw new NoSuchElementException();
jjg@1643:                 String str = lines[nextLine];
jjg@1643:                 TestSource inner = innerSrc.get(str);
jjg@1643:                 if (inner == null) {
jjg@1643:                     nextLine++;
jjg@1643:                     return str;
jjg@1643:                 }
jjg@1643:                 if (innerIt == null) {
jjg@1643:                     innerIt = inner.iterator();
jjg@1643:                 }
jjg@1643:                 if (innerIt.hasNext()) {
jjg@1643:                     return innerIt.next();
jjg@1643:                 }
jjg@1643:                 innerIt = null;
jjg@1643:                 nextLine++;
jjg@1643:                 return next();
jjg@1643:             }
jjg@1643: 
jjg@1643:             public void remove() {
jjg@1643:                 throw new UnsupportedOperationException();
jjg@1643:             }
jjg@1643:         }
jjg@1643:     }
jjg@1643: }