duke@1: /*
ksrini@1052:  * Copyright (c) 2002, 2011, Oracle and/or its affiliates. All rights reserved.
duke@1:  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
duke@1:  *
duke@1:  * This code is free software; you can redistribute it and/or modify it
duke@1:  * under the terms of the GNU General Public License version 2 only, as
duke@1:  * published by the Free Software Foundation.
duke@1:  *
duke@1:  * This code is distributed in the hope that it will be useful, but WITHOUT
duke@1:  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
duke@1:  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
duke@1:  * version 2 for more details (a copy is included in the LICENSE file that
duke@1:  * accompanied this code).
duke@1:  *
duke@1:  * You should have received a copy of the GNU General Public License version
duke@1:  * 2 along with this work; if not, write to the Free Software Foundation,
duke@1:  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
duke@1:  *
ohair@554:  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
ohair@554:  * or visit www.oracle.com if you need additional information or have any
ohair@554:  * questions.
duke@1:  */
duke@1: 
duke@1: import com.sun.javadoc.*;
duke@1: import java.util.*;
duke@1: import java.io.*;
duke@1: 
duke@1: 
duke@1: /**
duke@1:  * Runs javadoc and then runs regression tests on the resulting output.
duke@1:  * This class currently contains three tests:
duke@1:  * <ul>
duke@1:  * <li> String search: Reads each file, complete with newlines,
duke@1:  *      into a string.  Lets you search for strings that contain
duke@1:  *      newlines.  String matching is case-sensitive.
duke@1:  *      You can run javadoc multiple times with different arguments,
duke@1:  *      generating output into different destination directories, and
duke@1:  *      then perform a different array of tests on each one.
duke@1:  *      To do this, the run method accepts a test array for testing
duke@1:  *      that a string is found, and a negated test array for testing
duke@1:  *      that a string is not found.
duke@1:  * <li> Run diffs: Iterate through the list of given file pairs
duke@1:  *      and diff the pairs.
duke@1:  * <li> Check exit code: Check the exit code of Javadoc and
duke@1:  *      record whether the test passed or failed.
duke@1:  * </ul>
duke@1:  *
duke@1:  * @author Doug Kramer
duke@1:  * @author Jamie Ho
duke@1:  * @since 1.4.2
duke@1:  */
duke@1: public abstract class JavadocTester {
duke@1: 
duke@1:     protected static final String FS = System.getProperty("file.separator");
duke@1:     protected static final String PS = System.getProperty("path.separator");
duke@1:     protected static final String NL = System.getProperty("line.separator");
duke@1:     protected static final String SRC_DIR = System.getProperty("test.src", ".");
duke@1:     protected static final String JAVA_VERSION = System.getProperty("java.version");
duke@1:     protected static final String[][] NO_TEST = new String[][] {};
duke@1: 
duke@1:     /**
duke@1:      * Use this as the file name in the test array when you want to search
duke@1:      * for a string in the error output.
duke@1:      */
duke@1:     public static final String ERROR_OUTPUT = "ERROR_OUTPUT";
duke@1: 
duke@1:     /**
duke@1:      * Use this as the file name in the test array when you want to search
duke@1:      * for a string in the notice output.
duke@1:      */
duke@1:     public static final String NOTICE_OUTPUT = "NOTICE_OUTPUT";
duke@1: 
duke@1:     /**
duke@1:      * Use this as the file name in the test array when you want to search
duke@1:      * for a string in the warning output.
duke@1:      */
duke@1:     public static final String WARNING_OUTPUT = "WARNING_OUTPUT";
duke@1: 
duke@1:     /**
duke@1:      * Use this as the file name in the test array when you want to search
duke@1:      * for a string in standard output.
duke@1:      */
duke@1:     public static final String STANDARD_OUTPUT = "STANDARD_OUTPUT";
duke@1: 
duke@1:     /**
duke@1:      * The default doclet.
duke@1:      */
duke@1:     public static final String DEFAULT_DOCLET_CLASS = "com.sun.tools.doclets.formats.html.HtmlDoclet";
duke@1:     public static final String DEFAULT_DOCLET_CLASS_OLD = "com.sun.tools.doclets.standard.Standard";
duke@1: 
duke@1:     /**
duke@1:      * The writer to write error messages.
duke@1:      */
duke@1:     public StringWriter errors;
duke@1: 
duke@1:     /**
duke@1:      * The writer to write notices.
duke@1:      */
duke@1:     public StringWriter notices;
duke@1: 
duke@1:     /**
duke@1:      * The writer to write warnings.
duke@1:      */
duke@1:     public StringWriter warnings;
duke@1: 
duke@1:     /**
duke@1:      * The buffer of warning output..
duke@1:      */
duke@1:     public StringBuffer standardOut;
duke@1: 
duke@1:     /**
duke@1:      * The current subtest number.
duke@1:      */
duke@1:     private static int numTestsRun = 0;
duke@1: 
duke@1:     /**
duke@1:      * The number of subtests passed.
duke@1:      */
duke@1:     private static int numTestsPassed = 0;
duke@1: 
duke@1:     /**
duke@1:      * The current run of javadoc
duke@1:      */
duke@1:     private static int javadocRunNum = 0;
duke@1: 
duke@1:     /**
jjg@389:      * Whether or not to match newlines exactly.
jjg@389:      * Set this value to false if the match strings
jjg@389:      * contain text from javadoc comments containing
jjg@389:      * non-platform newlines.
jjg@389:      */
jjg@389:     protected boolean exactNewlineMatch = true;
jjg@389: 
jjg@389:     /**
duke@1:      * Construct a JavadocTester.
duke@1:      */
duke@1:     public JavadocTester() {
duke@1:     }
duke@1: 
duke@1:     /**
duke@1:      * Return the bug id.
duke@1:      * @return the bug id
duke@1:      */
duke@1:     public abstract String getBugId();
duke@1: 
duke@1:     /**
duke@1:      * Return the name of the bug.
duke@1:      * @return the name of the bug
duke@1:      */
duke@1:     public abstract String getBugName();
duke@1: 
duke@1:     /**
duke@1:      * Execute the tests.
duke@1:      *
duke@1:      * @param tester           the tester to execute
duke@1:      * @param args             the arguments to pass to Javadoc
duke@1:      * @param testArray        the array of tests
duke@1:      * @param negatedTestArray the array of negated tests
duke@1:      * @return                 the return code for the execution of Javadoc
duke@1:      */
duke@1:     public static int run(JavadocTester tester, String[] args,
duke@1:             String[][] testArray, String[][] negatedTestArray) {
duke@1:         int returnCode = tester.runJavadoc(args);
duke@1:         tester.runTestsOnHTML(testArray, negatedTestArray);
duke@1:         return returnCode;
duke@1:     }
duke@1: 
duke@1:     /**
duke@1:      * Execute Javadoc using the default doclet.
duke@1:      *
duke@1:      * @param args  the arguments to pass to Javadoc
duke@1:      * @return      the return code from the execution of Javadoc
duke@1:      */
duke@1:     public int runJavadoc(String[] args) {
duke@1:         float javaVersion = Float.parseFloat(JAVA_VERSION.substring(0,3));
duke@1:         String docletClass = javaVersion < 1.5 ?
duke@1:             DEFAULT_DOCLET_CLASS_OLD : DEFAULT_DOCLET_CLASS;
duke@1:         return runJavadoc(docletClass, args);
duke@1:     }
duke@1: 
duke@1: 
duke@1:     /**
duke@1:      * Execute Javadoc.
duke@1:      *
duke@1:      * @param docletClass the doclet being tested.
duke@1:      * @param args  the arguments to pass to Javadoc
duke@1:      * @return      the return code from the execution of Javadoc
duke@1:      */
duke@1:     public int runJavadoc(String docletClass, String[] args) {
duke@1:         javadocRunNum++;
duke@1:         if (javadocRunNum == 1) {
duke@1:             System.out.println("\n" + "Running javadoc...");
duke@1:         } else {
duke@1:             System.out.println("\n" + "Running javadoc (run "
duke@1:                                     + javadocRunNum + ")...");
duke@1:         }
duke@1:         initOutputBuffers();
duke@1: 
duke@1:         ByteArrayOutputStream stdout = new ByteArrayOutputStream();
ksrini@1052:         PrintStream prevOut = System.out;
duke@1:         System.setOut(new PrintStream(stdout));
ksrini@1052: 
ksrini@1052:         ByteArrayOutputStream stderr = new ByteArrayOutputStream();
ksrini@1052:         PrintStream prevErr = System.err;
ksrini@1052:         System.setErr(new PrintStream(stderr));
ksrini@1052: 
duke@1:         int returnCode = com.sun.tools.javadoc.Main.execute(
duke@1:                 getBugName(),
duke@1:                 new PrintWriter(errors, true),
duke@1:                 new PrintWriter(warnings, true),
duke@1:                 new PrintWriter(notices, true),
duke@1:                 docletClass,
jjg@140:                 getClass().getClassLoader(),
duke@1:                 args);
ksrini@1052:         System.setOut(prevOut);
duke@1:         standardOut = new StringBuffer(stdout.toString());
ksrini@1052:         System.setErr(prevErr);
ksrini@1052:         errors.write(NL + stderr.toString());
ksrini@1052: 
duke@1:         printJavadocOutput();
duke@1:         return returnCode;
duke@1:     }
duke@1: 
duke@1:     /**
duke@1:      * Create new string writer buffers
duke@1:      */
duke@1:     private void initOutputBuffers() {
duke@1:         errors   = new StringWriter();
duke@1:         notices  = new StringWriter();
duke@1:         warnings = new StringWriter();
duke@1:     }
duke@1: 
duke@1:     /**
duke@1:      * Run array of tests on the resulting HTML.
duke@1:      * This method accepts a testArray for testing that a string is found
duke@1:      * and a negatedTestArray for testing that a string is not found.
duke@1:      *
duke@1:      * @param testArray         the array of tests
duke@1:      * @param negatedTestArray  the array of negated tests
duke@1:      */
duke@1:     public void runTestsOnHTML(String[][] testArray, String[][] negatedTestArray) {
duke@1:         runTestsOnHTML(testArray, false);
duke@1:         runTestsOnHTML(negatedTestArray, true);
duke@1:     }
duke@1: 
duke@1:     /**
duke@1:      * Run the array of tests on the resulting HTML.
duke@1:      *
duke@1:      * @param testArray the array of tests
duke@1:      * @param isNegated true if test is negated; false otherwise
duke@1:      */
duke@1:     private void runTestsOnHTML(String[][] testArray , boolean isNegated) {
duke@1:         for (int i = 0; i < testArray.length; i++) {
duke@1: 
duke@1:             numTestsRun++;
duke@1: 
duke@1:             System.out.print("Running subtest #" + numTestsRun + "... ");
duke@1: 
duke@1:             // Get string to find
duke@1:             String stringToFind = testArray[i][1];
duke@1: 
duke@1:             // Read contents of file into a string
duke@1:             String fileString;
duke@1:             try {
duke@1:                 fileString = readFileToString(testArray[i][0]);
duke@1:             } catch (Error e) {
duke@1:                 if (isNegated) {
duke@1:                   numTestsPassed += 1;
duke@1:                   System.out.println("Passed\n not found:\n"
duke@1:                     + stringToFind + " in non-existent " + testArray[i][0] + "\n");
duke@1:                   continue;
duke@1:                 }
duke@1:                 throw e;
duke@1:             }
duke@1:             // Find string in file's contents
duke@1:             boolean isFound = findString(fileString, stringToFind);
duke@1:             if ((isNegated && !isFound) || (!isNegated && isFound) ) {
duke@1:                 numTestsPassed += 1;
duke@1:                 System.out.println( "Passed" + "\n"
duke@1:                                     + (isNegated ? "not found:" : "found:") + "\n"
duke@1:                                     + stringToFind + " in " + testArray[i][0] + "\n");
duke@1:             } else {
duke@1:                 System.out.println( "FAILED" + "\n"
duke@1:                                     + "for bug " + getBugId()
duke@1:                                     + " (" + getBugName() + ")" + "\n"
duke@1:                                     + "when searching for:" + "\n"
duke@1:                                     + stringToFind
duke@1:                                     + " in " + testArray[i][0] + "\n");
duke@1:             }
duke@1:         }
duke@1:     }
duke@1: 
duke@1:     /**
duke@1:      * Iterate through the list of given file pairs and diff each file.
duke@1:      *
duke@1:      * @param filePairs the pairs of files to diff.
duke@1:      * @throws an Error is thrown if any differences are found between
duke@1:      * file pairs.
duke@1:      */
duke@1:     public void runDiffs(String[][] filePairs) throws Error {
duke@1:         runDiffs(filePairs, true);
duke@1:     }
duke@1: 
duke@1:     /**
duke@1:      * Iterate through the list of given file pairs and diff each file.
duke@1:      *
duke@1:      * @param filePairs the pairs of files to diff.
duke@1:      * @param throwErrorIFNoMatch flag to indicate whether or not to throw
duke@1:      * an error if the files do not match.
duke@1:      *
duke@1:      * @throws an Error is thrown if any differences are found between
duke@1:      * file pairs and throwErrorIFNoMatch is true.
duke@1:      */
duke@1:     public void runDiffs(String[][] filePairs, boolean throwErrorIfNoMatch) throws Error {
duke@1:         for (int i = 0; i < filePairs.length; i++) {
duke@1:             diff(filePairs[i][0], filePairs[i][1], throwErrorIfNoMatch);
duke@1:         }
duke@1:     }
duke@1: 
duke@1:     /**
duke@1:      * Check the exit code of Javadoc and record whether the test passed
duke@1:      * or failed.
duke@1:      *
duke@1:      * @param expectedExitCode The exit code that is required for the test
duke@1:      * to pass.
duke@1:      * @param actualExitCode The actual exit code from the previous run of
duke@1:      * Javadoc.
duke@1:      */
duke@1:     public void checkExitCode(int expectedExitCode, int actualExitCode) {
duke@1:         numTestsRun++;
duke@1:         if (expectedExitCode == actualExitCode) {
duke@1:             System.out.println( "Passed" + "\n" + " got return code " +
duke@1:                 actualExitCode);
duke@1:             numTestsPassed++;
duke@1:         } else {
duke@1:             System.out.println( "FAILED" + "\n" + "for bug " + getBugId()
duke@1:                 + " (" + getBugName() + ")" + "\n" + "Expected return code " +
duke@1:                 expectedExitCode + " but got " + actualExitCode);
duke@1:         }
duke@1:     }
duke@1: 
duke@1:     /**
duke@1:      * Print a summary of the test results.
duke@1:      */
duke@1:     protected void printSummary() {
duke@1:         if ( numTestsRun != 0 && numTestsPassed == numTestsRun ) {
duke@1:             // Test passed
duke@1:             System.out.println("\n" + "All " + numTestsPassed
duke@1:                                              + " subtests passed");
duke@1:         } else {
duke@1:             // Test failed
duke@1:             throw new Error("\n" + (numTestsRun - numTestsPassed)
duke@1:                                     + " of " + (numTestsRun)
duke@1:                                     + " subtests failed for bug " + getBugId()
duke@1:                                     + " (" + getBugName() + ")" + "\n");
duke@1:         }
duke@1:     }
duke@1: 
duke@1:     /**
duke@1:      * Print the output stored in the buffers.
duke@1:      */
duke@1:     protected void printJavadocOutput() {
duke@1:         System.out.println(STANDARD_OUTPUT + " : \n" + getStandardOutput());
duke@1:         System.err.println(ERROR_OUTPUT + " : \n" + getErrorOutput());
duke@1:         System.err.println(WARNING_OUTPUT + " : \n" + getWarningOutput());
duke@1:         System.out.println(NOTICE_OUTPUT + " : \n" + getNoticeOutput());
duke@1:     }
duke@1: 
duke@1:     /**
duke@1:      * Read the file and return it as a string.
duke@1:      *
duke@1:      * @param fileName  the name of the file to read
duke@1:      * @return          the file in string format
duke@1:      */
duke@1:     public String readFileToString(String fileName) throws Error {
duke@1:         if (fileName.equals(ERROR_OUTPUT)) {
duke@1:             return getErrorOutput();
duke@1:         } else if (fileName.equals(NOTICE_OUTPUT)) {
duke@1:             return getNoticeOutput();
duke@1:         } else if (fileName.equals(WARNING_OUTPUT)) {
duke@1:             return getWarningOutput();
duke@1:         } else if (fileName.equals(STANDARD_OUTPUT)) {
duke@1:             return getStandardOutput();
duke@1:         }
duke@1:         try {
duke@1:             File file = new File(fileName);
duke@1:             if ( !file.exists() ) {
duke@1:                 System.out.println("\n" + "FILE DOES NOT EXIST: " + fileName);
duke@1:             }
duke@1:             BufferedReader in = new BufferedReader(new FileReader(file));
duke@1: 
duke@1:             // Create an array of characters the size of the file
duke@1:             char[] allChars = new char[(int)file.length()];
duke@1: 
duke@1:             // Read the characters into the allChars array
duke@1:             in.read(allChars, 0, (int)file.length());
duke@1:             in.close();
duke@1: 
duke@1:             // Convert to a string
duke@1:             String allCharsString = new String(allChars);
duke@1:             return allCharsString;
duke@1:         } catch (FileNotFoundException e) {
duke@1:             System.err.println(e);
duke@1:             throw new Error("File not found: " + fileName);
duke@1:         } catch (IOException e) {
duke@1:             System.err.println(e);
duke@1:             throw new Error("Error reading file: " + fileName);
duke@1:         }
duke@1:     }
duke@1: 
duke@1:     /**
duke@1:      * Compare the two given files.
duke@1:      *
duke@1:      * @param file1 the first file to compare.
duke@1:      * @param file2 the second file to compare.
duke@1:      * @param throwErrorIFNoMatch flag to indicate whether or not to throw
duke@1:      * an error if the files do not match.
duke@1:      * @return true if the files are the same and false otherwise.
duke@1:      */
duke@1:     public boolean diff(String file1, String file2, boolean throwErrorIFNoMatch) throws Error {
duke@1:         String file1Contents = readFileToString(file1);
duke@1:         String file2Contents = readFileToString(file2);
duke@1:         numTestsRun++;
duke@1:         if (file1Contents.trim().compareTo(file2Contents.trim()) == 0) {
duke@1:             System.out.println("Diff successful: " + file1 + ", " + file2);
duke@1:             numTestsPassed++;
duke@1:             return true;
duke@1:         } else if (throwErrorIFNoMatch) {
duke@1:             throw new Error("Diff failed: " + file1 + ", " + file2);
duke@1:         } else {
duke@1:             return false;
duke@1:         }
duke@1:     }
duke@1: 
duke@1:     /**
duke@1:      * Search for the string in the given file and return true
duke@1:      * if the string was found.
jjg@389:      * If exactNewlineMatch is false, newlines will be normalized
jjg@389:      * before the comparison.
duke@1:      *
duke@1:      * @param fileString    the contents of the file to search through
duke@1:      * @param stringToFind  the string to search for
duke@1:      * @return              true if the string was found
duke@1:      */
duke@1:     private boolean findString(String fileString, String stringToFind) {
jjg@389:         if (exactNewlineMatch) {
jjg@389:             return fileString.indexOf(stringToFind) >= 0;
jjg@389:         } else {
jjg@389:             return fileString.replace(NL, "\n").indexOf(stringToFind.replace(NL, "\n")) >= 0;
jjg@389:         }
duke@1:     }
duke@1: 
jjg@389: 
duke@1:     /**
duke@1:      * Return the standard output.
duke@1:      * @return the standard output
duke@1:      */
duke@1:     public String getStandardOutput() {
duke@1:         return standardOut.toString();
duke@1:     }
duke@1: 
duke@1:     /**
duke@1:      * Return the error output.
duke@1:      * @return the error output
duke@1:      */
duke@1:     public String getErrorOutput() {
duke@1:         return errors.getBuffer().toString();
duke@1:     }
duke@1: 
duke@1:     /**
duke@1:      * Return the notice output.
duke@1:      * @return the notice output
duke@1:      */
duke@1:     public String getNoticeOutput() {
duke@1:         return notices.getBuffer().toString();
duke@1:     }
duke@1: 
duke@1:     /**
duke@1:      * Return the warning output.
duke@1:      * @return the warning output
duke@1:      */
duke@1:     public String getWarningOutput() {
duke@1:         return warnings.getBuffer().toString();
duke@1:     }
duke@1: 
duke@1:     /**
duke@1:      * A utility to copy a directory from one place to another.
duke@1:      * We may possibly want to move this to our doclet toolkit in
duke@1:      * the near future and maintain it from there.
duke@1:      *
duke@1:      * @param targetDir the directory to copy.
duke@1:      * @param destDir the destination to copy the directory to.
duke@1:      */
duke@1:     public static void copyDir(String targetDir, String destDir) {
duke@1:         if (targetDir.endsWith("SCCS")) {
duke@1:             return;
duke@1:         }
duke@1:         try {
duke@1:             File targetDirObj = new File(targetDir);
duke@1:             File destDirParentObj = new File(destDir);
duke@1:             File destDirObj = new File(destDirParentObj, targetDirObj.getName());
duke@1:             if (! destDirParentObj.exists()) {
duke@1:                 destDirParentObj.mkdir();
duke@1:             }
duke@1:             if (! destDirObj.exists()) {
duke@1:                 destDirObj.mkdir();
duke@1:             }
duke@1:             String[] files = targetDirObj.list();
duke@1:             for (int i = 0; i < files.length; i++) {
duke@1:                 File srcFile = new File(targetDirObj, files[i]);
duke@1:                 File destFile = new File(destDirObj, files[i]);
duke@1:                 if (srcFile.isFile()) {
duke@1:                     System.out.println("Copying " + srcFile + " to " + destFile);
duke@1:                         copyFile(destFile, srcFile);
duke@1:                 } else if(srcFile.isDirectory()) {
duke@1:                     copyDir(srcFile.getAbsolutePath(), destDirObj.getAbsolutePath());
duke@1:                 }
duke@1:             }
duke@1:         } catch (IOException exc) {
duke@1:             throw new Error("Could not copy " + targetDir + " to " + destDir);
duke@1:         }
duke@1:     }
duke@1: 
duke@1:     /**
duke@1:      * Copy source file to destination file.
duke@1:      *
duke@1:      * @throws SecurityException
duke@1:      * @throws IOException
duke@1:      */
duke@1:     public static void copyFile(File destfile, File srcfile)
duke@1:         throws IOException {
duke@1:         byte[] bytearr = new byte[512];
duke@1:         int len = 0;
duke@1:         FileInputStream input = new FileInputStream(srcfile);
duke@1:         File destDir = destfile.getParentFile();
duke@1:         destDir.mkdirs();
duke@1:         FileOutputStream output = new FileOutputStream(destfile);
duke@1:         try {
duke@1:             while ((len = input.read(bytearr)) != -1) {
duke@1:                 output.write(bytearr, 0, len);
duke@1:             }
duke@1:         } catch (FileNotFoundException exc) {
duke@1:         } catch (SecurityException exc) {
duke@1:         } finally {
duke@1:             input.close();
duke@1:             output.close();
duke@1:         }
duke@1:     }
duke@1: }