jdk8-mips64-public/hotspot: changeset 1314:494244ae0171

     1.1 --- a/.hgtags	Mon Jul 27 09:06:22 2009 -0700
     1.2 +++ b/.hgtags	Mon Jul 27 17:23:52 2009 -0400
     1.3 @@ -35,3 +35,7 @@
     1.4  53d9bf689e80fcc76b221bbe6c5d58e08b80cbc6 jdk7-b58
     1.5  c55be0c7bd32c016c52218eb4c8b5da8a75450b5 jdk7-b59
     1.6  a77eddcd510c3972717c025cfcef9a60bfa4ecac jdk7-b60
     1.7 +27b728fd1281ab62e9d7e4424f8bbb6ca438d803 jdk7-b61
     1.8 +a88386380bdaaa5ab4ffbedf22c57bac5dbec034 jdk7-b62
     1.9 +32c83fb84370a35344676991a48440378e6b6c8a jdk7-b63
    1.10 +ba36394eb84b949b31212bdb32a518a8f92bab5b jdk7-b64

     2.1 --- a/agent/src/share/classes/sun/jvm/hotspot/code/DebugInfoReadStream.java	Mon Jul 27 09:06:22 2009 -0700
     2.2 +++ b/agent/src/share/classes/sun/jvm/hotspot/code/DebugInfoReadStream.java	Mon Jul 27 17:23:52 2009 -0400
     2.3 @@ -24,23 +24,64 @@
     2.4  
     2.5  package sun.jvm.hotspot.code;
     2.6  
     2.7 +import java.util.*;
     2.8 +
     2.9  import sun.jvm.hotspot.debugger.*;
    2.10  import sun.jvm.hotspot.runtime.VM;
    2.11 +import sun.jvm.hotspot.utilities.*;
    2.12  
    2.13  public class DebugInfoReadStream extends CompressedReadStream {
    2.14    private NMethod code;
    2.15    private int InvocationEntryBCI;
    2.16 +  private List objectPool; // ArrayList<ObjectValue>
    2.17  
    2.18    public DebugInfoReadStream(NMethod code, int offset) {
    2.19      super(code.scopesDataBegin(), offset);
    2.20      InvocationEntryBCI = VM.getVM().getInvocationEntryBCI();
    2.21      this.code = code;
    2.22 +    this.objectPool = null;
    2.23 +  }
    2.24 +
    2.25 +  public DebugInfoReadStream(NMethod code, int offset, List objectPool) {
    2.26 +    super(code.scopesDataBegin(), offset);
    2.27 +    InvocationEntryBCI = VM.getVM().getInvocationEntryBCI();
    2.28 +    this.code = code;
    2.29 +    this.objectPool = objectPool;
    2.30    }
    2.31  
    2.32    public OopHandle readOopHandle() {
    2.33      return code.getOopAt(readInt());
    2.34    }
    2.35  
    2.36 +  ScopeValue readObjectValue() {
    2.37 +    int id = readInt();
    2.38 +    if (Assert.ASSERTS_ENABLED) {
    2.39 +      Assert.that(objectPool != null, "object pool does not exist");
    2.40 +      for (Iterator itr = objectPool.iterator(); itr.hasNext();) {
    2.41 +        ObjectValue ov = (ObjectValue) itr.next();
    2.42 +        Assert.that(ov.id() != id, "should not be read twice");
    2.43 +      }
    2.44 +    }
    2.45 +    ObjectValue result = new ObjectValue(id);
    2.46 +    // Cache the object since an object field could reference it.
    2.47 +    objectPool.add(result);
    2.48 +    result.readObject(this);
    2.49 +    return result;
    2.50 +  }
    2.51 +
    2.52 +  ScopeValue getCachedObject() {
    2.53 +    int id = readInt();
    2.54 +    Assert.that(objectPool != null, "object pool does not exist");
    2.55 +    for (Iterator itr = objectPool.iterator(); itr.hasNext();) {
    2.56 +      ObjectValue ov = (ObjectValue) itr.next();
    2.57 +      if (ov.id() == id) {
    2.58 +        return ov;
    2.59 +      }
    2.60 +    }
    2.61 +    Assert.that(false, "should not reach here");
    2.62 +    return null;
    2.63 +  }
    2.64 +
    2.65    public int readBCI() {
    2.66      return readInt() + InvocationEntryBCI;
    2.67    }

     3.1 --- a/agent/src/share/classes/sun/jvm/hotspot/code/MonitorValue.java	Mon Jul 27 09:06:22 2009 -0700
     3.2 +++ b/agent/src/share/classes/sun/jvm/hotspot/code/MonitorValue.java	Mon Jul 27 17:23:52 2009 -0400
     3.3 @@ -29,6 +29,7 @@
     3.4  public class MonitorValue {
     3.5    private ScopeValue owner;
     3.6    private Location   basicLock;
     3.7 +  private boolean    eliminated;
     3.8  
     3.9    // FIXME: not useful yet
    3.10    //  MonitorValue(ScopeValue* owner, Location basic_lock);
    3.11 @@ -36,10 +37,12 @@
    3.12    public MonitorValue(DebugInfoReadStream stream) {
    3.13      basicLock = new Location(stream);
    3.14      owner     = ScopeValue.readFrom(stream);
    3.15 +    eliminated= stream.readBoolean();
    3.16    }
    3.17  
    3.18    public ScopeValue owner()     { return owner; }
    3.19    public Location   basicLock() { return basicLock; }
    3.20 +  public boolean   eliminated() { return eliminated; }
    3.21  
    3.22    // FIXME: not yet implementable
    3.23    //  void write_on(DebugInfoWriteStream* stream);
    3.24 @@ -50,5 +53,8 @@
    3.25      tty.print(",");
    3.26      basicLock().printOn(tty);
    3.27      tty.print("}");
    3.28 +    if (eliminated) {
    3.29 +      tty.print(" (eliminated)");
    3.30 +    }
    3.31    }
    3.32  }

     4.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     4.2 +++ b/agent/src/share/classes/sun/jvm/hotspot/code/ObjectValue.java	Mon Jul 27 17:23:52 2009 -0400
     4.3 @@ -0,0 +1,93 @@
     4.4 +/*
     4.5 + * Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
     4.6 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
     4.7 + *
     4.8 + * This code is free software; you can redistribute it and/or modify it
     4.9 + * under the terms of the GNU General Public License version 2 only, as
    4.10 + * published by the Free Software Foundation.
    4.11 + *
    4.12 + * This code is distributed in the hope that it will be useful, but WITHOUT
    4.13 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    4.14 + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
    4.15 + * version 2 for more details (a copy is included in the LICENSE file that
    4.16 + * accompanied this code).
    4.17 + *
    4.18 + * You should have received a copy of the GNU General Public License version
    4.19 + * 2 along with this work; if not, write to the Free Software Foundation,
    4.20 + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
    4.21 + *
    4.22 + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
    4.23 + * CA 95054 USA or visit www.sun.com if you need additional information or
    4.24 + * have any questions.
    4.25 + *
    4.26 + */
    4.27 +
    4.28 +package sun.jvm.hotspot.code;
    4.29 +
    4.30 +import java.io.*;
    4.31 +import java.util.*;
    4.32 +
    4.33 +import sun.jvm.hotspot.debugger.*;
    4.34 +import sun.jvm.hotspot.utilities.*;
    4.35 +
    4.36 +/** An ObjectValue describes an object eliminated by escape analysis. */
    4.37 +
    4.38 +public class ObjectValue extends ScopeValue {
    4.39 +  private int        id;
    4.40 +  private ScopeValue klass;
    4.41 +  private List       fieldsValue; // ArrayList<ScopeValue>
    4.42 +
    4.43 +  // Field "boolean visited" is not implemented here since
    4.44 +  // it is used only a during debug info creation.
    4.45 +
    4.46 +  public ObjectValue(int id) {
    4.47 +    this.id = id;
    4.48 +    klass   = null;
    4.49 +    fieldsValue = new ArrayList();
    4.50 +  }
    4.51 +
    4.52 +  public boolean isObject() { return true; }
    4.53 +  public int id() { return id; }
    4.54 +  public ScopeValue getKlass() { return klass; }
    4.55 +  public List getFieldsValue() { return fieldsValue; }
    4.56 +  public ScopeValue getFieldAt(int i) { return (ScopeValue)fieldsValue.get(i); }
    4.57 +  public int fieldsSize() { return fieldsValue.size(); }
    4.58 +
    4.59 +  // Field "value" is always NULL here since it is used
    4.60 +  // only during deoptimization of a compiled frame
    4.61 +  // pointing to reallocated object.
    4.62 +  public OopHandle getValue() { return null; }
    4.63 +
    4.64 +  /** Serialization of debugging information */
    4.65 +
    4.66 +  void readObject(DebugInfoReadStream stream) {
    4.67 +    klass = readFrom(stream);
    4.68 +    Assert.that(klass.isConstantOop(), "should be constant klass oop");
    4.69 +    int length = stream.readInt();
    4.70 +    for (int i = 0; i < length; i++) {
    4.71 +      ScopeValue val = readFrom(stream);
    4.72 +      fieldsValue.add(val);
    4.73 +    }
    4.74 +  }
    4.75 +
    4.76 +  // Printing
    4.77 +
    4.78 +  public void print() {
    4.79 +    printOn(System.out);
    4.80 +  }
    4.81 +
    4.82 +  public void printOn(PrintStream tty) {
    4.83 +    tty.print("scalarObj[" + id + "]");
    4.84 +  }
    4.85 +
    4.86 +  void printFieldsOn(PrintStream tty) {
    4.87 +    if (fieldsValue.size() > 0) {
    4.88 +      ((ScopeValue)fieldsValue.get(0)).printOn(tty);
    4.89 +    }
    4.90 +    for (int i = 1; i < fieldsValue.size(); i++) {
    4.91 +      tty.print(", ");
    4.92 +      ((ScopeValue)fieldsValue.get(i)).printOn(tty);
    4.93 +    }
    4.94 +  }
    4.95 +
    4.96 +};

     5.1 --- a/agent/src/share/classes/sun/jvm/hotspot/code/ScopeDesc.java	Mon Jul 27 09:06:22 2009 -0700
     5.2 +++ b/agent/src/share/classes/sun/jvm/hotspot/code/ScopeDesc.java	Mon Jul 27 17:23:52 2009 -0400
     5.3 @@ -27,8 +27,10 @@
     5.4  import java.io.*;
     5.5  import java.util.*;
     5.6  
     5.7 +import sun.jvm.hotspot.debugger.*;
     5.8  import sun.jvm.hotspot.oops.*;
     5.9  import sun.jvm.hotspot.runtime.*;
    5.10 +import sun.jvm.hotspot.utilities.*;
    5.11  
    5.12  /** ScopeDescs contain the information that makes source-level
    5.13      debugging of nmethods possible; each scopeDesc describes a method
    5.14 @@ -45,10 +47,31 @@
    5.15    private int     localsDecodeOffset;
    5.16    private int     expressionsDecodeOffset;
    5.17    private int     monitorsDecodeOffset;
    5.18 +  /** Scalar replaced bjects pool */
    5.19 +  private List    objects; // ArrayList<ScopeValue>
    5.20 +
    5.21  
    5.22    public ScopeDesc(NMethod code, int decodeOffset) {
    5.23      this.code = code;
    5.24      this.decodeOffset = decodeOffset;
    5.25 +    this.objects      = decodeObjectValues(DebugInformationRecorder.SERIALIZED_NULL);
    5.26 +
    5.27 +    // Decode header
    5.28 +    DebugInfoReadStream stream  = streamAt(decodeOffset);
    5.29 +
    5.30 +    senderDecodeOffset = stream.readInt();
    5.31 +    method = (Method) VM.getVM().getObjectHeap().newOop(stream.readOopHandle());
    5.32 +    bci    = stream.readBCI();
    5.33 +    // Decode offsets for body and sender
    5.34 +    localsDecodeOffset      = stream.readInt();
    5.35 +    expressionsDecodeOffset = stream.readInt();
    5.36 +    monitorsDecodeOffset    = stream.readInt();
    5.37 +  }
    5.38 +
    5.39 +  public ScopeDesc(NMethod code, int decodeOffset, int objectDecodeOffset) {
    5.40 +    this.code = code;
    5.41 +    this.decodeOffset = decodeOffset;
    5.42 +    this.objects      = decodeObjectValues(objectDecodeOffset);
    5.43  
    5.44      // Decode header
    5.45      DebugInfoReadStream stream  = streamAt(decodeOffset);
    5.46 @@ -81,6 +104,11 @@
    5.47      return decodeMonitorValues(monitorsDecodeOffset);
    5.48    }
    5.49  
    5.50 +  /** Returns a List&lt;MonitorValue&gt; */
    5.51 +  public List getObjects() {
    5.52 +    return objects;
    5.53 +  }
    5.54 +
    5.55    /** Stack walking. Returns null if this is the outermost scope. */
    5.56    public ScopeDesc sender() {
    5.57      if (isTop()) {
    5.58 @@ -131,7 +159,7 @@
    5.59    //
    5.60  
    5.61    private DebugInfoReadStream streamAt(int decodeOffset) {
    5.62 -    return new DebugInfoReadStream(code, decodeOffset);
    5.63 +    return new DebugInfoReadStream(code, decodeOffset, objects);
    5.64    }
    5.65  
    5.66    /** Returns a List&lt;ScopeValue&gt; or null if no values were present */
    5.67 @@ -161,4 +189,22 @@
    5.68      }
    5.69      return res;
    5.70    }
    5.71 +
    5.72 +  /** Returns a List&lt;ObjectValue&gt; or null if no values were present */
    5.73 +  private List decodeObjectValues(int decodeOffset) {
    5.74 +    if (decodeOffset == DebugInformationRecorder.SERIALIZED_NULL) {
    5.75 +      return null;
    5.76 +    }
    5.77 +    List res = new ArrayList();
    5.78 +    DebugInfoReadStream stream = new DebugInfoReadStream(code, decodeOffset, res);
    5.79 +    int length = stream.readInt();
    5.80 +    for (int i = 0; i < length; i++) {
    5.81 +      // Objects values are pushed to 'res' array during read so that
    5.82 +      // object's fields could reference it (OBJECT_ID_CODE).
    5.83 +      ScopeValue.readFrom(stream);
    5.84 +      // res.add(ScopeValue.readFrom(stream));
    5.85 +    }
    5.86 +    Assert.that(res.size() == length, "inconsistent debug information");
    5.87 +    return res;
    5.88 +  }
    5.89  }

     6.1 --- a/agent/src/share/classes/sun/jvm/hotspot/code/ScopeValue.java	Mon Jul 27 09:06:22 2009 -0700
     6.2 +++ b/agent/src/share/classes/sun/jvm/hotspot/code/ScopeValue.java	Mon Jul 27 17:23:52 2009 -0400
     6.3 @@ -49,12 +49,15 @@
     6.4    static final int CONSTANT_OOP_CODE    = 2;
     6.5    static final int CONSTANT_LONG_CODE   = 3;
     6.6    static final int CONSTANT_DOUBLE_CODE = 4;
     6.7 +  static final int CONSTANT_OBJECT_CODE = 5;
     6.8 +  static final int CONSTANT_OBJECT_ID_CODE = 6;
     6.9  
    6.10    public boolean isLocation()       { return false; }
    6.11    public boolean isConstantInt()    { return false; }
    6.12    public boolean isConstantDouble() { return false; }
    6.13    public boolean isConstantLong()   { return false; }
    6.14    public boolean isConstantOop()    { return false; }
    6.15 +  public boolean isObject()         { return false; }
    6.16  
    6.17    public static ScopeValue readFrom(DebugInfoReadStream stream) {
    6.18      switch (stream.readInt()) {
    6.19 @@ -68,6 +71,10 @@
    6.20        return new ConstantLongValue(stream);
    6.21      case CONSTANT_DOUBLE_CODE:
    6.22        return new ConstantDoubleValue(stream);
    6.23 +    case CONSTANT_OBJECT_CODE:
    6.24 +      return stream.readObjectValue();
    6.25 +    case CONSTANT_OBJECT_ID_CODE:
    6.26 +      return stream.getCachedObject();
    6.27      default:
    6.28        Assert.that(false, "should not reach here");
    6.29        return null;

     7.1 --- a/agent/src/share/classes/sun/jvm/hotspot/jdi/ObjectReferenceImpl.java	Mon Jul 27 09:06:22 2009 -0700
     7.2 +++ b/agent/src/share/classes/sun/jvm/hotspot/jdi/ObjectReferenceImpl.java	Mon Jul 27 17:23:52 2009 -0400
     7.3 @@ -249,6 +249,7 @@
     7.4              OopHandle givenHandle = obj.getHandle();
     7.5              for (Iterator itr = monitors.iterator(); itr.hasNext();) {
     7.6                  MonitorInfo mi = (MonitorInfo) itr.next();
     7.7 +                if (mi.eliminated() && frame.isCompiledFrame()) continue; // skip eliminated monitor
     7.8                  if (givenHandle.equals(mi.owner())) {
     7.9                      res++;
    7.10                  }

     8.1 --- a/agent/src/share/classes/sun/jvm/hotspot/jdi/ThreadReferenceImpl.java	Mon Jul 27 09:06:22 2009 -0700
     8.2 +++ b/agent/src/share/classes/sun/jvm/hotspot/jdi/ThreadReferenceImpl.java	Mon Jul 27 17:23:52 2009 -0400
     8.3 @@ -301,6 +301,9 @@
     8.4              List frameMonitors = frame.getMonitors();  // List<MonitorInfo>
     8.5              for (Iterator miItr = frameMonitors.iterator(); miItr.hasNext(); ) {
     8.6                  sun.jvm.hotspot.runtime.MonitorInfo mi = (sun.jvm.hotspot.runtime.MonitorInfo) miItr.next();
     8.7 +                if (mi.eliminated() && frame.isCompiledFrame()) {
     8.8 +                  continue; // skip eliminated monitor
     8.9 +                }
    8.10                  OopHandle obj = mi.owner();
    8.11                  if (obj == null) {
    8.12                      // this monitor doesn't have an owning object so skip it

     9.1 --- a/agent/src/share/classes/sun/jvm/hotspot/runtime/CompiledVFrame.java	Mon Jul 27 09:06:22 2009 -0700
     9.2 +++ b/agent/src/share/classes/sun/jvm/hotspot/runtime/CompiledVFrame.java	Mon Jul 27 17:23:52 2009 -0400
     9.3 @@ -131,8 +131,18 @@
     9.4      List result = new ArrayList(monitors.size());
     9.5      for (int i = 0; i < monitors.size(); i++) {
     9.6        MonitorValue mv = (MonitorValue) monitors.get(i);
     9.7 -      StackValue ownerSV = createStackValue(mv.owner()); // it is an oop
     9.8 -      result.add(new MonitorInfo(ownerSV.getObject(), resolveMonitorLock(mv.basicLock())));
     9.9 +      ScopeValue ov = mv.owner();
    9.10 +      StackValue ownerSV = createStackValue(ov); // it is an oop
    9.11 +      if (ov.isObject()) { // The owner object was scalar replaced
    9.12 +        Assert.that(mv.eliminated() && ownerSV.objIsScalarReplaced(), "monitor should be eliminated for scalar replaced object");
    9.13 +        // Put klass for scalar replaced object.
    9.14 +        ScopeValue kv = ((ObjectValue)ov).getKlass();
    9.15 +        Assert.that(kv.isConstantOop(), "klass should be oop constant for scalar replaced object");
    9.16 +        OopHandle k = ((ConstantOopReadValue)kv).getValue();
    9.17 +        result.add(new MonitorInfo(k, resolveMonitorLock(mv.basicLock()), mv.eliminated(), true));
    9.18 +      } else {
    9.19 +        result.add(new MonitorInfo(ownerSV.getObject(), resolveMonitorLock(mv.basicLock()), mv.eliminated(), false));
    9.20 +      }
    9.21      }
    9.22      return result;
    9.23    }
    9.24 @@ -212,12 +222,12 @@
    9.25            // long or is unused.  He always saves a long.  Here we know
    9.26            // a long was saved, but we only want an narrow oop back.  Narrow the
    9.27            // saved long to the narrow oop that the JVM wants.
    9.28 -          return new StackValue(valueAddr.getCompOopHandleAt(VM.getVM().getIntSize()));
    9.29 +          return new StackValue(valueAddr.getCompOopHandleAt(VM.getVM().getIntSize()), 0);
    9.30          } else {
    9.31 -          return new StackValue(valueAddr.getCompOopHandleAt(0));
    9.32 +          return new StackValue(valueAddr.getCompOopHandleAt(0), 0);
    9.33          }
    9.34        } else if( loc.holdsOop() ) {  // Holds an oop?
    9.35 -        return new StackValue(valueAddr.getOopHandleAt(0));
    9.36 +        return new StackValue(valueAddr.getOopHandleAt(0), 0);
    9.37        } else if( loc.holdsDouble() ) {
    9.38          // Double value in a single stack slot
    9.39          return new StackValue(valueAddr.getJIntAt(0) & 0xFFFFFFFF);
    9.40 @@ -277,7 +287,7 @@
    9.41        return new StackValue(((ConstantIntValue) sv).getValue() & 0xFFFFFFFF);
    9.42      } else if (sv.isConstantOop()) {
    9.43        // constant oop
    9.44 -      return new StackValue(((ConstantOopReadValue) sv).getValue());
    9.45 +      return new StackValue(((ConstantOopReadValue) sv).getValue(), 0);
    9.46      } else if (sv.isConstantDouble()) {
    9.47        // Constant double in a single stack slot
    9.48        double d = ((ConstantDoubleValue) sv).getValue();
    9.49 @@ -285,6 +295,9 @@
    9.50      } else if (VM.getVM().isLP64() && sv.isConstantLong()) {
    9.51        // Constant long in a single stack slot
    9.52        return new StackValue(((ConstantLongValue) sv).getValue() & 0xFFFFFFFF);
    9.53 +    } else if (sv.isObject()) {
    9.54 +      // Scalar replaced object in compiled frame
    9.55 +      return new StackValue(((ObjectValue)sv).getValue(), 1);
    9.56      }
    9.57  
    9.58      // Unknown ScopeValue type

    10.1 --- a/agent/src/share/classes/sun/jvm/hotspot/runtime/InterpretedVFrame.java	Mon Jul 27 09:06:22 2009 -0700
    10.2 +++ b/agent/src/share/classes/sun/jvm/hotspot/runtime/InterpretedVFrame.java	Mon Jul 27 17:23:52 2009 -0400
    10.3 @@ -61,7 +61,7 @@
    10.4        StackValue sv;
    10.5        if (oopMask.isOop(i)) {
    10.6          // oop value
    10.7 -        sv = new StackValue(addr.getOopHandleAt(0));
    10.8 +        sv = new StackValue(addr.getOopHandleAt(0), 0);
    10.9        } else {
   10.10          // integer
   10.11          // Fetch a signed integer the size of a stack slot
   10.12 @@ -95,7 +95,7 @@
   10.13        StackValue sv;
   10.14        if (oopMask.isOop(i + nofLocals)) {
   10.15          // oop value
   10.16 -        sv = new StackValue(addr.getOopHandleAt(0));
   10.17 +        sv = new StackValue(addr.getOopHandleAt(0), 0);
   10.18        } else {
   10.19          // integer
   10.20          // Fetch a signed integer the size of a stack slot
   10.21 @@ -113,7 +113,7 @@
   10.22      for (BasicObjectLock current = getFrame().interpreterFrameMonitorEnd();
   10.23           current.address().lessThan(getFrame().interpreterFrameMonitorBegin().address());
   10.24           current = getFrame().nextMonitorInInterpreterFrame(current)) {
   10.25 -      result.add(new MonitorInfo(current.obj(), current.lock()));
   10.26 +      result.add(new MonitorInfo(current.obj(), current.lock(), false, false));
   10.27      }
   10.28      return result;
   10.29    }

    11.1 --- a/agent/src/share/classes/sun/jvm/hotspot/runtime/MonitorInfo.java	Mon Jul 27 09:06:22 2009 -0700
    11.2 +++ b/agent/src/share/classes/sun/jvm/hotspot/runtime/MonitorInfo.java	Mon Jul 27 17:23:52 2009 -0400
    11.3 @@ -25,16 +25,39 @@
    11.4  package sun.jvm.hotspot.runtime;
    11.5  
    11.6  import sun.jvm.hotspot.debugger.*;
    11.7 +import sun.jvm.hotspot.utilities.*;
    11.8  
    11.9  public class MonitorInfo {
   11.10    private OopHandle owner;
   11.11    private BasicLock lock;
   11.12 +  private OopHandle ownerKlass;
   11.13 +  private boolean eliminated;
   11.14 +  private boolean ownerIsScalarReplaced;
   11.15  
   11.16 -  public MonitorInfo(OopHandle owner, BasicLock lock) {
   11.17 -    this.owner = owner;
   11.18 -    this.lock  = lock;
   11.19 +  public MonitorInfo(OopHandle owner, BasicLock lock, boolean eliminated, boolean ownerIsScalarReplaced) {
   11.20 +    if (!ownerIsScalarReplaced) {
   11.21 +      this.owner = owner;
   11.22 +      this.ownerKlass = null;
   11.23 +    } else {
   11.24 +      Assert.that(eliminated, "monitor should be eliminated for scalar replaced object");
   11.25 +      this.owner = null;
   11.26 +      this.ownerKlass = owner;
   11.27 +    }
   11.28 +    this.eliminated = eliminated;
   11.29 +    this.ownerIsScalarReplaced = ownerIsScalarReplaced;
   11.30    }
   11.31  
   11.32 -  public OopHandle owner() { return owner; }
   11.33 +  public OopHandle owner() {
   11.34 +   Assert.that(!ownerIsScalarReplaced, "should not be called for scalar replaced object");
   11.35 +   return owner;
   11.36 +  }
   11.37 +
   11.38 +  public OopHandle ownerKlass() {
   11.39 +   Assert.that(ownerIsScalarReplaced, "should not be called for not scalar replaced object");
   11.40 +   return ownerKlass;
   11.41 +  }
   11.42 +
   11.43    public BasicLock lock()  { return lock; }
   11.44 +  public boolean eliminated() { return eliminated; }
   11.45 +  public boolean ownerIsScalarReplaced() { return ownerIsScalarReplaced; }
   11.46  }

    12.1 --- a/agent/src/share/classes/sun/jvm/hotspot/runtime/StackValue.java	Mon Jul 27 09:06:22 2009 -0700
    12.2 +++ b/agent/src/share/classes/sun/jvm/hotspot/runtime/StackValue.java	Mon Jul 27 17:23:52 2009 -0400
    12.3 @@ -37,9 +37,11 @@
    12.4      type = BasicType.getTConflict();
    12.5    }
    12.6  
    12.7 -  public StackValue(OopHandle h) {
    12.8 +  public StackValue(OopHandle h, long scalar_replaced) {
    12.9      handleValue = h;
   12.10      type = BasicType.getTObject();
   12.11 +    integerValue = scalar_replaced;
   12.12 +    Assert.that(integerValue == 0 || handleValue == null, "not null object should not be marked as scalar replaced");
   12.13    }
   12.14  
   12.15    public StackValue(long i) {
   12.16 @@ -59,6 +61,13 @@
   12.17      return handleValue;
   12.18    }
   12.19  
   12.20 +  boolean objIsScalarReplaced() {
   12.21 +    if (Assert.ASSERTS_ENABLED) {
   12.22 +      Assert.that(type == BasicType.getTObject(), "type check");
   12.23 +    }
   12.24 +    return integerValue != 0;
   12.25 +  }
   12.26 +
   12.27    public long getInteger() {
   12.28      if (Assert.ASSERTS_ENABLED) {
   12.29        Assert.that(type == BasicType.getTInt(), "type check");

    13.1 --- a/agent/src/share/classes/sun/jvm/hotspot/ui/tree/OopTreeNodeAdapter.java	Mon Jul 27 09:06:22 2009 -0700
    13.2 +++ b/agent/src/share/classes/sun/jvm/hotspot/ui/tree/OopTreeNodeAdapter.java	Mon Jul 27 17:23:52 2009 -0400
    13.3 @@ -161,6 +161,8 @@
    13.4            child = new OopTreeNodeAdapter(field.getValue(getObj()), field.getID(), getTreeTableMode());
    13.5          } catch (AddressException e) {
    13.6            child = new BadOopTreeNodeAdapter(field.getValueAsOopHandle(getObj()), field.getID(), getTreeTableMode());
    13.7 +        } catch (UnknownOopException e) {
    13.8 +          child = new BadOopTreeNodeAdapter(field.getValueAsOopHandle(getObj()), field.getID(), getTreeTableMode());
    13.9          }
   13.10        }
   13.11        ++curField;

    14.1 --- a/agent/src/share/classes/sun/jvm/hotspot/utilities/soql/JSJavaThread.java	Mon Jul 27 09:06:22 2009 -0700
    14.2 +++ b/agent/src/share/classes/sun/jvm/hotspot/utilities/soql/JSJavaThread.java	Mon Jul 27 17:23:52 2009 -0400
    14.3 @@ -135,6 +135,10 @@
    14.4                      List frameMonitors = frame.getMonitors();  // List<MonitorInfo>
    14.5                      for (Iterator miItr = frameMonitors.iterator(); miItr.hasNext(); ) {
    14.6                          MonitorInfo mi = (MonitorInfo) miItr.next();
    14.7 +
    14.8 +                        if (mi.eliminated() && frame.isCompiledFrame()) {
    14.9 +                          continue; // skip eliminated monitor
   14.10 +                        }
   14.11                          OopHandle obj = mi.owner();
   14.12                          if (obj == null) {
   14.13                              // this monitor doesn't have an owning object so skip it

    15.1 --- a/make/hotspot_version	Mon Jul 27 09:06:22 2009 -0700
    15.2 +++ b/make/hotspot_version	Mon Jul 27 17:23:52 2009 -0400
    15.3 @@ -35,7 +35,7 @@
    15.4  
    15.5  HS_MAJOR_VER=16
    15.6  HS_MINOR_VER=0
    15.7 -HS_BUILD_NUMBER=04
    15.8 +HS_BUILD_NUMBER=06
    15.9  
   15.10  JDK_MAJOR_VER=1
   15.11  JDK_MINOR_VER=7

    16.1 --- a/make/solaris/makefiles/adlc.make	Mon Jul 27 09:06:22 2009 -0700
    16.2 +++ b/make/solaris/makefiles/adlc.make	Mon Jul 27 17:23:52 2009 -0400
    16.3 @@ -68,7 +68,9 @@
    16.4  
    16.5  # CFLAGS_WARN holds compiler options to suppress/enable warnings.
    16.6  # Compiler warnings are treated as errors
    16.7 -CFLAGS_WARN = +w -errwarn
    16.8 +ifeq ($(shell expr $(COMPILER_REV_NUMERIC) \>= 509), 1)
    16.9 +  CFLAGS_WARN = +w -errwarn
   16.10 +endif
   16.11  CFLAGS += $(CFLAGS_WARN)
   16.12  
   16.13  ifeq ("${Platform_compiler}", "sparcWorks")

    17.1 --- a/make/solaris/makefiles/optimized.make	Mon Jul 27 09:06:22 2009 -0700
    17.2 +++ b/make/solaris/makefiles/optimized.make	Mon Jul 27 17:23:52 2009 -0400
    17.3 @@ -41,7 +41,7 @@
    17.4  endif
    17.5  
    17.6  # Workaround SS11 bug 6345274 (all platforms) (Fixed in SS11 patch and SS12)
    17.7 -ifeq ($(COMPILER_REV_NUMERIC),508))
    17.8 +ifeq ($(COMPILER_REV_NUMERIC),508)
    17.9  OPT_CFLAGS/ciTypeFlow.o = $(OPT_CFLAGS/O2)
   17.10  endif # COMPILER_REV_NUMERIC == 508
   17.11

    18.1 --- a/src/cpu/sparc/vm/assembler_sparc.cpp	Mon Jul 27 09:06:22 2009 -0700
    18.2 +++ b/src/cpu/sparc/vm/assembler_sparc.cpp	Mon Jul 27 17:23:52 2009 -0400
    18.3 @@ -4208,6 +4208,7 @@
    18.4                    PtrQueue::byte_offset_of_active()),
    18.5           tmp);
    18.6    }
    18.7 +
    18.8    // Check on whether to annul.
    18.9    br_on_reg_cond(rc_z, /*annul*/false, Assembler::pt, tmp, filtered);
   18.10    delayed() -> nop();
   18.11 @@ -4215,13 +4216,13 @@
   18.12    // satb_log_barrier_work1(tmp, offset);
   18.13    if (index == noreg) {
   18.14      if (Assembler::is_simm13(offset)) {
   18.15 -      ld_ptr(obj, offset, tmp);
   18.16 +      load_heap_oop(obj, offset, tmp);
   18.17      } else {
   18.18        set(offset, tmp);
   18.19 -      ld_ptr(obj, tmp, tmp);
   18.20 +      load_heap_oop(obj, tmp, tmp);
   18.21      }
   18.22    } else {
   18.23 -    ld_ptr(obj, index, tmp);
   18.24 +    load_heap_oop(obj, index, tmp);
   18.25    }
   18.26  
   18.27    // satb_log_barrier_work2(obj, tmp, offset);

    19.1 --- a/src/cpu/sparc/vm/c1_LIRGenerator_sparc.cpp	Mon Jul 27 09:06:22 2009 -0700
    19.2 +++ b/src/cpu/sparc/vm/c1_LIRGenerator_sparc.cpp	Mon Jul 27 17:23:52 2009 -0400
    19.3 @@ -371,7 +371,7 @@
    19.4    }
    19.5    __ move(value.result(), array_addr, null_check_info);
    19.6    if (obj_store) {
    19.7 -    // Is this precise?
    19.8 +    // Precise card mark
    19.9      post_barrier(LIR_OprFact::address(array_addr), value.result());
   19.10    }
   19.11  }
   19.12 @@ -685,11 +685,8 @@
   19.13    LIR_Opr result = rlock_result(x);
   19.14    __ cmove(lir_cond_equal, LIR_OprFact::intConst(1), LIR_OprFact::intConst(0), result);
   19.15    if (type == objectType) {  // Write-barrier needed for Object fields.
   19.16 -#ifdef PRECISE_CARDMARK
   19.17 +    // Precise card mark since could either be object or array
   19.18      post_barrier(addr, val.result());
   19.19 -#else
   19.20 -    post_barrier(obj.result(), val.result());
   19.21 -#endif // PRECISE_CARDMARK
   19.22    }
   19.23  }
   19.24

    20.1 --- a/src/cpu/sparc/vm/sparc.ad	Mon Jul 27 09:06:22 2009 -0700
    20.2 +++ b/src/cpu/sparc/vm/sparc.ad	Mon Jul 27 17:23:52 2009 -0400
    20.3 @@ -1891,17 +1891,17 @@
    20.4  // The intptr_t operand types, defined by textual substitution.
    20.5  // (Cf. opto/type.hpp.  This lets us avoid many, many other ifdefs.)
    20.6  #ifdef _LP64
    20.7 -#define immX     immL
    20.8 -#define immX13   immL13
    20.9 -#define immX13m7 immL13m7
   20.10 -#define iRegX    iRegL
   20.11 -#define g1RegX   g1RegL
   20.12 +#define immX      immL
   20.13 +#define immX13    immL13
   20.14 +#define immX13m7  immL13m7
   20.15 +#define iRegX     iRegL
   20.16 +#define g1RegX    g1RegL
   20.17  #else
   20.18 -#define immX     immI
   20.19 -#define immX13   immI13
   20.20 -#define immX13m7 immI13m7
   20.21 -#define iRegX    iRegI
   20.22 -#define g1RegX   g1RegI
   20.23 +#define immX      immI
   20.24 +#define immX13    immI13
   20.25 +#define immX13m7  immI13m7
   20.26 +#define iRegX     iRegI
   20.27 +#define g1RegX    g1RegI
   20.28  #endif
   20.29  
   20.30  //----------ENCODING BLOCK-----------------------------------------------------
   20.31 @@ -3446,6 +3446,15 @@
   20.32    interface(CONST_INTER);
   20.33  %}
   20.34  
   20.35 +// Integer Immediate: 8-bit
   20.36 +operand immI8() %{
   20.37 +  predicate(Assembler::is_simm(n->get_int(), 8));
   20.38 +  match(ConI);
   20.39 +  op_cost(0);
   20.40 +  format %{ %}
   20.41 +  interface(CONST_INTER);
   20.42 +%}
   20.43 +
   20.44  // Integer Immediate: 13-bit
   20.45  operand immI13() %{
   20.46    predicate(Assembler::is_simm13(n->get_int()));
   20.47 @@ -3466,6 +3475,15 @@
   20.48    interface(CONST_INTER);
   20.49  %}
   20.50  
   20.51 +// Integer Immediate: 16-bit
   20.52 +operand immI16() %{
   20.53 +  predicate(Assembler::is_simm(n->get_int(), 16));
   20.54 +  match(ConI);
   20.55 +  op_cost(0);
   20.56 +  format %{ %}
   20.57 +  interface(CONST_INTER);
   20.58 +%}
   20.59 +
   20.60  // Unsigned (positive) Integer Immediate: 13-bit
   20.61  operand immU13() %{
   20.62    predicate((0 <= n->get_int()) && Assembler::is_simm13(n->get_int()));
   20.63 @@ -5544,7 +5562,7 @@
   20.64    ins_encode %{
   20.65      __ ldub($mem$$Address, $dst$$Register);
   20.66    %}
   20.67 -  ins_pipe(iload_mask_mem);
   20.68 +  ins_pipe(iload_mem);
   20.69  %}
   20.70  
   20.71  // Load Unsigned Byte (8bit UNsigned) into a Long Register
   20.72 @@ -5557,7 +5575,22 @@
   20.73    ins_encode %{
   20.74      __ ldub($mem$$Address, $dst$$Register);
   20.75    %}
   20.76 -  ins_pipe(iload_mask_mem);
   20.77 +  ins_pipe(iload_mem);
   20.78 +%}
   20.79 +
   20.80 +// Load Unsigned Byte (8 bit UNsigned) with 8-bit mask into Long Register
   20.81 +instruct loadUB2L_immI8(iRegL dst, memory mem, immI8 mask) %{
   20.82 +  match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
   20.83 +  ins_cost(MEMORY_REF_COST + DEFAULT_COST);
   20.84 +
   20.85 +  size(2*4);
   20.86 +  format %{ "LDUB   $mem,$dst\t# ubyte & 8-bit mask -> long\n\t"
   20.87 +            "AND    $dst,$mask,$dst" %}
   20.88 +  ins_encode %{
   20.89 +    __ ldub($mem$$Address, $dst$$Register);
   20.90 +    __ and3($dst$$Register, $mask$$constant, $dst$$Register);
   20.91 +  %}
   20.92 +  ins_pipe(iload_mem);
   20.93  %}
   20.94  
   20.95  // Load Short (16bit signed)
   20.96 @@ -5610,7 +5643,7 @@
   20.97    ins_encode %{
   20.98      __ lduh($mem$$Address, $dst$$Register);
   20.99    %}
  20.100 -  ins_pipe(iload_mask_mem);
  20.101 +  ins_pipe(iload_mem);
  20.102  %}
  20.103  
  20.104  // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
  20.105 @@ -5636,7 +5669,56 @@
  20.106    ins_encode %{
  20.107      __ lduh($mem$$Address, $dst$$Register);
  20.108    %}
  20.109 -  ins_pipe(iload_mask_mem);
  20.110 +  ins_pipe(iload_mem);
  20.111 +%}
  20.112 +
  20.113 +// Load Unsigned Short/Char (16bit UNsigned) with mask 0xFF into a Long Register
  20.114 +instruct loadUS2L_immI_255(iRegL dst, indOffset13m7 mem, immI_255 mask) %{
  20.115 +  match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
  20.116 +  ins_cost(MEMORY_REF_COST);
  20.117 +
  20.118 +  size(4);
  20.119 +  format %{ "LDUB   $mem+1,$dst\t! ushort/char & 0xFF -> long" %}
  20.120 +  ins_encode %{
  20.121 +    __ ldub($mem$$Address, $dst$$Register, 1);  // LSB is index+1 on BE
  20.122 +  %}
  20.123 +  ins_pipe(iload_mem);
  20.124 +%}
  20.125 +
  20.126 +// Load Unsigned Short/Char (16bit UNsigned) with a 13-bit mask into a Long Register
  20.127 +instruct loadUS2L_immI13(iRegL dst, memory mem, immI13 mask) %{
  20.128 +  match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
  20.129 +  ins_cost(MEMORY_REF_COST + DEFAULT_COST);
  20.130 +
  20.131 +  size(2*4);
  20.132 +  format %{ "LDUH   $mem,$dst\t! ushort/char & 13-bit mask -> long\n\t"
  20.133 +            "AND    $dst,$mask,$dst" %}
  20.134 +  ins_encode %{
  20.135 +    Register Rdst = $dst$$Register;
  20.136 +    __ lduh($mem$$Address, Rdst);
  20.137 +    __ and3(Rdst, $mask$$constant, Rdst);
  20.138 +  %}
  20.139 +  ins_pipe(iload_mem);
  20.140 +%}
  20.141 +
  20.142 +// Load Unsigned Short/Char (16bit UNsigned) with a 16-bit mask into a Long Register
  20.143 +instruct loadUS2L_immI16(iRegL dst, memory mem, immI16 mask, iRegL tmp) %{
  20.144 +  match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
  20.145 +  effect(TEMP dst, TEMP tmp);
  20.146 +  ins_cost(MEMORY_REF_COST + 2*DEFAULT_COST);
  20.147 +
  20.148 +  size(3*4);
  20.149 +  format %{ "LDUH   $mem,$dst\t! ushort/char & 16-bit mask -> long\n\t"
  20.150 +            "SET    $mask,$tmp\n\t"
  20.151 +            "AND    $dst,$tmp,$dst" %}
  20.152 +  ins_encode %{
  20.153 +    Register Rdst = $dst$$Register;
  20.154 +    Register Rtmp = $tmp$$Register;
  20.155 +    __ lduh($mem$$Address, Rdst);
  20.156 +    __ set($mask$$constant, Rtmp);
  20.157 +    __ and3(Rdst, Rtmp, Rdst);
  20.158 +  %}
  20.159 +  ins_pipe(iload_mem);
  20.160  %}
  20.161  
  20.162  // Load Integer
  20.163 @@ -5718,6 +5800,68 @@
  20.164    ins_encode %{
  20.165      __ ldsw($mem$$Address, $dst$$Register);
  20.166    %}
  20.167 +  ins_pipe(iload_mask_mem);
  20.168 +%}
  20.169 +
  20.170 +// Load Integer with mask 0xFF into a Long Register
  20.171 +instruct loadI2L_immI_255(iRegL dst, indOffset13m7 mem, immI_255 mask) %{
  20.172 +  match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
  20.173 +  ins_cost(MEMORY_REF_COST);
  20.174 +
  20.175 +  size(4);
  20.176 +  format %{ "LDUB   $mem+3,$dst\t! int & 0xFF -> long" %}
  20.177 +  ins_encode %{
  20.178 +    __ ldub($mem$$Address, $dst$$Register, 3);  // LSB is index+3 on BE
  20.179 +  %}
  20.180 +  ins_pipe(iload_mem);
  20.181 +%}
  20.182 +
  20.183 +// Load Integer with mask 0xFFFF into a Long Register
  20.184 +instruct loadI2L_immI_65535(iRegL dst, indOffset13m7 mem, immI_65535 mask) %{
  20.185 +  match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
  20.186 +  ins_cost(MEMORY_REF_COST);
  20.187 +
  20.188 +  size(4);
  20.189 +  format %{ "LDUH   $mem+2,$dst\t! int & 0xFFFF -> long" %}
  20.190 +  ins_encode %{
  20.191 +    __ lduh($mem$$Address, $dst$$Register, 2);  // LSW is index+2 on BE
  20.192 +  %}
  20.193 +  ins_pipe(iload_mem);
  20.194 +%}
  20.195 +
  20.196 +// Load Integer with a 13-bit mask into a Long Register
  20.197 +instruct loadI2L_immI13(iRegL dst, memory mem, immI13 mask) %{
  20.198 +  match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
  20.199 +  ins_cost(MEMORY_REF_COST + DEFAULT_COST);
  20.200 +
  20.201 +  size(2*4);
  20.202 +  format %{ "LDUW   $mem,$dst\t! int & 13-bit mask -> long\n\t"
  20.203 +            "AND    $dst,$mask,$dst" %}
  20.204 +  ins_encode %{
  20.205 +    Register Rdst = $dst$$Register;
  20.206 +    __ lduw($mem$$Address, Rdst);
  20.207 +    __ and3(Rdst, $mask$$constant, Rdst);
  20.208 +  %}
  20.209 +  ins_pipe(iload_mem);
  20.210 +%}
  20.211 +
  20.212 +// Load Integer with a 32-bit mask into a Long Register
  20.213 +instruct loadI2L_immI(iRegL dst, memory mem, immI mask, iRegL tmp) %{
  20.214 +  match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
  20.215 +  effect(TEMP dst, TEMP tmp);
  20.216 +  ins_cost(MEMORY_REF_COST + 2*DEFAULT_COST);
  20.217 +
  20.218 +  size(3*4);
  20.219 +  format %{ "LDUW   $mem,$dst\t! int & 32-bit mask -> long\n\t"
  20.220 +            "SET    $mask,$tmp\n\t"
  20.221 +            "AND    $dst,$tmp,$dst" %}
  20.222 +  ins_encode %{
  20.223 +    Register Rdst = $dst$$Register;
  20.224 +    Register Rtmp = $tmp$$Register;
  20.225 +    __ lduw($mem$$Address, Rdst);
  20.226 +    __ set($mask$$constant, Rtmp);
  20.227 +    __ and3(Rdst, Rtmp, Rdst);
  20.228 +  %}
  20.229    ins_pipe(iload_mem);
  20.230  %}
  20.231

    21.1 --- a/src/cpu/x86/vm/assembler_x86.cpp	Mon Jul 27 09:06:22 2009 -0700
    21.2 +++ b/src/cpu/x86/vm/assembler_x86.cpp	Mon Jul 27 17:23:52 2009 -0400
    21.3 @@ -6805,14 +6805,18 @@
    21.4    jcc(Assembler::equal, done);
    21.5  
    21.6    // if (x.f == NULL) goto done;
    21.7 -  cmpptr(Address(obj, 0), NULL_WORD);
    21.8 +#ifdef _LP64
    21.9 +  load_heap_oop(tmp2, Address(obj, 0));
   21.10 +#else
   21.11 +  movptr(tmp2, Address(obj, 0));
   21.12 +#endif
   21.13 +  cmpptr(tmp2, (int32_t) NULL_WORD);
   21.14    jcc(Assembler::equal, done);
   21.15  
   21.16    // Can we store original value in the thread's buffer?
   21.17  
   21.18 -  LP64_ONLY(movslq(tmp, index);)
   21.19 -  movptr(tmp2, Address(obj, 0));
   21.20  #ifdef _LP64
   21.21 +  movslq(tmp, index);
   21.22    cmpq(tmp, 0);
   21.23  #else
   21.24    cmpl(index, 0);
   21.25 @@ -6834,8 +6838,7 @@
   21.26    if(tosca_live) push(rax);
   21.27    push(obj);
   21.28  #ifdef _LP64
   21.29 -  movq(c_rarg0, Address(obj, 0));
   21.30 -  call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), c_rarg0, r15_thread);
   21.31 +  call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), tmp2, r15_thread);
   21.32  #else
   21.33    push(thread);
   21.34    call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), tmp2, thread);

    22.1 --- a/src/cpu/x86/vm/interp_masm_x86_32.cpp	Mon Jul 27 09:06:22 2009 -0700
    22.2 +++ b/src/cpu/x86/vm/interp_masm_x86_32.cpp	Mon Jul 27 17:23:52 2009 -0400
    22.3 @@ -1372,6 +1372,8 @@
    22.4      // If no method data exists, go to profile_continue.
    22.5      test_method_data_pointer(mdp, profile_continue);
    22.6  
    22.7 +    set_mdp_flag_at(mdp, BitData::null_seen_byte_constant());
    22.8 +
    22.9      // The method data pointer needs to be updated.
   22.10      int mdp_delta = in_bytes(BitData::bit_data_size());
   22.11      if (TypeProfileCasts) {

    23.1 --- a/src/cpu/x86/vm/interp_masm_x86_64.cpp	Mon Jul 27 09:06:22 2009 -0700
    23.2 +++ b/src/cpu/x86/vm/interp_masm_x86_64.cpp	Mon Jul 27 17:23:52 2009 -0400
    23.3 @@ -1409,6 +1409,8 @@
    23.4      // If no method data exists, go to profile_continue.
    23.5      test_method_data_pointer(mdp, profile_continue);
    23.6  
    23.7 +    set_mdp_flag_at(mdp, BitData::null_seen_byte_constant());
    23.8 +
    23.9      // The method data pointer needs to be updated.
   23.10      int mdp_delta = in_bytes(BitData::bit_data_size());
   23.11      if (TypeProfileCasts) {

    24.1 --- a/src/cpu/x86/vm/methodHandles_x86.cpp	Mon Jul 27 09:06:22 2009 -0700
    24.2 +++ b/src/cpu/x86/vm/methodHandles_x86.cpp	Mon Jul 27 17:23:52 2009 -0400
    24.3 @@ -269,11 +269,11 @@
    24.4  
    24.5  #ifndef PRODUCT
    24.6  void trace_method_handle_stub(const char* adaptername,
    24.7 -                              oop mh,
    24.8 +                              oopDesc* mh,
    24.9                                intptr_t* entry_sp,
   24.10                                intptr_t* saved_sp) {
   24.11    // called as a leaf from native code: do not block the JVM!
   24.12 -  printf("MH %s "PTR_FORMAT" "PTR_FORMAT" "INTX_FORMAT"\n", adaptername, mh, entry_sp, entry_sp - saved_sp);
   24.13 +  printf("MH %s "PTR_FORMAT" "PTR_FORMAT" "INTX_FORMAT"\n", adaptername, (void*)mh, entry_sp, entry_sp - saved_sp);
   24.14  }
   24.15  #endif //PRODUCT
   24.16

    25.1 --- a/src/cpu/x86/vm/sharedRuntime_x86_64.cpp	Mon Jul 27 09:06:22 2009 -0700
    25.2 +++ b/src/cpu/x86/vm/sharedRuntime_x86_64.cpp	Mon Jul 27 17:23:52 2009 -0400
    25.3 @@ -1302,22 +1302,19 @@
    25.4  
    25.5    const Register ic_reg = rax;
    25.6    const Register receiver = j_rarg0;
    25.7 -  const Register tmp = rdx;
    25.8  
    25.9    Label ok;
   25.10    Label exception_pending;
   25.11  
   25.12 +  assert_different_registers(ic_reg, receiver, rscratch1);
   25.13    __ verify_oop(receiver);
   25.14 -  __ push(tmp); // spill (any other registers free here???)
   25.15 -  __ load_klass(tmp, receiver);
   25.16 -  __ cmpq(ic_reg, tmp);
   25.17 +  __ load_klass(rscratch1, receiver);
   25.18 +  __ cmpq(ic_reg, rscratch1);
   25.19    __ jcc(Assembler::equal, ok);
   25.20  
   25.21 -  __ pop(tmp);
   25.22    __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
   25.23  
   25.24    __ bind(ok);
   25.25 -  __ pop(tmp);
   25.26  
   25.27    // Verified entry point must be aligned
   25.28    __ align(8);

    26.1 --- a/src/cpu/x86/vm/stubGenerator_x86_32.cpp	Mon Jul 27 09:06:22 2009 -0700
    26.2 +++ b/src/cpu/x86/vm/stubGenerator_x86_32.cpp	Mon Jul 27 17:23:52 2009 -0400
    26.3 @@ -709,7 +709,7 @@
    26.4    //
    26.5    //  Input:
    26.6    //     start   -  starting address
    26.7 -  //     end     -  element count
    26.8 +  //     count   -  element count
    26.9    void  gen_write_ref_array_pre_barrier(Register start, Register count) {
   26.10      assert_different_registers(start, count);
   26.11      BarrierSet* bs = Universe::heap()->barrier_set();
   26.12 @@ -757,7 +757,6 @@
   26.13            __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post)));
   26.14            __ addptr(rsp, 2*wordSize);
   26.15            __ popa();
   26.16 -
   26.17          }
   26.18          break;
   26.19

    27.1 --- a/src/cpu/x86/vm/stubGenerator_x86_64.cpp	Mon Jul 27 09:06:22 2009 -0700
    27.2 +++ b/src/cpu/x86/vm/stubGenerator_x86_64.cpp	Mon Jul 27 17:23:52 2009 -0400
    27.3 @@ -1207,9 +1207,9 @@
    27.4            __ pusha();                      // push registers (overkill)
    27.5            // must compute element count unless barrier set interface is changed (other platforms supply count)
    27.6            assert_different_registers(start, end, scratch);
    27.7 -          __ lea(scratch, Address(end, wordSize));
    27.8 -          __ subptr(scratch, start);
    27.9 -          __ shrptr(scratch, LogBytesPerWord);
   27.10 +          __ lea(scratch, Address(end, BytesPerHeapOop));
   27.11 +          __ subptr(scratch, start);               // subtract start to get #bytes
   27.12 +          __ shrptr(scratch, LogBytesPerHeapOop);  // convert to element count
   27.13            __ mov(c_rarg0, start);
   27.14            __ mov(c_rarg1, scratch);
   27.15            __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post)));
   27.16 @@ -1225,6 +1225,7 @@
   27.17            Label L_loop;
   27.18  
   27.19             __ shrptr(start, CardTableModRefBS::card_shift);
   27.20 +           __ addptr(end, BytesPerHeapOop);
   27.21             __ shrptr(end, CardTableModRefBS::card_shift);
   27.22             __ subptr(end, start); // number of bytes to copy
   27.23  
   27.24 @@ -2251,6 +2252,7 @@
   27.25      // and report their number to the caller.
   27.26      assert_different_registers(rax, r14_length, count, to, end_to, rcx);
   27.27      __ lea(end_to, to_element_addr);
   27.28 +    __ addptr(end_to, -heapOopSize);      // make an inclusive end pointer
   27.29      gen_write_ref_array_post_barrier(to, end_to, rscratch1);
   27.30      __ movptr(rax, r14_length);           // original oops
   27.31      __ addptr(rax, count);                // K = (original - remaining) oops
   27.32 @@ -2259,7 +2261,7 @@
   27.33  
   27.34      // Come here on success only.
   27.35      __ BIND(L_do_card_marks);
   27.36 -    __ addptr(end_to, -wordSize);         // make an inclusive end pointer
   27.37 +    __ addptr(end_to, -heapOopSize);         // make an inclusive end pointer
   27.38      gen_write_ref_array_post_barrier(to, end_to, rscratch1);
   27.39      __ xorptr(rax, rax);                  // return 0 on success
   27.40

    28.1 --- a/src/cpu/x86/vm/x86_32.ad	Mon Jul 27 09:06:22 2009 -0700
    28.2 +++ b/src/cpu/x86/vm/x86_32.ad	Mon Jul 27 17:23:52 2009 -0400
    28.3 @@ -6885,8 +6885,9 @@
    28.4  %}
    28.5  
    28.6  // Load Byte (8bit signed) into Long Register
    28.7 -instruct loadB2L(eRegL dst, memory mem) %{
    28.8 +instruct loadB2L(eRegL dst, memory mem, eFlagsReg cr) %{
    28.9    match(Set dst (ConvI2L (LoadB mem)));
   28.10 +  effect(KILL cr);
   28.11  
   28.12    ins_cost(375);
   28.13    format %{ "MOVSX8 $dst.lo,$mem\t# byte -> long\n\t"
   28.14 @@ -6917,19 +6918,37 @@
   28.15  %}
   28.16  
   28.17  // Load Unsigned Byte (8 bit UNsigned) into Long Register
   28.18 -instruct loadUB2L(eRegL dst, memory mem)
   28.19 -%{
   28.20 +instruct loadUB2L(eRegL dst, memory mem, eFlagsReg cr) %{
   28.21    match(Set dst (ConvI2L (LoadUB mem)));
   28.22 +  effect(KILL cr);
   28.23  
   28.24    ins_cost(250);
   28.25    format %{ "MOVZX8 $dst.lo,$mem\t# ubyte -> long\n\t"
   28.26              "XOR    $dst.hi,$dst.hi" %}
   28.27  
   28.28    ins_encode %{
   28.29 -    __ movzbl($dst$$Register, $mem$$Address);
   28.30 -    __ xorl(HIGH_FROM_LOW($dst$$Register), HIGH_FROM_LOW($dst$$Register));
   28.31 -  %}
   28.32 -
   28.33 +    Register Rdst = $dst$$Register;
   28.34 +    __ movzbl(Rdst, $mem$$Address);
   28.35 +    __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
   28.36 +  %}
   28.37 +
   28.38 +  ins_pipe(ialu_reg_mem);
   28.39 +%}
   28.40 +
   28.41 +// Load Unsigned Byte (8 bit UNsigned) with mask into Long Register
   28.42 +instruct loadUB2L_immI8(eRegL dst, memory mem, immI8 mask, eFlagsReg cr) %{
   28.43 +  match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
   28.44 +  effect(KILL cr);
   28.45 +
   28.46 +  format %{ "MOVZX8 $dst.lo,$mem\t# ubyte & 8-bit mask -> long\n\t"
   28.47 +            "XOR    $dst.hi,$dst.hi\n\t"
   28.48 +            "AND    $dst.lo,$mask" %}
   28.49 +  ins_encode %{
   28.50 +    Register Rdst = $dst$$Register;
   28.51 +    __ movzbl(Rdst, $mem$$Address);
   28.52 +    __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
   28.53 +    __ andl(Rdst, $mask$$constant);
   28.54 +  %}
   28.55    ins_pipe(ialu_reg_mem);
   28.56  %}
   28.57  
   28.58 @@ -6960,8 +6979,9 @@
   28.59  %}
   28.60  
   28.61  // Load Short (16bit signed) into Long Register
   28.62 -instruct loadS2L(eRegL dst, memory mem) %{
   28.63 +instruct loadS2L(eRegL dst, memory mem, eFlagsReg cr) %{
   28.64    match(Set dst (ConvI2L (LoadS mem)));
   28.65 +  effect(KILL cr);
   28.66  
   28.67    ins_cost(375);
   28.68    format %{ "MOVSX  $dst.lo,$mem\t# short -> long\n\t"
   28.69 @@ -7004,8 +7024,9 @@
   28.70  %}
   28.71  
   28.72  // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
   28.73 -instruct loadUS2L(eRegL dst, memory mem) %{
   28.74 +instruct loadUS2L(eRegL dst, memory mem, eFlagsReg cr) %{
   28.75    match(Set dst (ConvI2L (LoadUS mem)));
   28.76 +  effect(KILL cr);
   28.77  
   28.78    ins_cost(250);
   28.79    format %{ "MOVZX  $dst.lo,$mem\t# ushort/char -> long\n\t"
   28.80 @@ -7019,6 +7040,38 @@
   28.81    ins_pipe(ialu_reg_mem);
   28.82  %}
   28.83  
   28.84 +// Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
   28.85 +instruct loadUS2L_immI_255(eRegL dst, memory mem, immI_255 mask, eFlagsReg cr) %{
   28.86 +  match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
   28.87 +  effect(KILL cr);
   28.88 +
   28.89 +  format %{ "MOVZX8 $dst.lo,$mem\t# ushort/char & 0xFF -> long\n\t"
   28.90 +            "XOR    $dst.hi,$dst.hi" %}
   28.91 +  ins_encode %{
   28.92 +    Register Rdst = $dst$$Register;
   28.93 +    __ movzbl(Rdst, $mem$$Address);
   28.94 +    __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
   28.95 +  %}
   28.96 +  ins_pipe(ialu_reg_mem);
   28.97 +%}
   28.98 +
   28.99 +// Load Unsigned Short/Char (16 bit UNsigned) with a 16-bit mask into Long Register
  28.100 +instruct loadUS2L_immI16(eRegL dst, memory mem, immI16 mask, eFlagsReg cr) %{
  28.101 +  match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
  28.102 +  effect(KILL cr);
  28.103 +
  28.104 +  format %{ "MOVZX  $dst.lo, $mem\t# ushort/char & 16-bit mask -> long\n\t"
  28.105 +            "XOR    $dst.hi,$dst.hi\n\t"
  28.106 +            "AND    $dst.lo,$mask" %}
  28.107 +  ins_encode %{
  28.108 +    Register Rdst = $dst$$Register;
  28.109 +    __ movzwl(Rdst, $mem$$Address);
  28.110 +    __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
  28.111 +    __ andl(Rdst, $mask$$constant);
  28.112 +  %}
  28.113 +  ins_pipe(ialu_reg_mem);
  28.114 +%}
  28.115 +
  28.116  // Load Integer
  28.117  instruct loadI(eRegI dst, memory mem) %{
  28.118    match(Set dst (LoadI mem));
  28.119 @@ -7082,8 +7135,9 @@
  28.120  %}
  28.121  
  28.122  // Load Integer into Long Register
  28.123 -instruct loadI2L(eRegL dst, memory mem) %{
  28.124 +instruct loadI2L(eRegL dst, memory mem, eFlagsReg cr) %{
  28.125    match(Set dst (ConvI2L (LoadI mem)));
  28.126 +  effect(KILL cr);
  28.127  
  28.128    ins_cost(375);
  28.129    format %{ "MOV    $dst.lo,$mem\t# int -> long\n\t"
  28.130 @@ -7099,9 +7153,57 @@
  28.131    ins_pipe(ialu_reg_mem);
  28.132  %}
  28.133  
  28.134 +// Load Integer with mask 0xFF into Long Register
  28.135 +instruct loadI2L_immI_255(eRegL dst, memory mem, immI_255 mask, eFlagsReg cr) %{
  28.136 +  match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
  28.137 +  effect(KILL cr);
  28.138 +
  28.139 +  format %{ "MOVZX8 $dst.lo,$mem\t# int & 0xFF -> long\n\t"
  28.140 +            "XOR    $dst.hi,$dst.hi" %}
  28.141 +  ins_encode %{
  28.142 +    Register Rdst = $dst$$Register;
  28.143 +    __ movzbl(Rdst, $mem$$Address);
  28.144 +    __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
  28.145 +  %}
  28.146 +  ins_pipe(ialu_reg_mem);
  28.147 +%}
  28.148 +
  28.149 +// Load Integer with mask 0xFFFF into Long Register
  28.150 +instruct loadI2L_immI_65535(eRegL dst, memory mem, immI_65535 mask, eFlagsReg cr) %{
  28.151 +  match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
  28.152 +  effect(KILL cr);
  28.153 +
  28.154 +  format %{ "MOVZX  $dst.lo,$mem\t# int & 0xFFFF -> long\n\t"
  28.155 +            "XOR    $dst.hi,$dst.hi" %}
  28.156 +  ins_encode %{
  28.157 +    Register Rdst = $dst$$Register;
  28.158 +    __ movzwl(Rdst, $mem$$Address);
  28.159 +    __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
  28.160 +  %}
  28.161 +  ins_pipe(ialu_reg_mem);
  28.162 +%}
  28.163 +
  28.164 +// Load Integer with 32-bit mask into Long Register
  28.165 +instruct loadI2L_immI(eRegL dst, memory mem, immI mask, eFlagsReg cr) %{
  28.166 +  match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
  28.167 +  effect(KILL cr);
  28.168 +
  28.169 +  format %{ "MOV    $dst.lo,$mem\t# int & 32-bit mask -> long\n\t"
  28.170 +            "XOR    $dst.hi,$dst.hi\n\t"
  28.171 +            "AND    $dst.lo,$mask" %}
  28.172 +  ins_encode %{
  28.173 +    Register Rdst = $dst$$Register;
  28.174 +    __ movl(Rdst, $mem$$Address);
  28.175 +    __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
  28.176 +    __ andl(Rdst, $mask$$constant);
  28.177 +  %}
  28.178 +  ins_pipe(ialu_reg_mem);
  28.179 +%}
  28.180 +
  28.181  // Load Unsigned Integer into Long Register
  28.182 -instruct loadUI2L(eRegL dst, memory mem) %{
  28.183 +instruct loadUI2L(eRegL dst, memory mem, eFlagsReg cr) %{
  28.184    match(Set dst (LoadUI2L mem));
  28.185 +  effect(KILL cr);
  28.186  
  28.187    ins_cost(250);
  28.188    format %{ "MOV    $dst.lo,$mem\t# uint -> long\n\t"
  28.189 @@ -7695,6 +7797,17 @@
  28.190    ins_pipe( ialu_mem_long_reg );
  28.191  %}
  28.192  
  28.193 +// Store Long to Integer
  28.194 +instruct storeL2I(memory mem, eRegL src) %{
  28.195 +  match(Set mem (StoreI mem (ConvL2I src)));
  28.196 +
  28.197 +  format %{ "MOV    $mem,$src.lo\t# long -> int" %}
  28.198 +  ins_encode %{
  28.199 +    __ movl($mem$$Address, $src$$Register);
  28.200 +  %}
  28.201 +  ins_pipe(ialu_mem_reg);
  28.202 +%}
  28.203 +
  28.204  // Volatile Store Long.  Must be atomic, so move it into
  28.205  // the FP TOS and then do a 64-bit FIST.  Has to probe the
  28.206  // target address before the store (for null-ptr checks)

    29.1 --- a/src/cpu/x86/vm/x86_64.ad	Mon Jul 27 09:06:22 2009 -0700
    29.2 +++ b/src/cpu/x86/vm/x86_64.ad	Mon Jul 27 17:23:52 2009 -0400
    29.3 @@ -6444,6 +6444,21 @@
    29.4    ins_pipe(ialu_reg_mem);
    29.5  %}
    29.6  
    29.7 +// Load Unsigned Byte (8 bit UNsigned) with a 8-bit mask into Long Register
    29.8 +instruct loadUB2L_immI8(rRegL dst, memory mem, immI8 mask, rFlagsReg cr) %{
    29.9 +  match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
   29.10 +  effect(KILL cr);
   29.11 +
   29.12 +  format %{ "movzbq  $dst, $mem\t# ubyte & 8-bit mask -> long\n\t"
   29.13 +            "andl    $dst, $mask" %}
   29.14 +  ins_encode %{
   29.15 +    Register Rdst = $dst$$Register;
   29.16 +    __ movzbq(Rdst, $mem$$Address);
   29.17 +    __ andl(Rdst, $mask$$constant);
   29.18 +  %}
   29.19 +  ins_pipe(ialu_reg_mem);
   29.20 +%}
   29.21 +
   29.22  // Load Short (16 bit signed)
   29.23  instruct loadS(rRegI dst, memory mem)
   29.24  %{
   29.25 @@ -6528,6 +6543,32 @@
   29.26    ins_pipe(ialu_reg_mem);
   29.27  %}
   29.28  
   29.29 +// Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
   29.30 +instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
   29.31 +  match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
   29.32 +
   29.33 +  format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
   29.34 +  ins_encode %{
   29.35 +    __ movzbq($dst$$Register, $mem$$Address);
   29.36 +  %}
   29.37 +  ins_pipe(ialu_reg_mem);
   29.38 +%}
   29.39 +
   29.40 +// Load Unsigned Short/Char (16 bit UNsigned) with mask into Long Register
   29.41 +instruct loadUS2L_immI16(rRegL dst, memory mem, immI16 mask, rFlagsReg cr) %{
   29.42 +  match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
   29.43 +  effect(KILL cr);
   29.44 +
   29.45 +  format %{ "movzwq  $dst, $mem\t# ushort/char & 16-bit mask -> long\n\t"
   29.46 +            "andl    $dst, $mask" %}
   29.47 +  ins_encode %{
   29.48 +    Register Rdst = $dst$$Register;
   29.49 +    __ movzwq(Rdst, $mem$$Address);
   29.50 +    __ andl(Rdst, $mask$$constant);
   29.51 +  %}
   29.52 +  ins_pipe(ialu_reg_mem);
   29.53 +%}
   29.54 +
   29.55  // Load Integer
   29.56  instruct loadI(rRegI dst, memory mem)
   29.57  %{
   29.58 @@ -6606,6 +6647,43 @@
   29.59    ins_pipe(ialu_reg_mem);
   29.60  %}
   29.61  
   29.62 +// Load Integer with mask 0xFF into Long Register
   29.63 +instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
   29.64 +  match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
   29.65 +
   29.66 +  format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
   29.67 +  ins_encode %{
   29.68 +    __ movzbq($dst$$Register, $mem$$Address);
   29.69 +  %}
   29.70 +  ins_pipe(ialu_reg_mem);
   29.71 +%}
   29.72 +
   29.73 +// Load Integer with mask 0xFFFF into Long Register
   29.74 +instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
   29.75 +  match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
   29.76 +
   29.77 +  format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
   29.78 +  ins_encode %{
   29.79 +    __ movzwq($dst$$Register, $mem$$Address);
   29.80 +  %}
   29.81 +  ins_pipe(ialu_reg_mem);
   29.82 +%}
   29.83 +
   29.84 +// Load Integer with a 32-bit mask into Long Register
   29.85 +instruct loadI2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
   29.86 +  match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
   29.87 +  effect(KILL cr);
   29.88 +
   29.89 +  format %{ "movl    $dst, $mem\t# int & 32-bit mask -> long\n\t"
   29.90 +            "andl    $dst, $mask" %}
   29.91 +  ins_encode %{
   29.92 +    Register Rdst = $dst$$Register;
   29.93 +    __ movl(Rdst, $mem$$Address);
   29.94 +    __ andl(Rdst, $mask$$constant);
   29.95 +  %}
   29.96 +  ins_pipe(ialu_reg_mem);
   29.97 +%}
   29.98 +
   29.99  // Load Unsigned Integer into Long Register
  29.100  instruct loadUI2L(rRegL dst, memory mem)
  29.101  %{
  29.102 @@ -11673,8 +11751,9 @@
  29.103  
  29.104    ins_cost(125);
  29.105    format %{ "movslq  $dst, $src\t# i2l" %}
  29.106 -  opcode(0x63); // needs REX.W
  29.107 -  ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
  29.108 +  ins_encode %{
  29.109 +    __ movslq($dst$$Register, $src$$Register);
  29.110 +  %}
  29.111    ins_pipe(ialu_reg_reg);
  29.112  %}
  29.113

    30.1 --- a/src/share/tools/MakeDeps/Database.java	Mon Jul 27 09:06:22 2009 -0700
    30.2 +++ b/src/share/tools/MakeDeps/Database.java	Mon Jul 27 17:23:52 2009 -0400
    30.3 @@ -1,5 +1,5 @@
    30.4  /*
    30.5 - * Copyright 1999-2005 Sun Microsystems, Inc.  All Rights Reserved.
    30.6 + * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
    30.7   * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    30.8   *
    30.9   * This code is free software; you can redistribute it and/or modify it
   30.10 @@ -39,7 +39,6 @@
   30.11    private HashMap<String,String> platformDepFiles;
   30.12    private long threshold;
   30.13    private int nOuterFiles;
   30.14 -  private int nPrecompiledFiles;
   30.15    private boolean missingOk;
   30.16    private Platform plat;
   30.17    /** These allow you to specify files not in the include database
   30.18 @@ -62,7 +61,6 @@
   30.19  
   30.20      threshold = t;
   30.21      nOuterFiles = 0;
   30.22 -    nPrecompiledFiles = 0;
   30.23      missingOk = false;
   30.24      firstFile = null;
   30.25      lastFile = null;
   30.26 @@ -343,7 +341,6 @@
   30.27              plat.getGIFileTemplate().getInvDir() +
   30.28              list.getName() +
   30.29              "\"");
   30.30 -        nPrecompiledFiles += 1;
   30.31        }
   30.32      }
   30.33      inclFile.println();
   30.34 @@ -408,22 +405,22 @@
   30.35        gd.println();
   30.36      }
   30.37  
   30.38 -    if (nPrecompiledFiles > 0) {
   30.39 -      // write Precompiled_Files = ...
   30.40 -      gd.println("Precompiled_Files = \\");
   30.41 -      for (Iterator iter = grandInclude.iterator(); iter.hasNext(); ) {
   30.42 -        FileList list = (FileList) iter.next();
   30.43 +    // write Precompiled_Files = ...
   30.44 +    gd.println("Precompiled_Files = \\");
   30.45 +    for (Iterator iter = grandInclude.iterator(); iter.hasNext(); ) {
   30.46 +      FileList list = (FileList) iter.next();
   30.47 +      if (list.getCount() >= threshold) {
   30.48          gd.println(list.getName() + " \\");
   30.49          String platformDep = platformDepFiles.get(list.getName());
   30.50          if (platformDep != null) {
   30.51 -            // make sure changes to the platform dependent file will
   30.52 -            // cause regeneration of the pch file.
   30.53 -            gd.println(platformDep + " \\");
   30.54 +          // make sure changes to the platform dependent file will
   30.55 +          // cause regeneration of the pch file.
   30.56 +          gd.println(platformDep + " \\");
   30.57          }
   30.58        }
   30.59 -      gd.println();
   30.60 -      gd.println();
   30.61      }
   30.62 +    gd.println();
   30.63 +    gd.println();
   30.64  
   30.65      gd.println("DTraced_Files = \\");
   30.66      for (Iterator iter = outerFiles.iterator(); iter.hasNext(); ) {
   30.67 @@ -483,7 +480,6 @@
   30.68          }
   30.69  
   30.70          if (plat.includeGIDependencies()
   30.71 -            && nPrecompiledFiles > 0
   30.72              && anII.getUseGrandInclude()) {
   30.73            gd.println("    $(Precompiled_Files) \\");
   30.74          }

    31.1 --- a/src/share/vm/adlc/formssel.cpp	Mon Jul 27 09:06:22 2009 -0700
    31.2 +++ b/src/share/vm/adlc/formssel.cpp	Mon Jul 27 17:23:52 2009 -0400
    31.3 @@ -420,6 +420,13 @@
    31.4    return  _matrule->is_ideal_load();
    31.5  }
    31.6  
    31.7 +// Return 'true' if this instruction matches an ideal 'LoadKlass' node
    31.8 +bool InstructForm::skip_antidep_check() const {
    31.9 +  if( _matrule == NULL ) return false;
   31.10 +
   31.11 +  return  _matrule->skip_antidep_check();
   31.12 +}
   31.13 +
   31.14  // Return 'true' if this instruction matches an ideal 'Load?' node
   31.15  Form::DataType InstructForm::is_ideal_store() const {
   31.16    if( _matrule == NULL ) return Form::none;
   31.17 @@ -567,6 +574,8 @@
   31.18  
   31.19  // loads from memory, so must check for anti-dependence
   31.20  bool InstructForm::needs_anti_dependence_check(FormDict &globals) const {
   31.21 +  if ( skip_antidep_check() ) return false;
   31.22 +
   31.23    // Machine independent loads must be checked for anti-dependences
   31.24    if( is_ideal_load() != Form::none )  return true;
   31.25  
   31.26 @@ -3957,6 +3966,28 @@
   31.27  }
   31.28  
   31.29  
   31.30 +bool MatchRule::skip_antidep_check() const {
   31.31 +  // Some loads operate on what is effectively immutable memory so we
   31.32 +  // should skip the anti dep computations.  For some of these nodes
   31.33 +  // the rewritable field keeps the anti dep logic from triggering but
   31.34 +  // for certain kinds of LoadKlass it does not since they are
   31.35 +  // actually reading memory which could be rewritten by the runtime,
   31.36 +  // though never by generated code.  This disables it uniformly for
   31.37 +  // the nodes that behave like this: LoadKlass, LoadNKlass and
   31.38 +  // LoadRange.
   31.39 +  if ( _opType && (strcmp(_opType,"Set") == 0) && _rChild ) {
   31.40 +    const char *opType = _rChild->_opType;
   31.41 +    if (strcmp("LoadKlass", opType) == 0 ||
   31.42 +        strcmp("LoadNKlass", opType) == 0 ||
   31.43 +        strcmp("LoadRange", opType) == 0) {
   31.44 +      return true;
   31.45 +    }
   31.46 +  }
   31.47 +
   31.48 +  return false;
   31.49 +}
   31.50 +
   31.51 +
   31.52  Form::DataType MatchRule::is_ideal_store() const {
   31.53    Form::DataType ideal_store = Form::none;
   31.54

    32.1 --- a/src/share/vm/adlc/formssel.hpp	Mon Jul 27 09:06:22 2009 -0700
    32.2 +++ b/src/share/vm/adlc/formssel.hpp	Mon Jul 27 17:23:52 2009 -0400
    32.3 @@ -158,6 +158,9 @@
    32.4  
    32.5    virtual Form::CallType is_ideal_call() const; // matches ideal 'Call'
    32.6    virtual Form::DataType is_ideal_load() const; // node matches ideal 'LoadXNode'
    32.7 +  // Should antidep checks be disabled for this Instruct
    32.8 +  // See definition of MatchRule::skip_antidep_check
    32.9 +  bool skip_antidep_check() const;
   32.10    virtual Form::DataType is_ideal_store() const;// node matches ideal 'StoreXNode'
   32.11            bool        is_ideal_mem() const { return is_ideal_load() != Form::none || is_ideal_store() != Form::none; }
   32.12    virtual uint        two_address(FormDict &globals); // output reg must match input reg
   32.13 @@ -1003,6 +1006,9 @@
   32.14    bool       is_ideal_loopEnd() const; // node matches ideal 'LoopEnd'
   32.15    bool       is_ideal_bool() const;    // node matches ideal 'Bool'
   32.16    Form::DataType is_ideal_load() const;// node matches ideal 'LoadXNode'
   32.17 +  // Should antidep checks be disabled for this rule
   32.18 +  // See definition of MatchRule::skip_antidep_check
   32.19 +  bool skip_antidep_check() const;
   32.20    Form::DataType is_ideal_store() const;// node matches ideal 'StoreXNode'
   32.21  
   32.22    // Check if 'mRule2' is a cisc-spill variant of this MatchRule

    33.1 --- a/src/share/vm/adlc/output_h.cpp	Mon Jul 27 09:06:22 2009 -0700
    33.2 +++ b/src/share/vm/adlc/output_h.cpp	Mon Jul 27 17:23:52 2009 -0400
    33.3 @@ -1367,11 +1367,11 @@
    33.4          else if (!strcmp(oper->ideal_type(_globalNames), "ConN")) {
    33.5            // Access the locally stored constant
    33.6            fprintf(fp,"  virtual intptr_t       constant() const {");
    33.7 -          fprintf(fp,   " return _c0->make_oopptr()->get_con();");
    33.8 +          fprintf(fp,   " return _c0->get_ptrtype()->get_con();");
    33.9            fprintf(fp, " }\n");
   33.10            // Generate query to determine if this pointer is an oop
   33.11            fprintf(fp,"  virtual bool           constant_is_oop() const {");
   33.12 -          fprintf(fp,   " return _c0->make_oopptr()->isa_oop_ptr();");
   33.13 +          fprintf(fp,   " return _c0->get_ptrtype()->isa_oop_ptr();");
   33.14            fprintf(fp, " }\n");
   33.15          }
   33.16          else if (!strcmp(oper->ideal_type(_globalNames), "ConL")) {

    34.1 --- a/src/share/vm/c1/c1_LIRGenerator.cpp	Mon Jul 27 09:06:22 2009 -0700
    34.2 +++ b/src/share/vm/c1/c1_LIRGenerator.cpp	Mon Jul 27 17:23:52 2009 -0400
    34.3 @@ -1534,12 +1534,8 @@
    34.4    }
    34.5  
    34.6    if (is_oop) {
    34.7 -#ifdef PRECISE_CARDMARK
    34.8 -    // Precise cardmarks don't work
    34.9 -    post_barrier(LIR_OprFact::address(address), value.result());
   34.10 -#else
   34.11 +    // Store to object so mark the card of the header
   34.12      post_barrier(object.result(), value.result());
   34.13 -#endif // PRECISE_CARDMARK
   34.14    }
   34.15  
   34.16    if (is_volatile && os::is_MP()) {

    35.1 --- a/src/share/vm/classfile/classFileParser.cpp	Mon Jul 27 09:06:22 2009 -0700
    35.2 +++ b/src/share/vm/classfile/classFileParser.cpp	Mon Jul 27 17:23:52 2009 -0400
    35.3 @@ -3237,6 +3237,16 @@
    35.4      this_klass->set_minor_version(minor_version);
    35.5      this_klass->set_major_version(major_version);
    35.6  
    35.7 +    // Set up methodOop::intrinsic_id as soon as we know the names of methods.
    35.8 +    // (We used to do this lazily, but now we query it in Rewriter,
    35.9 +    // which is eagerly done for every method, so we might as well do it now,
   35.10 +    // when everything is fresh in memory.)
   35.11 +    if (methodOopDesc::klass_id_for_intrinsics(this_klass->as_klassOop()) != vmSymbols::NO_SID) {
   35.12 +      for (int j = 0; j < methods->length(); j++) {
   35.13 +        ((methodOop)methods->obj_at(j))->init_intrinsic_id();
   35.14 +      }
   35.15 +    }
   35.16 +
   35.17      if (cached_class_file_bytes != NULL) {
   35.18        // JVMTI: we have an instanceKlass now, tell it about the cached bytes
   35.19        this_klass->set_cached_class_file(cached_class_file_bytes,

    36.1 --- a/src/share/vm/classfile/vmSymbols.hpp	Mon Jul 27 09:06:22 2009 -0700
    36.2 +++ b/src/share/vm/classfile/vmSymbols.hpp	Mon Jul 27 17:23:52 2009 -0400
    36.3 @@ -513,9 +513,6 @@
    36.4  //
    36.5  // for Emacs: (let ((c-backslash-column 120) (c-backslash-max-column 120)) (c-backslash-region (point) (point-max) nil t))
    36.6  #define VM_INTRINSICS_DO(do_intrinsic, do_class, do_name, do_signature, do_alias)                                       \
    36.7 -  do_intrinsic(_Object_init,              java_lang_Object, object_initializer_name, void_method_signature,      F_R)   \
    36.8 -  /*    (symbol object_initializer_name defined above) */                                                               \
    36.9 -                                                                                                                        \
   36.10    do_intrinsic(_hashCode,                 java_lang_Object,       hashCode_name, void_int_signature,             F_R)   \
   36.11     do_name(     hashCode_name,                                   "hashCode")                                            \
   36.12    do_intrinsic(_getClass,                 java_lang_Object,       getClass_name, void_class_signature,           F_R)   \
   36.13 @@ -635,9 +632,6 @@
   36.14    do_intrinsic(_equalsC,                  java_util_Arrays,       equals_name,    equalsC_signature,             F_S)   \
   36.15     do_signature(equalsC_signature,                               "([C[C)Z")                                             \
   36.16                                                                                                                          \
   36.17 -  do_intrinsic(_invoke,                   java_lang_reflect_Method, invoke_name, object_array_object_object_signature, F_R) \
   36.18 -  /*   (symbols invoke_name and invoke_signature defined above) */                                                      \
   36.19 -                                                                                                                        \
   36.20    do_intrinsic(_compareTo,                java_lang_String,       compareTo_name, string_int_signature,          F_R)   \
   36.21     do_name(     compareTo_name,                                  "compareTo")                                           \
   36.22    do_intrinsic(_indexOf,                  java_lang_String,       indexOf_name, string_int_signature,            F_R)   \
   36.23 @@ -656,8 +650,6 @@
   36.24     do_name(     attemptUpdate_name,                                 "attemptUpdate")                                    \
   36.25     do_signature(attemptUpdate_signature,                            "(JJ)Z")                                            \
   36.26                                                                                                                          \
   36.27 -  do_intrinsic(_fillInStackTrace,         java_lang_Throwable, fillInStackTrace_name, void_throwable_signature,  F_RNY) \
   36.28 -                                                                                                                        \
   36.29    /* support for sun.misc.Unsafe */                                                                                     \
   36.30    do_class(sun_misc_Unsafe,               "sun/misc/Unsafe")                                                            \
   36.31                                                                                                                          \
   36.32 @@ -819,10 +811,22 @@
   36.33     do_name(     prefetchReadStatic_name,                         "prefetchReadStatic")                                  \
   36.34    do_intrinsic(_prefetchWriteStatic,      sun_misc_Unsafe,        prefetchWriteStatic_name, prefetch_signature,  F_SN)  \
   36.35     do_name(     prefetchWriteStatic_name,                        "prefetchWriteStatic")                                 \
   36.36 +    /*== LAST_COMPILER_INLINE*/                                                                                         \
   36.37 +    /*the compiler does have special inlining code for these; bytecode inline is just fine */                           \
   36.38 +                                                                                                                        \
   36.39 +  do_intrinsic(_fillInStackTrace,         java_lang_Throwable, fillInStackTrace_name, void_throwable_signature,  F_RNY) \
   36.40 +                                                                                                                        \
   36.41 +  do_intrinsic(_Object_init,              java_lang_Object, object_initializer_name, void_method_signature,      F_R)   \
   36.42 +  /*    (symbol object_initializer_name defined above) */                                                               \
   36.43 +                                                                                                                        \
   36.44 +  do_intrinsic(_invoke,                   java_lang_reflect_Method, invoke_name, object_array_object_object_signature, F_R) \
   36.45 +  /*   (symbols invoke_name and invoke_signature defined above) */                                                      \
   36.46 +                                                                                                                        \
   36.47      /*end*/
   36.48  
   36.49  
   36.50  
   36.51 +
   36.52  // Class vmSymbols
   36.53  
   36.54  class vmSymbols: AllStatic {
   36.55 @@ -935,6 +939,7 @@
   36.56      #undef VM_INTRINSIC_ENUM
   36.57  
   36.58      ID_LIMIT,
   36.59 +    LAST_COMPILER_INLINE = _prefetchWriteStatic,
   36.60      FIRST_ID = _none + 1
   36.61    };
   36.62  
   36.63 @@ -972,4 +977,7 @@
   36.64    static Flags              flags_for(ID id);
   36.65  
   36.66    static const char* short_name_as_C_string(ID id, char* buf, int size);
   36.67 +
   36.68 +  // Access to intrinsic methods:
   36.69 +  static methodOop method_for(ID id);
   36.70  };

    37.1 --- a/src/share/vm/compiler/oopMap.cpp	Mon Jul 27 09:06:22 2009 -0700
    37.2 +++ b/src/share/vm/compiler/oopMap.cpp	Mon Jul 27 17:23:52 2009 -0400
    37.3 @@ -379,7 +379,15 @@
    37.4          if ( loc != NULL ) {
    37.5            oop *base_loc    = fr->oopmapreg_to_location(omv.content_reg(), reg_map);
    37.6            oop *derived_loc = loc;
    37.7 -          derived_oop_fn(base_loc, derived_loc);
    37.8 +          oop val = *base_loc;
    37.9 +          if (val == (oop)NULL || Universe::is_narrow_oop_base(val)) {
   37.10 +            // Ignore NULL oops and decoded NULL narrow oops which
   37.11 +            // equal to Universe::narrow_oop_base when a narrow oop
   37.12 +            // implicit null check is used in compiled code.
   37.13 +            // The narrow_oop_base could be NULL or be the address
   37.14 +            // of the page below heap depending on compressed oops mode.
   37.15 +          } else
   37.16 +            derived_oop_fn(base_loc, derived_loc);
   37.17          }
   37.18          oms.next();
   37.19        }  while (!oms.is_done());
   37.20 @@ -394,6 +402,15 @@
   37.21        oop* loc = fr->oopmapreg_to_location(omv.reg(),reg_map);
   37.22        if ( loc != NULL ) {
   37.23          if ( omv.type() == OopMapValue::oop_value ) {
   37.24 +          oop val = *loc;
   37.25 +          if (val == (oop)NULL || Universe::is_narrow_oop_base(val)) {
   37.26 +            // Ignore NULL oops and decoded NULL narrow oops which
   37.27 +            // equal to Universe::narrow_oop_base when a narrow oop
   37.28 +            // implicit null check is used in compiled code.
   37.29 +            // The narrow_oop_base could be NULL or be the address
   37.30 +            // of the page below heap depending on compressed oops mode.
   37.31 +            continue;
   37.32 +          }
   37.33  #ifdef ASSERT
   37.34            if ((((uintptr_t)loc & (sizeof(*loc)-1)) != 0) ||
   37.35               !Universe::heap()->is_in_or_null(*loc)) {
   37.36 @@ -410,6 +427,8 @@
   37.37  #endif // ASSERT
   37.38            oop_fn->do_oop(loc);
   37.39          } else if ( omv.type() == OopMapValue::value_value ) {
   37.40 +          assert((*loc) == (oop)NULL || !Universe::is_narrow_oop_base(*loc),
   37.41 +                 "found invalid value pointer");
   37.42            value_fn->do_oop(loc);
   37.43          } else if ( omv.type() == OopMapValue::narrowoop_value ) {
   37.44            narrowOop *nl = (narrowOop*)loc;

    38.1 --- a/src/share/vm/compiler/oopMap.hpp	Mon Jul 27 09:06:22 2009 -0700
    38.2 +++ b/src/share/vm/compiler/oopMap.hpp	Mon Jul 27 17:23:52 2009 -0400
    38.3 @@ -233,6 +233,10 @@
    38.4    int heap_size() const;
    38.5    void copy_to(address addr);
    38.6  
    38.7 +  // Methods oops_do() and all_do() filter out NULL oops and
    38.8 +  // oop == Universe::narrow_oop_base() before passing oops
    38.9 +  // to closures.
   38.10 +
   38.11    // Iterates through frame for a compiled method
   38.12    static void oops_do            (const frame* fr,
   38.13                                    const RegisterMap* reg_map, OopClosure* f);

    39.1 --- a/src/share/vm/gc_implementation/g1/bufferingOopClosure.hpp	Mon Jul 27 09:06:22 2009 -0700
    39.2 +++ b/src/share/vm/gc_implementation/g1/bufferingOopClosure.hpp	Mon Jul 27 17:23:52 2009 -0400
    39.3 @@ -42,35 +42,40 @@
    39.4      BufferLength = 1024
    39.5    };
    39.6  
    39.7 -  oop          *_buffer[BufferLength];
    39.8 -  oop         **_buffer_top;
    39.9 -  oop         **_buffer_curr;
   39.10 +  StarTask  _buffer[BufferLength];
   39.11 +  StarTask* _buffer_top;
   39.12 +  StarTask* _buffer_curr;
   39.13  
   39.14 -  OopClosure  *_oc;
   39.15 -  double       _closure_app_seconds;
   39.16 +  OopClosure* _oc;
   39.17 +  double      _closure_app_seconds;
   39.18  
   39.19    void process_buffer () {
   39.20 -
   39.21      double start = os::elapsedTime();
   39.22 -    for (oop **curr = _buffer; curr < _buffer_curr; ++curr) {
   39.23 -      _oc->do_oop(*curr);
   39.24 +    for (StarTask* curr = _buffer; curr < _buffer_curr; ++curr) {
   39.25 +      if (curr->is_narrow()) {
   39.26 +        assert(UseCompressedOops, "Error");
   39.27 +        _oc->do_oop((narrowOop*)(*curr));
   39.28 +      } else {
   39.29 +        _oc->do_oop((oop*)(*curr));
   39.30 +      }
   39.31      }
   39.32      _buffer_curr = _buffer;
   39.33      _closure_app_seconds += (os::elapsedTime() - start);
   39.34    }
   39.35  
   39.36 -public:
   39.37 -  virtual void do_oop(narrowOop* p) {
   39.38 -    guarantee(false, "NYI");
   39.39 -  }
   39.40 -  virtual void do_oop(oop *p) {
   39.41 +  template <class T> inline void do_oop_work(T* p) {
   39.42      if (_buffer_curr == _buffer_top) {
   39.43        process_buffer();
   39.44      }
   39.45 -
   39.46 -    *_buffer_curr = p;
   39.47 +    StarTask new_ref(p);
   39.48 +    *_buffer_curr = new_ref;
   39.49      ++_buffer_curr;
   39.50    }
   39.51 +
   39.52 +public:
   39.53 +  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
   39.54 +  virtual void do_oop(oop* p)       { do_oop_work(p); }
   39.55 +
   39.56    void done () {
   39.57      if (_buffer_curr > _buffer) {
   39.58        process_buffer();
   39.59 @@ -88,18 +93,17 @@
   39.60  class BufferingOopsInGenClosure: public OopsInGenClosure {
   39.61    BufferingOopClosure _boc;
   39.62    OopsInGenClosure* _oc;
   39.63 -public:
   39.64 + protected:
   39.65 +  template <class T> inline void do_oop_work(T* p) {
   39.66 +    assert(generation()->is_in_reserved((void*)p), "Must be in!");
   39.67 +    _boc.do_oop(p);
   39.68 +  }
   39.69 + public:
   39.70    BufferingOopsInGenClosure(OopsInGenClosure *oc) :
   39.71      _boc(oc), _oc(oc) {}
   39.72  
   39.73 -  virtual void do_oop(narrowOop* p) {
   39.74 -    guarantee(false, "NYI");
   39.75 -  }
   39.76 -
   39.77 -  virtual void do_oop(oop* p) {
   39.78 -    assert(generation()->is_in_reserved(p), "Must be in!");
   39.79 -    _boc.do_oop(p);
   39.80 -  }
   39.81 +  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
   39.82 +  virtual void do_oop(oop* p)       { do_oop_work(p); }
   39.83  
   39.84    void done() {
   39.85      _boc.done();
   39.86 @@ -130,14 +134,14 @@
   39.87      BufferLength = 1024
   39.88    };
   39.89  
   39.90 -  oop                      *_buffer[BufferLength];
   39.91 -  oop                     **_buffer_top;
   39.92 -  oop                     **_buffer_curr;
   39.93 +  StarTask     _buffer[BufferLength];
   39.94 +  StarTask*    _buffer_top;
   39.95 +  StarTask*    _buffer_curr;
   39.96  
   39.97 -  HeapRegion               *_hr_buffer[BufferLength];
   39.98 -  HeapRegion              **_hr_curr;
   39.99 +  HeapRegion*  _hr_buffer[BufferLength];
  39.100 +  HeapRegion** _hr_curr;
  39.101  
  39.102 -  OopsInHeapRegionClosure  *_oc;
  39.103 +  OopsInHeapRegionClosure*  _oc;
  39.104    double                    _closure_app_seconds;
  39.105  
  39.106    void process_buffer () {
  39.107 @@ -146,15 +150,20 @@
  39.108             "the two lengths should be the same");
  39.109  
  39.110      double start = os::elapsedTime();
  39.111 -    HeapRegion **hr_curr = _hr_buffer;
  39.112 -    HeapRegion *hr_prev = NULL;
  39.113 -    for (oop **curr = _buffer; curr < _buffer_curr; ++curr) {
  39.114 -      HeapRegion *region = *hr_curr;
  39.115 +    HeapRegion** hr_curr = _hr_buffer;
  39.116 +    HeapRegion*  hr_prev = NULL;
  39.117 +    for (StarTask* curr = _buffer; curr < _buffer_curr; ++curr) {
  39.118 +      HeapRegion* region = *hr_curr;
  39.119        if (region != hr_prev) {
  39.120          _oc->set_region(region);
  39.121          hr_prev = region;
  39.122        }
  39.123 -      _oc->do_oop(*curr);
  39.124 +      if (curr->is_narrow()) {
  39.125 +        assert(UseCompressedOops, "Error");
  39.126 +        _oc->do_oop((narrowOop*)(*curr));
  39.127 +      } else {
  39.128 +        _oc->do_oop((oop*)(*curr));
  39.129 +      }
  39.130        ++hr_curr;
  39.131      }
  39.132      _buffer_curr = _buffer;
  39.133 @@ -163,17 +172,16 @@
  39.134    }
  39.135  
  39.136  public:
  39.137 -  virtual void do_oop(narrowOop *p) {
  39.138 -    guarantee(false, "NYI");
  39.139 -  }
  39.140 +  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
  39.141 +  virtual void do_oop(      oop* p) { do_oop_work(p); }
  39.142  
  39.143 -  virtual void do_oop(oop *p) {
  39.144 +  template <class T> void do_oop_work(T* p) {
  39.145      if (_buffer_curr == _buffer_top) {
  39.146        assert(_hr_curr > _hr_buffer, "_hr_curr should be consistent with _buffer_curr");
  39.147        process_buffer();
  39.148      }
  39.149 -
  39.150 -    *_buffer_curr = p;
  39.151 +    StarTask new_ref(p);
  39.152 +    *_buffer_curr = new_ref;
  39.153      ++_buffer_curr;
  39.154      *_hr_curr = _from;
  39.155      ++_hr_curr;

    40.1 --- a/src/share/vm/gc_implementation/g1/concurrentMark.cpp	Mon Jul 27 09:06:22 2009 -0700
    40.2 +++ b/src/share/vm/gc_implementation/g1/concurrentMark.cpp	Mon Jul 27 17:23:52 2009 -0400
    40.3 @@ -452,13 +452,10 @@
    40.4    _regionStack.allocate(G1MarkRegionStackSize);
    40.5  
    40.6    // Create & start a ConcurrentMark thread.
    40.7 -  if (G1ConcMark) {
    40.8 -    _cmThread = new ConcurrentMarkThread(this);
    40.9 -    assert(cmThread() != NULL, "CM Thread should have been created");
   40.10 -    assert(cmThread()->cm() != NULL, "CM Thread should refer to this cm");
   40.11 -  } else {
   40.12 -    _cmThread = NULL;
   40.13 -  }
   40.14 +  _cmThread = new ConcurrentMarkThread(this);
   40.15 +  assert(cmThread() != NULL, "CM Thread should have been created");
   40.16 +  assert(cmThread()->cm() != NULL, "CM Thread should refer to this cm");
   40.17 +
   40.18    _g1h = G1CollectedHeap::heap();
   40.19    assert(CGC_lock != NULL, "Where's the CGC_lock?");
   40.20    assert(_markBitMap1.covers(rs), "_markBitMap1 inconsistency");
   40.21 @@ -783,18 +780,18 @@
   40.22                       bool do_barrier) : _cm(cm), _g1h(g1h),
   40.23                                          _do_barrier(do_barrier) { }
   40.24  
   40.25 -  virtual void do_oop(narrowOop* p) {
   40.26 -    guarantee(false, "NYI");
   40.27 -  }
   40.28 -
   40.29 -  virtual void do_oop(oop* p) {
   40.30 -    oop thisOop = *p;
   40.31 -    if (thisOop != NULL) {
   40.32 -      assert(thisOop->is_oop() || thisOop->mark() == NULL,
   40.33 +  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
   40.34 +  virtual void do_oop(      oop* p) { do_oop_work(p); }
   40.35 +
   40.36 +  template <class T> void do_oop_work(T* p) {
   40.37 +    T heap_oop = oopDesc::load_heap_oop(p);
   40.38 +    if (!oopDesc::is_null(heap_oop)) {
   40.39 +      oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
   40.40 +      assert(obj->is_oop() || obj->mark() == NULL,
   40.41               "expected an oop, possibly with mark word displaced");
   40.42 -      HeapWord* addr = (HeapWord*)thisOop;
   40.43 +      HeapWord* addr = (HeapWord*)obj;
   40.44        if (_g1h->is_in_g1_reserved(addr)) {
   40.45 -        _cm->grayRoot(thisOop);
   40.46 +        _cm->grayRoot(obj);
   40.47        }
   40.48      }
   40.49      if (_do_barrier) {
   40.50 @@ -850,16 +847,6 @@
   40.51    double start = os::elapsedTime();
   40.52    GCOverheadReporter::recordSTWStart(start);
   40.53  
   40.54 -  // If there has not been a GC[n-1] since last GC[n] cycle completed,
   40.55 -  // precede our marking with a collection of all
   40.56 -  // younger generations to keep floating garbage to a minimum.
   40.57 -  // YSR: we won't do this for now -- it's an optimization to be
   40.58 -  // done post-beta.
   40.59 -
   40.60 -  // YSR:    ignoring weak refs for now; will do at bug fixing stage
   40.61 -  // EVM:    assert(discoveredRefsAreClear());
   40.62 -
   40.63 -
   40.64    G1CollectorPolicy* g1p = G1CollectedHeap::heap()->g1_policy();
   40.65    g1p->record_concurrent_mark_init_start();
   40.66    checkpointRootsInitialPre();
   40.67 @@ -1135,6 +1122,13 @@
   40.68      return;
   40.69    }
   40.70  
   40.71 +  if (VerifyDuringGC) {
   40.72 +    HandleMark hm;  // handle scope
   40.73 +    gclog_or_tty->print(" VerifyDuringGC:(before)");
   40.74 +    Universe::heap()->prepare_for_verify();
   40.75 +    Universe::verify(true, false, true);
   40.76 +  }
   40.77 +
   40.78    G1CollectorPolicy* g1p = g1h->g1_policy();
   40.79    g1p->record_concurrent_mark_remark_start();
   40.80  
   40.81 @@ -1159,10 +1153,12 @@
   40.82      JavaThread::satb_mark_queue_set().set_active_all_threads(false);
   40.83  
   40.84      if (VerifyDuringGC) {
   40.85 -      g1h->prepare_for_verify();
   40.86 -      g1h->verify(/* allow_dirty */      true,
   40.87 -                  /* silent */           false,
   40.88 -                  /* use_prev_marking */ false);
   40.89 +      HandleMark hm;  // handle scope
   40.90 +      gclog_or_tty->print(" VerifyDuringGC:(after)");
   40.91 +      Universe::heap()->prepare_for_verify();
   40.92 +      Universe::heap()->verify(/* allow_dirty */      true,
   40.93 +                               /* silent */           false,
   40.94 +                               /* use_prev_marking */ false);
   40.95      }
   40.96    }
   40.97  
   40.98 @@ -1233,6 +1229,41 @@
   40.99                 CardTableModRefBS::card_shift);
  40.100    }
  40.101  
  40.102 +  // It takes a region that's not empty (i.e., it has at least one
  40.103 +  // live object in it and sets its corresponding bit on the region
  40.104 +  // bitmap to 1. If the region is "starts humongous" it will also set
  40.105 +  // to 1 the bits on the region bitmap that correspond to its
  40.106 +  // associated "continues humongous" regions.
  40.107 +  void set_bit_for_region(HeapRegion* hr) {
  40.108 +    assert(!hr->continuesHumongous(), "should have filtered those out");
  40.109 +
  40.110 +    size_t index = hr->hrs_index();
  40.111 +    if (!hr->startsHumongous()) {
  40.112 +      // Normal (non-humongous) case: just set the bit.
  40.113 +      _region_bm->par_at_put((BitMap::idx_t) index, true);
  40.114 +    } else {
  40.115 +      // Starts humongous case: calculate how many regions are part of
  40.116 +      // this humongous region and then set the bit range. It might
  40.117 +      // have been a bit more efficient to look at the object that
  40.118 +      // spans these humongous regions to calculate their number from
  40.119 +      // the object's size. However, it's a good idea to calculate
  40.120 +      // this based on the metadata itself, and not the region
  40.121 +      // contents, so that this code is not aware of what goes into
  40.122 +      // the humongous regions (in case this changes in the future).
  40.123 +      G1CollectedHeap* g1h = G1CollectedHeap::heap();
  40.124 +      size_t end_index = index + 1;
  40.125 +      while (end_index < g1h->n_regions()) {
  40.126 +        HeapRegion* chr = g1h->region_at(end_index);
  40.127 +        if (!chr->continuesHumongous()) {
  40.128 +          break;
  40.129 +        }
  40.130 +        end_index += 1;
  40.131 +      }
  40.132 +      _region_bm->par_at_put_range((BitMap::idx_t) index,
  40.133 +                                   (BitMap::idx_t) end_index, true);
  40.134 +    }
  40.135 +  }
  40.136 +
  40.137    bool doHeapRegion(HeapRegion* hr) {
  40.138      if (_co_tracker != NULL)
  40.139        _co_tracker->update();
  40.140 @@ -1241,13 +1272,13 @@
  40.141        _start_vtime_sec = os::elapsedVTime();
  40.142  
  40.143      if (hr->continuesHumongous()) {
  40.144 -      HeapRegion* hum_start = hr->humongous_start_region();
  40.145 -      // If the head region of the humongous region has been determined
  40.146 -      // to be alive, then all the tail regions should be marked
  40.147 -      // such as well.
  40.148 -      if (_region_bm->at(hum_start->hrs_index())) {
  40.149 -        _region_bm->par_at_put(hr->hrs_index(), 1);
  40.150 -      }
  40.151 +      // We will ignore these here and process them when their
  40.152 +      // associated "starts humongous" region is processed (see
  40.153 +      // set_bit_for_heap_region()). Note that we cannot rely on their
  40.154 +      // associated "starts humongous" region to have their bit set to
  40.155 +      // 1 since, due to the region chunking in the parallel region
  40.156 +      // iteration, a "continues humongous" region might be visited
  40.157 +      // before its associated "starts humongous".
  40.158        return false;
  40.159      }
  40.160  
  40.161 @@ -1343,14 +1374,14 @@
  40.162            intptr_t(uintptr_t(tp) >> CardTableModRefBS::card_shift);
  40.163          mark_card_num_range(start_card_num, last_card_num);
  40.164          // This definitely means the region has live objects.
  40.165 -        _region_bm->par_at_put(hr->hrs_index(), 1);
  40.166 +        set_bit_for_region(hr);
  40.167        }
  40.168      }
  40.169  
  40.170      hr->add_to_marked_bytes(marked_bytes);
  40.171      // Update the live region bitmap.
  40.172      if (marked_bytes > 0) {
  40.173 -      _region_bm->par_at_put(hr->hrs_index(), 1);
  40.174 +      set_bit_for_region(hr);
  40.175      }
  40.176      hr->set_top_at_conc_mark_count(nextTop);
  40.177      _tot_live += hr->next_live_bytes();
  40.178 @@ -1623,6 +1654,15 @@
  40.179      return;
  40.180    }
  40.181  
  40.182 +  if (VerifyDuringGC) {
  40.183 +    HandleMark hm;  // handle scope
  40.184 +    gclog_or_tty->print(" VerifyDuringGC:(before)");
  40.185 +    Universe::heap()->prepare_for_verify();
  40.186 +    Universe::verify(/* allow dirty  */ true,
  40.187 +                     /* silent       */ false,
  40.188 +                     /* prev marking */ true);
  40.189 +  }
  40.190 +
  40.191    _cleanup_co_tracker.disable();
  40.192  
  40.193    G1CollectorPolicy* g1p = G1CollectedHeap::heap()->g1_policy();
  40.194 @@ -1755,10 +1795,12 @@
  40.195    g1h->increment_total_collections();
  40.196  
  40.197    if (VerifyDuringGC) {
  40.198 -    g1h->prepare_for_verify();
  40.199 -    g1h->verify(/* allow_dirty */      true,
  40.200 -                /* silent */           false,
  40.201 -                /* use_prev_marking */ true);
  40.202 +    HandleMark hm;  // handle scope
  40.203 +    gclog_or_tty->print(" VerifyDuringGC:(after)");
  40.204 +    Universe::heap()->prepare_for_verify();
  40.205 +    Universe::verify(/* allow dirty  */ true,
  40.206 +                     /* silent       */ false,
  40.207 +                     /* prev marking */ true);
  40.208    }
  40.209  }
  40.210  
  40.211 @@ -1817,12 +1859,11 @@
  40.212      _g1(g1), _cm(cm),
  40.213      _bitMap(bitMap) {}
  40.214  
  40.215 -  void do_oop(narrowOop* p) {
  40.216 -    guarantee(false, "NYI");
  40.217 -  }
  40.218 -
  40.219 -  void do_oop(oop* p) {
  40.220 -    oop thisOop = *p;
  40.221 +  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
  40.222 +  virtual void do_oop(      oop* p) { do_oop_work(p); }
  40.223 +
  40.224 +  template <class T> void do_oop_work(T* p) {
  40.225 +    oop thisOop = oopDesc::load_decode_heap_oop(p);
  40.226      HeapWord* addr = (HeapWord*)thisOop;
  40.227      if (_g1->is_in_g1_reserved(addr) && _g1->is_obj_ill(thisOop)) {
  40.228        _bitMap->mark(addr);
  40.229 @@ -1981,12 +2022,11 @@
  40.230    ReachablePrinterOopClosure(CMBitMapRO* bitmap, outputStream* out) :
  40.231      _bitmap(bitmap), _g1h(G1CollectedHeap::heap()), _out(out) { }
  40.232  
  40.233 -  void do_oop(narrowOop* p) {
  40.234 -    guarantee(false, "NYI");
  40.235 -  }
  40.236 -
  40.237 -  void do_oop(oop* p) {
  40.238 -    oop         obj = *p;
  40.239 +  void do_oop(narrowOop* p) { do_oop_work(p); }
  40.240 +  void do_oop(      oop* p) { do_oop_work(p); }
  40.241 +
  40.242 +  template <class T> void do_oop_work(T* p) {
  40.243 +    oop         obj = oopDesc::load_decode_heap_oop(p);
  40.244      const char* str = NULL;
  40.245      const char* str2 = "";
  40.246  
  40.247 @@ -2128,6 +2168,7 @@
  40.248  
  40.249  
  40.250    HeapWord* objAddr = (HeapWord*) obj;
  40.251 +  assert(obj->is_oop_or_null(true /* ignore mark word */), "Error");
  40.252    if (_g1h->is_in_g1_reserved(objAddr)) {
  40.253      tmp_guarantee_CM( obj != NULL, "is_in_g1_reserved should ensure this" );
  40.254      HeapRegion* hr = _g1h->heap_region_containing(obj);
  40.255 @@ -2345,7 +2386,7 @@
  40.256      }
  40.257    }
  40.258  
  40.259 -  bool drain() {
  40.260 +  template <class T> bool drain() {
  40.261      while (_ms_ind > 0) {
  40.262        oop obj = pop();
  40.263        assert(obj != NULL, "Since index was non-zero.");
  40.264 @@ -2359,9 +2400,8 @@
  40.265          }
  40.266          // Now process this portion of this one.
  40.267          int lim = MIN2(next_arr_ind, len);
  40.268 -        assert(!UseCompressedOops, "This needs to be fixed");
  40.269          for (int j = arr_ind; j < lim; j++) {
  40.270 -          do_oop(aobj->obj_at_addr<oop>(j));
  40.271 +          do_oop(aobj->obj_at_addr<T>(j));
  40.272          }
  40.273  
  40.274        } else {
  40.275 @@ -2388,13 +2428,13 @@
  40.276      FREE_C_HEAP_ARRAY(jint, _array_ind_stack);
  40.277    }
  40.278  
  40.279 -  void do_oop(narrowOop* p) {
  40.280 -    guarantee(false, "NYI");
  40.281 -  }
  40.282 -
  40.283 -  void do_oop(oop* p) {
  40.284 -    oop obj = *p;
  40.285 -    if (obj == NULL) return;
  40.286 +  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
  40.287 +  virtual void do_oop(      oop* p) { do_oop_work(p); }
  40.288 +
  40.289 +  template <class T> void do_oop_work(T* p) {
  40.290 +    T heap_oop = oopDesc::load_heap_oop(p);
  40.291 +    if (oopDesc::is_null(heap_oop)) return;
  40.292 +    oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
  40.293      if (obj->is_forwarded()) {
  40.294        // If the object has already been forwarded, we have to make sure
  40.295        // that it's marked.  So follow the forwarding pointer.  Note that
  40.296 @@ -2443,7 +2483,11 @@
  40.297      oop obj = oop(addr);
  40.298      if (!obj->is_forwarded()) {
  40.299        if (!_oop_cl.push(obj)) return false;
  40.300 -      if (!_oop_cl.drain()) return false;
  40.301 +      if (UseCompressedOops) {
  40.302 +        if (!_oop_cl.drain<narrowOop>()) return false;
  40.303 +      } else {
  40.304 +        if (!_oop_cl.drain<oop>()) return false;
  40.305 +      }
  40.306      }
  40.307      // Otherwise...
  40.308      return true;
  40.309 @@ -2601,9 +2645,6 @@
  40.310  
  40.311  // abandon current marking iteration due to a Full GC
  40.312  void ConcurrentMark::abort() {
  40.313 -  // If we're not marking, nothing to do.
  40.314 -  if (!G1ConcMark) return;
  40.315 -
  40.316    // Clear all marks to force marking thread to do nothing
  40.317    _nextMarkBitMap->clearAll();
  40.318    // Empty mark stack
  40.319 @@ -2779,14 +2820,14 @@
  40.320    CMTask*            _task;
  40.321  
  40.322  public:
  40.323 -  void do_oop(narrowOop* p) {
  40.324 -    guarantee(false, "NYI");
  40.325 -  }
  40.326 -
  40.327 -  void do_oop(oop* p) {
  40.328 +  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
  40.329 +  virtual void do_oop(      oop* p) { do_oop_work(p); }
  40.330 +
  40.331 +  template <class T> void do_oop_work(T* p) {
  40.332      tmp_guarantee_CM( _g1h->is_in_g1_reserved((HeapWord*) p), "invariant" );
  40.333 -
  40.334 -    oop obj = *p;
  40.335 +    tmp_guarantee_CM( !_g1h->heap_region_containing((HeapWord*) p)->is_on_free_list(), "invariant" );
  40.336 +
  40.337 +    oop obj = oopDesc::load_decode_heap_oop(p);
  40.338      if (_cm->verbose_high())
  40.339        gclog_or_tty->print_cr("[%d] we're looking at location "
  40.340                               "*"PTR_FORMAT" = "PTR_FORMAT,
  40.341 @@ -2932,6 +2973,7 @@
  40.342    ++_refs_reached;
  40.343  
  40.344    HeapWord* objAddr = (HeapWord*) obj;
  40.345 +  assert(obj->is_oop_or_null(true /* ignore mark word */), "Error");
  40.346    if (_g1h->is_in_g1_reserved(objAddr)) {
  40.347      tmp_guarantee_CM( obj != NULL, "is_in_g1_reserved should ensure this" );
  40.348      HeapRegion* hr =  _g1h->heap_region_containing(obj);
  40.349 @@ -2995,6 +3037,7 @@
  40.350  void CMTask::push(oop obj) {
  40.351    HeapWord* objAddr = (HeapWord*) obj;
  40.352    tmp_guarantee_CM( _g1h->is_in_g1_reserved(objAddr), "invariant" );
  40.353 +  tmp_guarantee_CM( !_g1h->heap_region_containing(objAddr)->is_on_free_list(), "invariant" );
  40.354    tmp_guarantee_CM( !_g1h->is_obj_ill(obj), "invariant" );
  40.355    tmp_guarantee_CM( _nextMarkBitMap->isMarked(objAddr), "invariant" );
  40.356  
  40.357 @@ -3240,6 +3283,8 @@
  40.358  
  40.359        tmp_guarantee_CM( _g1h->is_in_g1_reserved((HeapWord*) obj),
  40.360                          "invariant" );
  40.361 +      tmp_guarantee_CM( !_g1h->heap_region_containing(obj)->is_on_free_list(),
  40.362 +                        "invariant" );
  40.363  
  40.364        scan_object(obj);
  40.365

    41.1 --- a/src/share/vm/gc_implementation/g1/concurrentMark.hpp	Mon Jul 27 09:06:22 2009 -0700
    41.2 +++ b/src/share/vm/gc_implementation/g1/concurrentMark.hpp	Mon Jul 27 17:23:52 2009 -0400
    41.3 @@ -763,6 +763,7 @@
    41.4    CMBitMap*                   _nextMarkBitMap;
    41.5    // the task queue of this task
    41.6    CMTaskQueue*                _task_queue;
    41.7 +private:
    41.8    // the task queue set---needed for stealing
    41.9    CMTaskQueueSet*             _task_queues;
   41.10    // indicates whether the task has been claimed---this is only  for

    42.1 --- a/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp	Mon Jul 27 09:06:22 2009 -0700
    42.2 +++ b/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp	Mon Jul 27 17:23:52 2009 -0400
    42.3 @@ -424,7 +424,7 @@
    42.4        while (n <= next_boundary) {
    42.5          q = n;
    42.6          oop obj = oop(q);
    42.7 -        if (obj->klass() == NULL) return q;
    42.8 +        if (obj->klass_or_null() == NULL) return q;
    42.9          n += obj->size();
   42.10        }
   42.11        assert(q <= next_boundary && n > next_boundary, "Consequence of loop");
   42.12 @@ -436,7 +436,7 @@
   42.13        while (n <= next_boundary) {
   42.14          q = n;
   42.15          oop obj = oop(q);
   42.16 -        if (obj->klass() == NULL) return q;
   42.17 +        if (obj->klass_or_null() == NULL) return q;
   42.18          n += _sp->block_size(q);
   42.19        }
   42.20        assert(q <= next_boundary && n > next_boundary, "Consequence of loop");

    43.1 --- a/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.inline.hpp	Mon Jul 27 09:06:22 2009 -0700
    43.2 +++ b/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.inline.hpp	Mon Jul 27 17:23:52 2009 -0400
    43.3 @@ -96,14 +96,14 @@
    43.4      while (n <= addr) {
    43.5        q = n;
    43.6        oop obj = oop(q);
    43.7 -      if (obj->klass() == NULL) return q;
    43.8 +      if (obj->klass_or_null() == NULL) return q;
    43.9        n += obj->size();
   43.10      }
   43.11    } else {
   43.12      while (n <= addr) {
   43.13        q = n;
   43.14        oop obj = oop(q);
   43.15 -      if (obj->klass() == NULL) return q;
   43.16 +      if (obj->klass_or_null() == NULL) return q;
   43.17        n += _sp->block_size(q);
   43.18      }
   43.19    }
   43.20 @@ -115,7 +115,7 @@
   43.21  inline HeapWord*
   43.22  G1BlockOffsetArray::forward_to_block_containing_addr(HeapWord* q,
   43.23                                                       const void* addr) {
   43.24 -  if (oop(q)->klass() == NULL) return q;
   43.25 +  if (oop(q)->klass_or_null() == NULL) return q;
   43.26    HeapWord* n = q + _sp->block_size(q);
   43.27    // In the normal case, where the query "addr" is a card boundary, and the
   43.28    // offset table chunks are the same size as cards, the block starting at

    44.1 --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp	Mon Jul 27 09:06:22 2009 -0700
    44.2 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp	Mon Jul 27 17:23:52 2009 -0400
    44.3 @@ -902,6 +902,10 @@
    44.4                                      size_t word_size) {
    44.5    ResourceMark rm;
    44.6  
    44.7 +  if (PrintHeapAtGC) {
    44.8 +    Universe::print_heap_before_gc();
    44.9 +  }
   44.10 +
   44.11    if (full && DisableExplicitGC) {
   44.12      gclog_or_tty->print("\n\n\nDisabling Explicit GC\n\n\n");
   44.13      return;
   44.14 @@ -927,7 +931,7 @@
   44.15      g1_policy()->record_full_collection_start();
   44.16  
   44.17      gc_prologue(true);
   44.18 -    increment_total_collections();
   44.19 +    increment_total_collections(true /* full gc */);
   44.20  
   44.21      size_t g1h_prev_used = used();
   44.22      assert(used() == recalculate_used(), "Should be equal");
   44.23 @@ -1066,6 +1070,10 @@
   44.24      assert( check_young_list_empty(false, false),
   44.25              "young list should be empty at this point");
   44.26    }
   44.27 +
   44.28 +  if (PrintHeapAtGC) {
   44.29 +    Universe::print_heap_after_gc();
   44.30 +  }
   44.31  }
   44.32  
   44.33  void G1CollectedHeap::do_full_collection(bool clear_all_soft_refs) {
   44.34 @@ -1650,8 +1658,15 @@
   44.35    assert(Heap_lock->owner() != NULL,
   44.36           "Should be owned on this thread's behalf.");
   44.37    size_t result = _summary_bytes_used;
   44.38 -  if (_cur_alloc_region != NULL)
   44.39 -    result += _cur_alloc_region->used();
   44.40 +  // Read only once in case it is set to NULL concurrently
   44.41 +  HeapRegion* hr = _cur_alloc_region;
   44.42 +  if (hr != NULL)
   44.43 +    result += hr->used();
   44.44 +  return result;
   44.45 +}
   44.46 +
   44.47 +size_t G1CollectedHeap::used_unlocked() const {
   44.48 +  size_t result = _summary_bytes_used;
   44.49    return result;
   44.50  }
   44.51  
   44.52 @@ -2125,13 +2140,13 @@
   44.53    VerifyLivenessOopClosure(G1CollectedHeap* _g1h) {
   44.54      g1h = _g1h;
   44.55    }
   44.56 -  void do_oop(narrowOop *p) {
   44.57 -    guarantee(false, "NYI");
   44.58 -  }
   44.59 -  void do_oop(oop *p) {
   44.60 -    oop obj = *p;
   44.61 -    assert(obj == NULL || !g1h->is_obj_dead(obj),
   44.62 -           "Dead object referenced by a not dead object");
   44.63 +  void do_oop(narrowOop *p) { do_oop_work(p); }
   44.64 +  void do_oop(      oop *p) { do_oop_work(p); }
   44.65 +
   44.66 +  template <class T> void do_oop_work(T *p) {
   44.67 +    oop obj = oopDesc::load_decode_heap_oop(p);
   44.68 +    guarantee(obj == NULL || !g1h->is_obj_dead(obj),
   44.69 +              "Dead object referenced by a not dead object");
   44.70    }
   44.71  };
   44.72  
   44.73 @@ -2198,8 +2213,10 @@
   44.74    // use_prev_marking == true  -> use "prev" marking information,
   44.75    // use_prev_marking == false -> use "next" marking information
   44.76    VerifyRegionClosure(bool allow_dirty, bool par, bool use_prev_marking)
   44.77 -    : _allow_dirty(allow_dirty), _par(par),
   44.78 +    : _allow_dirty(allow_dirty),
   44.79 +      _par(par),
   44.80        _use_prev_marking(use_prev_marking) {}
   44.81 +
   44.82    bool doHeapRegion(HeapRegion* r) {
   44.83      guarantee(_par || r->claim_value() == HeapRegion::InitialClaimValue,
   44.84                "Should be unclaimed at verify points.");
   44.85 @@ -2223,18 +2240,16 @@
   44.86    // use_prev_marking == true  -> use "prev" marking information,
   44.87    // use_prev_marking == false -> use "next" marking information
   44.88    VerifyRootsClosure(bool use_prev_marking) :
   44.89 -    _g1h(G1CollectedHeap::heap()), _failures(false),
   44.90 +    _g1h(G1CollectedHeap::heap()),
   44.91 +    _failures(false),
   44.92      _use_prev_marking(use_prev_marking) { }
   44.93  
   44.94    bool failures() { return _failures; }
   44.95  
   44.96 -  void do_oop(narrowOop* p) {
   44.97 -    guarantee(false, "NYI");
   44.98 -  }
   44.99 -
  44.100 -  void do_oop(oop* p) {
  44.101 -    oop obj = *p;
  44.102 -    if (obj != NULL) {
  44.103 +  template <class T> void do_oop_nv(T* p) {
  44.104 +    T heap_oop = oopDesc::load_heap_oop(p);
  44.105 +    if (!oopDesc::is_null(heap_oop)) {
  44.106 +      oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
  44.107        if (_g1h->is_obj_dead_cond(obj, _use_prev_marking)) {
  44.108          gclog_or_tty->print_cr("Root location "PTR_FORMAT" "
  44.109                                 "points to dead obj "PTR_FORMAT, p, (void*) obj);
  44.110 @@ -2243,6 +2258,9 @@
  44.111        }
  44.112      }
  44.113    }
  44.114 +
  44.115 +  void do_oop(oop* p)       { do_oop_nv(p); }
  44.116 +  void do_oop(narrowOop* p) { do_oop_nv(p); }
  44.117  };
  44.118  
  44.119  // This is the task used for parallel heap verification.
  44.120 @@ -2259,7 +2277,8 @@
  44.121    G1ParVerifyTask(G1CollectedHeap* g1h, bool allow_dirty,
  44.122                    bool use_prev_marking) :
  44.123      AbstractGangTask("Parallel verify task"),
  44.124 -    _g1h(g1h), _allow_dirty(allow_dirty),
  44.125 +    _g1h(g1h),
  44.126 +    _allow_dirty(allow_dirty),
  44.127      _use_prev_marking(use_prev_marking) { }
  44.128  
  44.129    void work(int worker_i) {
  44.130 @@ -2325,9 +2344,37 @@
  44.131    }
  44.132  };
  44.133  
  44.134 -void G1CollectedHeap::print() const { print_on(gclog_or_tty); }
  44.135 +void G1CollectedHeap::print() const { print_on(tty); }
  44.136  
  44.137  void G1CollectedHeap::print_on(outputStream* st) const {
  44.138 +  print_on(st, PrintHeapAtGCExtended);
  44.139 +}
  44.140 +
  44.141 +void G1CollectedHeap::print_on(outputStream* st, bool extended) const {
  44.142 +  st->print(" %-20s", "garbage-first heap");
  44.143 +  st->print(" total " SIZE_FORMAT "K, used " SIZE_FORMAT "K",
  44.144 +            capacity()/K, used_unlocked()/K);
  44.145 +  st->print(" [" INTPTR_FORMAT ", " INTPTR_FORMAT ", " INTPTR_FORMAT ")",
  44.146 +            _g1_storage.low_boundary(),
  44.147 +            _g1_storage.high(),
  44.148 +            _g1_storage.high_boundary());
  44.149 +  st->cr();
  44.150 +  st->print("  region size " SIZE_FORMAT "K, ",
  44.151 +            HeapRegion::GrainBytes/K);
  44.152 +  size_t young_regions = _young_list->length();
  44.153 +  st->print(SIZE_FORMAT " young (" SIZE_FORMAT "K), ",
  44.154 +            young_regions, young_regions * HeapRegion::GrainBytes / K);
  44.155 +  size_t survivor_regions = g1_policy()->recorded_survivor_regions();
  44.156 +  st->print(SIZE_FORMAT " survivors (" SIZE_FORMAT "K)",
  44.157 +            survivor_regions, survivor_regions * HeapRegion::GrainBytes / K);
  44.158 +  st->cr();
  44.159 +  perm()->as_gen()->print_on(st);
  44.160 +  if (extended) {
  44.161 +    print_on_extended(st);
  44.162 +  }
  44.163 +}
  44.164 +
  44.165 +void G1CollectedHeap::print_on_extended(outputStream* st) const {
  44.166    PrintRegionClosure blk(st);
  44.167    _hrs->iterate(&blk);
  44.168  }
  44.169 @@ -2408,10 +2455,6 @@
  44.170  }
  44.171  
  44.172  void G1CollectedHeap::gc_prologue(bool full /* Ignored */) {
  44.173 -  if (PrintHeapAtGC){
  44.174 -    gclog_or_tty->print_cr(" {Heap before GC collections=%d:", total_collections());
  44.175 -    Universe::print();
  44.176 -  }
  44.177    assert(InlineCacheBuffer::is_empty(), "should have cleaned up ICBuffer");
  44.178    // Call allocation profiler
  44.179    AllocationProfiler::iterate_since_last_gc();
  44.180 @@ -2425,12 +2468,6 @@
  44.181    // is set.
  44.182    COMPILER2_PRESENT(assert(DerivedPointerTable::is_empty(),
  44.183                          "derived pointer present"));
  44.184 -
  44.185 -  if (PrintHeapAtGC){
  44.186 -    gclog_or_tty->print_cr(" Heap after GC collections=%d:", total_collections());
  44.187 -    Universe::print();
  44.188 -    gclog_or_tty->print("} ");
  44.189 -  }
  44.190  }
  44.191  
  44.192  void G1CollectedHeap::do_collection_pause() {
  44.193 @@ -2453,12 +2490,10 @@
  44.194  
  44.195  void
  44.196  G1CollectedHeap::doConcurrentMark() {
  44.197 -  if (G1ConcMark) {
  44.198 -    MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
  44.199 -    if (!_cmThread->in_progress()) {
  44.200 -      _cmThread->set_started();
  44.201 -      CGC_lock->notify();
  44.202 -    }
  44.203 +  MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
  44.204 +  if (!_cmThread->in_progress()) {
  44.205 +    _cmThread->set_started();
  44.206 +    CGC_lock->notify();
  44.207    }
  44.208  }
  44.209  
  44.210 @@ -2535,9 +2570,11 @@
  44.211                            "Not enough space for young surv words summary.");
  44.212    }
  44.213    memset(_surviving_young_words, 0, array_length * sizeof(size_t));
  44.214 +#ifdef ASSERT
  44.215    for (size_t i = 0;  i < array_length; ++i) {
  44.216 -    guarantee( _surviving_young_words[i] == 0, "invariant" );
  44.217 -  }
  44.218 +    assert( _surviving_young_words[i] == 0, "memset above" );
  44.219 +  }
  44.220 +#endif // !ASSERT
  44.221  }
  44.222  
  44.223  void
  44.224 @@ -2559,241 +2596,254 @@
  44.225  
  44.226  void
  44.227  G1CollectedHeap::do_collection_pause_at_safepoint() {
  44.228 -  char verbose_str[128];
  44.229 -  sprintf(verbose_str, "GC pause ");
  44.230 -  if (g1_policy()->in_young_gc_mode()) {
  44.231 -    if (g1_policy()->full_young_gcs())
  44.232 -      strcat(verbose_str, "(young)");
  44.233 -    else
  44.234 -      strcat(verbose_str, "(partial)");
  44.235 -  }
  44.236 -  if (g1_policy()->should_initiate_conc_mark())
  44.237 -    strcat(verbose_str, " (initial-mark)");
  44.238 -
  44.239 -  GCCauseSetter x(this, GCCause::_g1_inc_collection_pause);
  44.240 -
  44.241 -  // if PrintGCDetails is on, we'll print long statistics information
  44.242 -  // in the collector policy code, so let's not print this as the output
  44.243 -  // is messy if we do.
  44.244 -  gclog_or_tty->date_stamp(PrintGC && PrintGCDateStamps);
  44.245 -  TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
  44.246 -  TraceTime t(verbose_str, PrintGC && !PrintGCDetails, true, gclog_or_tty);
  44.247 -
  44.248 -  ResourceMark rm;
  44.249 -  assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");
  44.250 -  assert(Thread::current() == VMThread::vm_thread(), "should be in vm thread");
  44.251 -  guarantee(!is_gc_active(), "collection is not reentrant");
  44.252 -  assert(regions_accounted_for(), "Region leakage!");
  44.253 -
  44.254 -  increment_gc_time_stamp();
  44.255 -
  44.256 -  if (g1_policy()->in_young_gc_mode()) {
  44.257 -    assert(check_young_list_well_formed(),
  44.258 -                "young list should be well formed");
  44.259 -  }
  44.260 -
  44.261 -  if (GC_locker::is_active()) {
  44.262 -    return; // GC is disabled (e.g. JNI GetXXXCritical operation)
  44.263 -  }
  44.264 -
  44.265 -  bool abandoned = false;
  44.266 -  { // Call to jvmpi::post_class_unload_events must occur outside of active GC
  44.267 -    IsGCActiveMark x;
  44.268 -
  44.269 -    gc_prologue(false);
  44.270 -    increment_total_collections();
  44.271 +  if (PrintHeapAtGC) {
  44.272 +    Universe::print_heap_before_gc();
  44.273 +  }
  44.274 +
  44.275 +  {
  44.276 +    char verbose_str[128];
  44.277 +    sprintf(verbose_str, "GC pause ");
  44.278 +    if (g1_policy()->in_young_gc_mode()) {
  44.279 +      if (g1_policy()->full_young_gcs())
  44.280 +        strcat(verbose_str, "(young)");
  44.281 +      else
  44.282 +        strcat(verbose_str, "(partial)");
  44.283 +    }
  44.284 +    if (g1_policy()->should_initiate_conc_mark())
  44.285 +      strcat(verbose_str, " (initial-mark)");
  44.286 +
  44.287 +    GCCauseSetter x(this, GCCause::_g1_inc_collection_pause);
  44.288 +
  44.289 +    // if PrintGCDetails is on, we'll print long statistics information
  44.290 +    // in the collector policy code, so let's not print this as the output
  44.291 +    // is messy if we do.
  44.292 +    gclog_or_tty->date_stamp(PrintGC && PrintGCDateStamps);
  44.293 +    TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
  44.294 +    TraceTime t(verbose_str, PrintGC && !PrintGCDetails, true, gclog_or_tty);
  44.295 +
  44.296 +    ResourceMark rm;
  44.297 +    assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");
  44.298 +    assert(Thread::current() == VMThread::vm_thread(), "should be in vm thread");
  44.299 +    guarantee(!is_gc_active(), "collection is not reentrant");
  44.300 +    assert(regions_accounted_for(), "Region leakage!");
  44.301 +
  44.302 +    increment_gc_time_stamp();
  44.303 +
  44.304 +    if (g1_policy()->in_young_gc_mode()) {
  44.305 +      assert(check_young_list_well_formed(),
  44.306 +             "young list should be well formed");
  44.307 +    }
  44.308 +
  44.309 +    if (GC_locker::is_active()) {
  44.310 +      return; // GC is disabled (e.g. JNI GetXXXCritical operation)
  44.311 +    }
  44.312 +
  44.313 +    bool abandoned = false;
  44.314 +    { // Call to jvmpi::post_class_unload_events must occur outside of active GC
  44.315 +      IsGCActiveMark x;
  44.316 +
  44.317 +      gc_prologue(false);
  44.318 +      increment_total_collections(false /* full gc */);
  44.319  
  44.320  #if G1_REM_SET_LOGGING
  44.321 -    gclog_or_tty->print_cr("\nJust chose CS, heap:");
  44.322 -    print();
  44.323 -#endif
  44.324 -
  44.325 -    if (VerifyBeforeGC && total_collections() >= VerifyGCStartAt) {
  44.326 -      HandleMark hm;  // Discard invalid handles created during verification
  44.327 -      prepare_for_verify();
  44.328 -      gclog_or_tty->print(" VerifyBeforeGC:");
  44.329 -      Universe::verify(false);
  44.330 -    }
  44.331 -
  44.332 -    COMPILER2_PRESENT(DerivedPointerTable::clear());
  44.333 -
  44.334 -    // We want to turn off ref discovery, if necessary, and turn it back on
  44.335 -    // on again later if we do.
  44.336 -    bool was_enabled = ref_processor()->discovery_enabled();
  44.337 -    if (was_enabled) ref_processor()->disable_discovery();
  44.338 -
  44.339 -    // Forget the current alloc region (we might even choose it to be part
  44.340 -    // of the collection set!).
  44.341 -    abandon_cur_alloc_region();
  44.342 -
  44.343 -    // The elapsed time induced by the start time below deliberately elides
  44.344 -    // the possible verification above.
  44.345 -    double start_time_sec = os::elapsedTime();
  44.346 -    GCOverheadReporter::recordSTWStart(start_time_sec);
  44.347 -    size_t start_used_bytes = used();
  44.348 -    if (!G1ConcMark) {
  44.349 -      do_sync_mark();
  44.350 -    }
  44.351 -
  44.352 -    g1_policy()->record_collection_pause_start(start_time_sec,
  44.353 -                                               start_used_bytes);
  44.354 -
  44.355 -    guarantee(_in_cset_fast_test == NULL, "invariant");
  44.356 -    guarantee(_in_cset_fast_test_base == NULL, "invariant");
  44.357 -    _in_cset_fast_test_length = max_regions();
  44.358 -    _in_cset_fast_test_base =
  44.359 -                             NEW_C_HEAP_ARRAY(bool, _in_cset_fast_test_length);
  44.360 -    memset(_in_cset_fast_test_base, false,
  44.361 -                                     _in_cset_fast_test_length * sizeof(bool));
  44.362 -    // We're biasing _in_cset_fast_test to avoid subtracting the
  44.363 -    // beginning of the heap every time we want to index; basically
  44.364 -    // it's the same with what we do with the card table.
  44.365 -    _in_cset_fast_test = _in_cset_fast_test_base -
  44.366 -              ((size_t) _g1_reserved.start() >> HeapRegion::LogOfHRGrainBytes);
  44.367 -
  44.368 -#if SCAN_ONLY_VERBOSE
  44.369 -    _young_list->print();
  44.370 -#endif // SCAN_ONLY_VERBOSE
  44.371 -
  44.372 -    if (g1_policy()->should_initiate_conc_mark()) {
  44.373 -      concurrent_mark()->checkpointRootsInitialPre();
  44.374 -    }
  44.375 -    save_marks();
  44.376 -
  44.377 -    // We must do this before any possible evacuation that should propagate
  44.378 -    // marks.
  44.379 -    if (mark_in_progress()) {
  44.380 -      double start_time_sec = os::elapsedTime();
  44.381 -
  44.382 -      _cm->drainAllSATBBuffers();
  44.383 -      double finish_mark_ms = (os::elapsedTime() - start_time_sec) * 1000.0;
  44.384 -      g1_policy()->record_satb_drain_time(finish_mark_ms);
  44.385 -
  44.386 -    }
  44.387 -    // Record the number of elements currently on the mark stack, so we
  44.388 -    // only iterate over these.  (Since evacuation may add to the mark
  44.389 -    // stack, doing more exposes race conditions.)  If no mark is in
  44.390 -    // progress, this will be zero.
  44.391 -    _cm->set_oops_do_bound();
  44.392 -
  44.393 -    assert(regions_accounted_for(), "Region leakage.");
  44.394 -
  44.395 -    if (mark_in_progress())
  44.396 -      concurrent_mark()->newCSet();
  44.397 -
  44.398 -    // Now choose the CS.
  44.399 -    g1_policy()->choose_collection_set();
  44.400 -
  44.401 -    // We may abandon a pause if we find no region that will fit in the MMU
  44.402 -    // pause.
  44.403 -    bool abandoned = (g1_policy()->collection_set() == NULL);
  44.404 -
  44.405 -    // Nothing to do if we were unable to choose a collection set.
  44.406 -    if (!abandoned) {
  44.407 -#if G1_REM_SET_LOGGING
  44.408 -      gclog_or_tty->print_cr("\nAfter pause, heap:");
  44.409 +      gclog_or_tty->print_cr("\nJust chose CS, heap:");
  44.410        print();
  44.411  #endif
  44.412  
  44.413 -      setup_surviving_young_words();
  44.414 -
  44.415 -      // Set up the gc allocation regions.
  44.416 -      get_gc_alloc_regions();
  44.417 -
  44.418 -      // Actually do the work...
  44.419 -      evacuate_collection_set();
  44.420 -      free_collection_set(g1_policy()->collection_set());
  44.421 -      g1_policy()->clear_collection_set();
  44.422 -
  44.423 -      FREE_C_HEAP_ARRAY(bool, _in_cset_fast_test_base);
  44.424 -      // this is more for peace of mind; we're nulling them here and
  44.425 -      // we're expecting them to be null at the beginning of the next GC
  44.426 -      _in_cset_fast_test = NULL;
  44.427 -      _in_cset_fast_test_base = NULL;
  44.428 -
  44.429 -      release_gc_alloc_regions(false /* totally */);
  44.430 -
  44.431 -      cleanup_surviving_young_words();
  44.432 -
  44.433 -      if (g1_policy()->in_young_gc_mode()) {
  44.434 -        _young_list->reset_sampled_info();
  44.435 -        assert(check_young_list_empty(true),
  44.436 -               "young list should be empty");
  44.437 +      if (VerifyBeforeGC && total_collections() >= VerifyGCStartAt) {
  44.438 +        HandleMark hm;  // Discard invalid handles created during verification
  44.439 +        prepare_for_verify();
  44.440 +        gclog_or_tty->print(" VerifyBeforeGC:");
  44.441 +        Universe::verify(false);
  44.442 +      }
  44.443 +
  44.444 +      COMPILER2_PRESENT(DerivedPointerTable::clear());
  44.445 +
  44.446 +      // We want to turn off ref discovery, if necessary, and turn it back on
  44.447 +      // on again later if we do. XXX Dubious: why is discovery disabled?
  44.448 +      bool was_enabled = ref_processor()->discovery_enabled();
  44.449 +      if (was_enabled) ref_processor()->disable_discovery();
  44.450 +
  44.451 +      // Forget the current alloc region (we might even choose it to be part
  44.452 +      // of the collection set!).
  44.453 +      abandon_cur_alloc_region();
  44.454 +
  44.455 +      // The elapsed time induced by the start time below deliberately elides
  44.456 +      // the possible verification above.
  44.457 +      double start_time_sec = os::elapsedTime();
  44.458 +      GCOverheadReporter::recordSTWStart(start_time_sec);
  44.459 +      size_t start_used_bytes = used();
  44.460 +
  44.461 +      g1_policy()->record_collection_pause_start(start_time_sec,
  44.462 +                                                 start_used_bytes);
  44.463 +
  44.464 +      guarantee(_in_cset_fast_test == NULL, "invariant");
  44.465 +      guarantee(_in_cset_fast_test_base == NULL, "invariant");
  44.466 +      _in_cset_fast_test_length = max_regions();
  44.467 +      _in_cset_fast_test_base =
  44.468 +                             NEW_C_HEAP_ARRAY(bool, _in_cset_fast_test_length);
  44.469 +      memset(_in_cset_fast_test_base, false,
  44.470 +                                     _in_cset_fast_test_length * sizeof(bool));
  44.471 +      // We're biasing _in_cset_fast_test to avoid subtracting the
  44.472 +      // beginning of the heap every time we want to index; basically
  44.473 +      // it's the same with what we do with the card table.
  44.474 +      _in_cset_fast_test = _in_cset_fast_test_base -
  44.475 +              ((size_t) _g1_reserved.start() >> HeapRegion::LogOfHRGrainBytes);
  44.476  
  44.477  #if SCAN_ONLY_VERBOSE
  44.478 -        _young_list->print();
  44.479 +      _young_list->print();
  44.480  #endif // SCAN_ONLY_VERBOSE
  44.481  
  44.482 -        g1_policy()->record_survivor_regions(_young_list->survivor_length(),
  44.483 -                                             _young_list->first_survivor_region(),
  44.484 -                                             _young_list->last_survivor_region());
  44.485 -        _young_list->reset_auxilary_lists();
  44.486 +      if (g1_policy()->should_initiate_conc_mark()) {
  44.487 +        concurrent_mark()->checkpointRootsInitialPre();
  44.488        }
  44.489 -    } else {
  44.490 -      COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
  44.491 +      save_marks();
  44.492 +
  44.493 +      // We must do this before any possible evacuation that should propagate
  44.494 +      // marks.
  44.495 +      if (mark_in_progress()) {
  44.496 +        double start_time_sec = os::elapsedTime();
  44.497 +
  44.498 +        _cm->drainAllSATBBuffers();
  44.499 +        double finish_mark_ms = (os::elapsedTime() - start_time_sec) * 1000.0;
  44.500 +        g1_policy()->record_satb_drain_time(finish_mark_ms);
  44.501 +      }
  44.502 +      // Record the number of elements currently on the mark stack, so we
  44.503 +      // only iterate over these.  (Since evacuation may add to the mark
  44.504 +      // stack, doing more exposes race conditions.)  If no mark is in
  44.505 +      // progress, this will be zero.
  44.506 +      _cm->set_oops_do_bound();
  44.507 +
  44.508 +      assert(regions_accounted_for(), "Region leakage.");
  44.509 +
  44.510 +      if (mark_in_progress())
  44.511 +        concurrent_mark()->newCSet();
  44.512 +
  44.513 +      // Now choose the CS.
  44.514 +      g1_policy()->choose_collection_set();
  44.515 +
  44.516 +      // We may abandon a pause if we find no region that will fit in the MMU
  44.517 +      // pause.
  44.518 +      bool abandoned = (g1_policy()->collection_set() == NULL);
  44.519 +
  44.520 +      // Nothing to do if we were unable to choose a collection set.
  44.521 +      if (!abandoned) {
  44.522 +#if G1_REM_SET_LOGGING
  44.523 +        gclog_or_tty->print_cr("\nAfter pause, heap:");
  44.524 +        print();
  44.525 +#endif
  44.526 +
  44.527 +        setup_surviving_young_words();
  44.528 +
  44.529 +        // Set up the gc allocation regions.
  44.530 +        get_gc_alloc_regions();
  44.531 +
  44.532 +        // Actually do the work...
  44.533 +        evacuate_collection_set();
  44.534 +        free_collection_set(g1_policy()->collection_set());
  44.535 +        g1_policy()->clear_collection_set();
  44.536 +
  44.537 +        FREE_C_HEAP_ARRAY(bool, _in_cset_fast_test_base);
  44.538 +        // this is more for peace of mind; we're nulling them here and
  44.539 +        // we're expecting them to be null at the beginning of the next GC
  44.540 +        _in_cset_fast_test = NULL;
  44.541 +        _in_cset_fast_test_base = NULL;
  44.542 +
  44.543 +        release_gc_alloc_regions(false /* totally */);
  44.544 +
  44.545 +        cleanup_surviving_young_words();
  44.546 +
  44.547 +        if (g1_policy()->in_young_gc_mode()) {
  44.548 +          _young_list->reset_sampled_info();
  44.549 +          assert(check_young_list_empty(true),
  44.550 +                 "young list should be empty");
  44.551 +
  44.552 +#if SCAN_ONLY_VERBOSE
  44.553 +          _young_list->print();
  44.554 +#endif // SCAN_ONLY_VERBOSE
  44.555 +
  44.556 +          g1_policy()->record_survivor_regions(_young_list->survivor_length(),
  44.557 +                                          _young_list->first_survivor_region(),
  44.558 +                                          _young_list->last_survivor_region());
  44.559 +          _young_list->reset_auxilary_lists();
  44.560 +        }
  44.561 +      } else {
  44.562 +        COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
  44.563 +      }
  44.564 +
  44.565 +      if (evacuation_failed()) {
  44.566 +        _summary_bytes_used = recalculate_used();
  44.567 +      } else {
  44.568 +        // The "used" of the the collection set have already been subtracted
  44.569 +        // when they were freed.  Add in the bytes evacuated.
  44.570 +        _summary_bytes_used += g1_policy()->bytes_in_to_space();
  44.571 +      }
  44.572 +
  44.573 +      if (g1_policy()->in_young_gc_mode() &&
  44.574 +          g1_policy()->should_initiate_conc_mark()) {
  44.575 +        concurrent_mark()->checkpointRootsInitialPost();
  44.576 +        set_marking_started();
  44.577 +        // CAUTION: after the doConcurrentMark() call below,
  44.578 +        // the concurrent marking thread(s) could be running
  44.579 +        // concurrently with us. Make sure that anything after
  44.580 +        // this point does not assume that we are the only GC thread
  44.581 +        // running. Note: of course, the actual marking work will
  44.582 +        // not start until the safepoint itself is released in
  44.583 +        // ConcurrentGCThread::safepoint_desynchronize().
  44.584 +        doConcurrentMark();
  44.585 +      }
  44.586 +
  44.587 +#if SCAN_ONLY_VERBOSE
  44.588 +      _young_list->print();
  44.589 +#endif // SCAN_ONLY_VERBOSE
  44.590 +
  44.591 +      double end_time_sec = os::elapsedTime();
  44.592 +      double pause_time_ms = (end_time_sec - start_time_sec) * MILLIUNITS;
  44.593 +      g1_policy()->record_pause_time_ms(pause_time_ms);
  44.594 +      GCOverheadReporter::recordSTWEnd(end_time_sec);
  44.595 +      g1_policy()->record_collection_pause_end(abandoned);
  44.596 +
  44.597 +      assert(regions_accounted_for(), "Region leakage.");
  44.598 +
  44.599 +      if (VerifyAfterGC && total_collections() >= VerifyGCStartAt) {
  44.600 +        HandleMark hm;  // Discard invalid handles created during verification
  44.601 +        gclog_or_tty->print(" VerifyAfterGC:");
  44.602 +        prepare_for_verify();
  44.603 +        Universe::verify(false);
  44.604 +      }
  44.605 +
  44.606 +      if (was_enabled) ref_processor()->enable_discovery();
  44.607 +
  44.608 +      {
  44.609 +        size_t expand_bytes = g1_policy()->expansion_amount();
  44.610 +        if (expand_bytes > 0) {
  44.611 +          size_t bytes_before = capacity();
  44.612 +          expand(expand_bytes);
  44.613 +        }
  44.614 +      }
  44.615 +
  44.616 +      if (mark_in_progress()) {
  44.617 +        concurrent_mark()->update_g1_committed();
  44.618 +      }
  44.619 +
  44.620 +#ifdef TRACESPINNING
  44.621 +      ParallelTaskTerminator::print_termination_counts();
  44.622 +#endif
  44.623 +
  44.624 +      gc_epilogue(false);
  44.625      }
  44.626  
  44.627 -    if (evacuation_failed()) {
  44.628 -      _summary_bytes_used = recalculate_used();
  44.629 -    } else {
  44.630 -      // The "used" of the the collection set have already been subtracted
  44.631 -      // when they were freed.  Add in the bytes evacuated.
  44.632 -      _summary_bytes_used += g1_policy()->bytes_in_to_space();
  44.633 +    assert(verify_region_lists(), "Bad region lists.");
  44.634 +
  44.635 +    if (ExitAfterGCNum > 0 && total_collections() == ExitAfterGCNum) {
  44.636 +      gclog_or_tty->print_cr("Stopping after GC #%d", ExitAfterGCNum);
  44.637 +      print_tracing_info();
  44.638 +      vm_exit(-1);
  44.639      }
  44.640 -
  44.641 -    if (g1_policy()->in_young_gc_mode() &&
  44.642 -        g1_policy()->should_initiate_conc_mark()) {
  44.643 -      concurrent_mark()->checkpointRootsInitialPost();
  44.644 -      set_marking_started();
  44.645 -      doConcurrentMark();
  44.646 -    }
  44.647 -
  44.648 -#if SCAN_ONLY_VERBOSE
  44.649 -    _young_list->print();
  44.650 -#endif // SCAN_ONLY_VERBOSE
  44.651 -
  44.652 -    double end_time_sec = os::elapsedTime();
  44.653 -    double pause_time_ms = (end_time_sec - start_time_sec) * MILLIUNITS;
  44.654 -    g1_policy()->record_pause_time_ms(pause_time_ms);
  44.655 -    GCOverheadReporter::recordSTWEnd(end_time_sec);
  44.656 -    g1_policy()->record_collection_pause_end(abandoned);
  44.657 -
  44.658 -    assert(regions_accounted_for(), "Region leakage.");
  44.659 -
  44.660 -    if (VerifyAfterGC && total_collections() >= VerifyGCStartAt) {
  44.661 -      HandleMark hm;  // Discard invalid handles created during verification
  44.662 -      gclog_or_tty->print(" VerifyAfterGC:");
  44.663 -      prepare_for_verify();
  44.664 -      Universe::verify(false);
  44.665 -    }
  44.666 -
  44.667 -    if (was_enabled) ref_processor()->enable_discovery();
  44.668 -
  44.669 -    {
  44.670 -      size_t expand_bytes = g1_policy()->expansion_amount();
  44.671 -      if (expand_bytes > 0) {
  44.672 -        size_t bytes_before = capacity();
  44.673 -        expand(expand_bytes);
  44.674 -      }
  44.675 -    }
  44.676 -
  44.677 -    if (mark_in_progress()) {
  44.678 -      concurrent_mark()->update_g1_committed();
  44.679 -    }
  44.680 -
  44.681 -#ifdef TRACESPINNING
  44.682 -    ParallelTaskTerminator::print_termination_counts();
  44.683 -#endif
  44.684 -
  44.685 -    gc_epilogue(false);
  44.686 -  }
  44.687 -
  44.688 -  assert(verify_region_lists(), "Bad region lists.");
  44.689 -
  44.690 -  if (ExitAfterGCNum > 0 && total_collections() == ExitAfterGCNum) {
  44.691 -    gclog_or_tty->print_cr("Stopping after GC #%d", ExitAfterGCNum);
  44.692 -    print_tracing_info();
  44.693 -    vm_exit(-1);
  44.694 +  }
  44.695 +
  44.696 +  if (PrintHeapAtGC) {
  44.697 +    Universe::print_heap_after_gc();
  44.698    }
  44.699  }
  44.700  
  44.701 @@ -2930,6 +2980,7 @@
  44.702  
  44.703    for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {
  44.704      assert(_gc_alloc_regions[ap] == NULL, "invariant");
  44.705 +    assert(_gc_alloc_region_counts[ap] == 0, "invariant");
  44.706  
  44.707      // Create new GC alloc regions.
  44.708      HeapRegion* alloc_region = _retained_gc_alloc_regions[ap];
  44.709 @@ -2958,6 +3009,9 @@
  44.710      if (alloc_region == NULL) {
  44.711        // we will get a new GC alloc region
  44.712        alloc_region = newAllocRegionWithExpansion(ap, 0);
  44.713 +    } else {
  44.714 +      // the region was retained from the last collection
  44.715 +      ++_gc_alloc_region_counts[ap];
  44.716      }
  44.717  
  44.718      if (alloc_region != NULL) {
  44.719 @@ -2996,11 +3050,11 @@
  44.720    for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {
  44.721      HeapRegion* r = _gc_alloc_regions[ap];
  44.722      _retained_gc_alloc_regions[ap] = NULL;
  44.723 +    _gc_alloc_region_counts[ap] = 0;
  44.724  
  44.725      if (r != NULL) {
  44.726        // we retain nothing on _gc_alloc_regions between GCs
  44.727        set_gc_alloc_region(ap, NULL);
  44.728 -      _gc_alloc_region_counts[ap] = 0;
  44.729  
  44.730        if (r->is_empty()) {
  44.731          // we didn't actually allocate anything in it; let's just put
  44.732 @@ -3088,10 +3142,8 @@
  44.733    G1CollectedHeap* _g1;
  44.734  public:
  44.735    G1KeepAliveClosure(G1CollectedHeap* g1) : _g1(g1) {}
  44.736 -  void do_oop(narrowOop* p) {
  44.737 -    guarantee(false, "NYI");
  44.738 -  }
  44.739 -  void do_oop(oop* p) {
  44.740 +  void do_oop(narrowOop* p) { guarantee(false, "Not needed"); }
  44.741 +  void do_oop(      oop* p) {
  44.742      oop obj = *p;
  44.743  #ifdef G1_DEBUG
  44.744      if (PrintGC && Verbose) {
  44.745 @@ -3103,7 +3155,6 @@
  44.746      if (_g1->obj_in_cs(obj)) {
  44.747        assert( obj->is_forwarded(), "invariant" );
  44.748        *p = obj->forwardee();
  44.749 -
  44.750  #ifdef G1_DEBUG
  44.751        gclog_or_tty->print_cr("     in CSet: moved "PTR_FORMAT" -> "PTR_FORMAT,
  44.752                               (void*) obj, (void*) *p);
  44.753 @@ -3120,12 +3171,12 @@
  44.754    UpdateRSetImmediate(G1CollectedHeap* g1) :
  44.755      _g1(g1), _g1_rem_set(g1->g1_rem_set()) {}
  44.756  
  44.757 -  void do_oop(narrowOop* p) {
  44.758 -    guarantee(false, "NYI");
  44.759 -  }
  44.760 -  void do_oop(oop* p) {
  44.761 +  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
  44.762 +  virtual void do_oop(      oop* p) { do_oop_work(p); }
  44.763 +  template <class T> void do_oop_work(T* p) {
  44.764      assert(_from->is_in_reserved(p), "paranoia");
  44.765 -    if (*p != NULL && !_from->is_survivor()) {
  44.766 +    T heap_oop = oopDesc::load_heap_oop(p);
  44.767 +    if (!oopDesc::is_null(heap_oop) && !_from->is_survivor()) {
  44.768        _g1_rem_set->par_write_ref(_from, p, 0);
  44.769      }
  44.770    }
  44.771 @@ -3141,12 +3192,12 @@
  44.772    UpdateRSetDeferred(G1CollectedHeap* g1, DirtyCardQueue* dcq) :
  44.773      _g1(g1), _ct_bs((CardTableModRefBS*)_g1->barrier_set()), _dcq(dcq) {}
  44.774  
  44.775 -  void do_oop(narrowOop* p) {
  44.776 -    guarantee(false, "NYI");
  44.777 -  }
  44.778 -  void do_oop(oop* p) {
  44.779 +  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
  44.780 +  virtual void do_oop(      oop* p) { do_oop_work(p); }
  44.781 +  template <class T> void do_oop_work(T* p) {
  44.782      assert(_from->is_in_reserved(p), "paranoia");
  44.783 -    if (!_from->is_in_reserved(*p) && !_from->is_survivor()) {
  44.784 +    if (!_from->is_in_reserved(oopDesc::load_decode_heap_oop(p)) &&
  44.785 +        !_from->is_survivor()) {
  44.786        size_t card_index = _ct_bs->index_for(p);
  44.787        if (_ct_bs->mark_card_deferred(card_index)) {
  44.788          _dcq->enqueue((jbyte*)_ct_bs->byte_for_index(card_index));
  44.789 @@ -3501,614 +3552,66 @@
  44.790    fill_with_object(block, free_words);
  44.791  }
  44.792  
  44.793 -#define use_local_bitmaps         1
  44.794 -#define verify_local_bitmaps      0
  44.795 -
  44.796  #ifndef PRODUCT
  44.797 -
  44.798 -class GCLabBitMap;
  44.799 -class GCLabBitMapClosure: public BitMapClosure {
  44.800 -private:
  44.801 -  ConcurrentMark* _cm;
  44.802 -  GCLabBitMap*    _bitmap;
  44.803 -
  44.804 -public:
  44.805 -  GCLabBitMapClosure(ConcurrentMark* cm,
  44.806 -                     GCLabBitMap* bitmap) {
  44.807 -    _cm     = cm;
  44.808 -    _bitmap = bitmap;
  44.809 -  }
  44.810 -
  44.811 -  virtual bool do_bit(size_t offset);
  44.812 -};
  44.813 -
  44.814 -#endif // PRODUCT
  44.815 -
  44.816 -#define oop_buffer_length 256
  44.817 -
  44.818 -class GCLabBitMap: public BitMap {
  44.819 -private:
  44.820 -  ConcurrentMark* _cm;
  44.821 -
  44.822 -  int       _shifter;
  44.823 -  size_t    _bitmap_word_covers_words;
  44.824 -
  44.825 -  // beginning of the heap
  44.826 -  HeapWord* _heap_start;
  44.827 -
  44.828 -  // this is the actual start of the GCLab
  44.829 -  HeapWord* _real_start_word;
  44.830 -
  44.831 -  // this is the actual end of the GCLab
  44.832 -  HeapWord* _real_end_word;
  44.833 -
  44.834 -  // this is the first word, possibly located before the actual start
  44.835 -  // of the GCLab, that corresponds to the first bit of the bitmap
  44.836 -  HeapWord* _start_word;
  44.837 -
  44.838 -  // size of a GCLab in words
  44.839 -  size_t _gclab_word_size;
  44.840 -
  44.841 -  static int shifter() {
  44.842 -    return MinObjAlignment - 1;
  44.843 -  }
  44.844 -
  44.845 -  // how many heap words does a single bitmap word corresponds to?
  44.846 -  static size_t bitmap_word_covers_words() {
  44.847 -    return BitsPerWord << shifter();
  44.848 -  }
  44.849 -
  44.850 -  static size_t gclab_word_size() {
  44.851 -    return G1ParallelGCAllocBufferSize / HeapWordSize;
  44.852 -  }
  44.853 -
  44.854 -  static size_t bitmap_size_in_bits() {
  44.855 -    size_t bits_in_bitmap = gclab_word_size() >> shifter();
  44.856 -    // We are going to ensure that the beginning of a word in this
  44.857 -    // bitmap also corresponds to the beginning of a word in the
  44.858 -    // global marking bitmap. To handle the case where a GCLab
  44.859 -    // starts from the middle of the bitmap, we need to add enough
  44.860 -    // space (i.e. up to a bitmap word) to ensure that we have
  44.861 -    // enough bits in the bitmap.
  44.862 -    return bits_in_bitmap + BitsPerWord - 1;
  44.863 -  }
  44.864 -public:
  44.865 -  GCLabBitMap(HeapWord* heap_start)
  44.866 -    : BitMap(bitmap_size_in_bits()),
  44.867 -      _cm(G1CollectedHeap::heap()->concurrent_mark()),
  44.868 -      _shifter(shifter()),
  44.869 -      _bitmap_word_covers_words(bitmap_word_covers_words()),
  44.870 -      _heap_start(heap_start),
  44.871 -      _gclab_word_size(gclab_word_size()),
  44.872 -      _real_start_word(NULL),
  44.873 -      _real_end_word(NULL),
  44.874 -      _start_word(NULL)
  44.875 -  {
  44.876 -    guarantee( size_in_words() >= bitmap_size_in_words(),
  44.877 -               "just making sure");
  44.878 -  }
  44.879 -
  44.880 -  inline unsigned heapWordToOffset(HeapWord* addr) {
  44.881 -    unsigned offset = (unsigned) pointer_delta(addr, _start_word) >> _shifter;
  44.882 -    assert(offset < size(), "offset should be within bounds");
  44.883 -    return offset;
  44.884 -  }
  44.885 -
  44.886 -  inline HeapWord* offsetToHeapWord(size_t offset) {
  44.887 -    HeapWord* addr =  _start_word + (offset << _shifter);
  44.888 -    assert(_real_start_word <= addr && addr < _real_end_word, "invariant");
  44.889 -    return addr;
  44.890 -  }
  44.891 -
  44.892 -  bool fields_well_formed() {
  44.893 -    bool ret1 = (_real_start_word == NULL) &&
  44.894 -                (_real_end_word == NULL) &&
  44.895 -                (_start_word == NULL);
  44.896 -    if (ret1)
  44.897 -      return true;
  44.898 -
  44.899 -    bool ret2 = _real_start_word >= _start_word &&
  44.900 -      _start_word < _real_end_word &&
  44.901 -      (_real_start_word + _gclab_word_size) == _real_end_word &&
  44.902 -      (_start_word + _gclab_word_size + _bitmap_word_covers_words)
  44.903 -                                                              > _real_end_word;
  44.904 -    return ret2;
  44.905 -  }
  44.906 -
  44.907 -  inline bool mark(HeapWord* addr) {
  44.908 -    guarantee(use_local_bitmaps, "invariant");
  44.909 -    assert(fields_well_formed(), "invariant");
  44.910 -
  44.911 -    if (addr >= _real_start_word && addr < _real_end_word) {
  44.912 -      assert(!isMarked(addr), "should not have already been marked");
  44.913 -
  44.914 -      // first mark it on the bitmap
  44.915 -      at_put(heapWordToOffset(addr), true);
  44.916 -
  44.917 -      return true;
  44.918 -    } else {
  44.919 -      return false;
  44.920 -    }
  44.921 -  }
  44.922 -
  44.923 -  inline bool isMarked(HeapWord* addr) {
  44.924 -    guarantee(use_local_bitmaps, "invariant");
  44.925 -    assert(fields_well_formed(), "invariant");
  44.926 -
  44.927 -    return at(heapWordToOffset(addr));
  44.928 -  }
  44.929 -
  44.930 -  void set_buffer(HeapWord* start) {
  44.931 -    guarantee(use_local_bitmaps, "invariant");
  44.932 -    clear();
  44.933 -
  44.934 -    assert(start != NULL, "invariant");
  44.935 -    _real_start_word = start;
  44.936 -    _real_end_word   = start + _gclab_word_size;
  44.937 -
  44.938 -    size_t diff =
  44.939 -      pointer_delta(start, _heap_start) % _bitmap_word_covers_words;
  44.940 -    _start_word = start - diff;
  44.941 -
  44.942 -    assert(fields_well_formed(), "invariant");
  44.943 -  }
  44.944 -
  44.945 -#ifndef PRODUCT
  44.946 -  void verify() {
  44.947 -    // verify that the marks have been propagated
  44.948 -    GCLabBitMapClosure cl(_cm, this);
  44.949 -    iterate(&cl);
  44.950 -  }
  44.951 -#endif // PRODUCT
  44.952 -
  44.953 -  void retire() {
  44.954 -    guarantee(use_local_bitmaps, "invariant");
  44.955 -    assert(fields_well_formed(), "invariant");
  44.956 -
  44.957 -    if (_start_word != NULL) {
  44.958 -      CMBitMap*       mark_bitmap = _cm->nextMarkBitMap();
  44.959 -
  44.960 -      // this means that the bitmap was set up for the GCLab
  44.961 -      assert(_real_start_word != NULL && _real_end_word != NULL, "invariant");
  44.962 -
  44.963 -      mark_bitmap->mostly_disjoint_range_union(this,
  44.964 -                                0, // always start from the start of the bitmap
  44.965 -                                _start_word,
  44.966 -                                size_in_words());
  44.967 -      _cm->grayRegionIfNecessary(MemRegion(_real_start_word, _real_end_word));
  44.968 -
  44.969 -#ifndef PRODUCT
  44.970 -      if (use_local_bitmaps && verify_local_bitmaps)
  44.971 -        verify();
  44.972 -#endif // PRODUCT
  44.973 -    } else {
  44.974 -      assert(_real_start_word == NULL && _real_end_word == NULL, "invariant");
  44.975 -    }
  44.976 -  }
  44.977 -
  44.978 -  static size_t bitmap_size_in_words() {
  44.979 -    return (bitmap_size_in_bits() + BitsPerWord - 1) / BitsPerWord;
  44.980 -  }
  44.981 -};
  44.982 -
  44.983 -#ifndef PRODUCT
  44.984 -
  44.985  bool GCLabBitMapClosure::do_bit(size_t offset) {
  44.986    HeapWord* addr = _bitmap->offsetToHeapWord(offset);
  44.987    guarantee(_cm->isMarked(oop(addr)), "it should be!");
  44.988    return true;
  44.989  }
  44.990 -
  44.991  #endif // PRODUCT
  44.992  
  44.993 -class G1ParGCAllocBuffer: public ParGCAllocBuffer {
  44.994 -private:
  44.995 -  bool        _retired;
  44.996 -  bool        _during_marking;
  44.997 -  GCLabBitMap _bitmap;
  44.998 -
  44.999 -public:
 44.1000 -  G1ParGCAllocBuffer() :
 44.1001 -    ParGCAllocBuffer(G1ParallelGCAllocBufferSize / HeapWordSize),
 44.1002 -    _during_marking(G1CollectedHeap::heap()->mark_in_progress()),
 44.1003 -    _bitmap(G1CollectedHeap::heap()->reserved_region().start()),
 44.1004 -    _retired(false)
 44.1005 -  { }
 44.1006 -
 44.1007 -  inline bool mark(HeapWord* addr) {
 44.1008 -    guarantee(use_local_bitmaps, "invariant");
 44.1009 -    assert(_during_marking, "invariant");
 44.1010 -    return _bitmap.mark(addr);
 44.1011 -  }
 44.1012 -
 44.1013 -  inline void set_buf(HeapWord* buf) {
 44.1014 -    if (use_local_bitmaps && _during_marking)
 44.1015 -      _bitmap.set_buffer(buf);
 44.1016 -    ParGCAllocBuffer::set_buf(buf);
 44.1017 -    _retired = false;
 44.1018 -  }
 44.1019 -
 44.1020 -  inline void retire(bool end_of_gc, bool retain) {
 44.1021 -    if (_retired)
 44.1022 -      return;
 44.1023 -    if (use_local_bitmaps && _during_marking) {
 44.1024 -      _bitmap.retire();
 44.1025 -    }
 44.1026 -    ParGCAllocBuffer::retire(end_of_gc, retain);
 44.1027 -    _retired = true;
 44.1028 -  }
 44.1029 -};
 44.1030 -
 44.1031 -
 44.1032 -class G1ParScanThreadState : public StackObj {
 44.1033 -protected:
 44.1034 -  G1CollectedHeap* _g1h;
 44.1035 -  RefToScanQueue*  _refs;
 44.1036 -  DirtyCardQueue   _dcq;
 44.1037 -  CardTableModRefBS* _ct_bs;
 44.1038 -  G1RemSet* _g1_rem;
 44.1039 -
 44.1040 -  typedef GrowableArray<oop*> OverflowQueue;
 44.1041 -  OverflowQueue* _overflowed_refs;
 44.1042 -
 44.1043 -  G1ParGCAllocBuffer _alloc_buffers[GCAllocPurposeCount];
 44.1044 -  ageTable           _age_table;
 44.1045 -
 44.1046 -  size_t           _alloc_buffer_waste;
 44.1047 -  size_t           _undo_waste;
 44.1048 -
 44.1049 -  OopsInHeapRegionClosure*      _evac_failure_cl;
 44.1050 -  G1ParScanHeapEvacClosure*     _evac_cl;
 44.1051 -  G1ParScanPartialArrayClosure* _partial_scan_cl;
 44.1052 -
 44.1053 -  int _hash_seed;
 44.1054 -  int _queue_num;
 44.1055 -
 44.1056 -  int _term_attempts;
 44.1057 +G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h, int queue_num)
 44.1058 +  : _g1h(g1h),
 44.1059 +    _refs(g1h->task_queue(queue_num)),
 44.1060 +    _dcq(&g1h->dirty_card_queue_set()),
 44.1061 +    _ct_bs((CardTableModRefBS*)_g1h->barrier_set()),
 44.1062 +    _g1_rem(g1h->g1_rem_set()),
 44.1063 +    _hash_seed(17), _queue_num(queue_num),
 44.1064 +    _term_attempts(0),
 44.1065 +    _age_table(false),
 44.1066  #if G1_DETAILED_STATS
 44.1067 -  int _pushes, _pops, _steals, _steal_attempts;
 44.1068 -  int _overflow_pushes;
 44.1069 +    _pushes(0), _pops(0), _steals(0),
 44.1070 +    _steal_attempts(0),  _overflow_pushes(0),
 44.1071  #endif
 44.1072 -
 44.1073 -  double _start;
 44.1074 -  double _start_strong_roots;
 44.1075 -  double _strong_roots_time;
 44.1076 -  double _start_term;
 44.1077 -  double _term_time;
 44.1078 -
 44.1079 -  // Map from young-age-index (0 == not young, 1 is youngest) to
 44.1080 -  // surviving words. base is what we get back from the malloc call
 44.1081 -  size_t* _surviving_young_words_base;
 44.1082 -  // this points into the array, as we use the first few entries for padding
 44.1083 -  size_t* _surviving_young_words;
 44.1084 -
 44.1085 -#define PADDING_ELEM_NUM (64 / sizeof(size_t))
 44.1086 -
 44.1087 -  void   add_to_alloc_buffer_waste(size_t waste) { _alloc_buffer_waste += waste; }
 44.1088 -
 44.1089 -  void   add_to_undo_waste(size_t waste)         { _undo_waste += waste; }
 44.1090 -
 44.1091 -  DirtyCardQueue& dirty_card_queue()             { return _dcq;  }
 44.1092 -  CardTableModRefBS* ctbs()                      { return _ct_bs; }
 44.1093 -
 44.1094 -  void immediate_rs_update(HeapRegion* from, oop* p, int tid) {
 44.1095 -    if (!from->is_survivor()) {
 44.1096 -      _g1_rem->par_write_ref(from, p, tid);
 44.1097 -    }
 44.1098 -  }
 44.1099 -
 44.1100 -  void deferred_rs_update(HeapRegion* from, oop* p, int tid) {
 44.1101 -    // If the new value of the field points to the same region or
 44.1102 -    // is the to-space, we don't need to include it in the Rset updates.
 44.1103 -    if (!from->is_in_reserved(*p) && !from->is_survivor()) {
 44.1104 -      size_t card_index = ctbs()->index_for(p);
 44.1105 -      // If the card hasn't been added to the buffer, do it.
 44.1106 -      if (ctbs()->mark_card_deferred(card_index)) {
 44.1107 -        dirty_card_queue().enqueue((jbyte*)ctbs()->byte_for_index(card_index));
 44.1108 -      }
 44.1109 -    }
 44.1110 -  }
 44.1111 -
 44.1112 -public:
 44.1113 -  G1ParScanThreadState(G1CollectedHeap* g1h, int queue_num)
 44.1114 -    : _g1h(g1h),
 44.1115 -      _refs(g1h->task_queue(queue_num)),
 44.1116 -      _dcq(&g1h->dirty_card_queue_set()),
 44.1117 -      _ct_bs((CardTableModRefBS*)_g1h->barrier_set()),
 44.1118 -      _g1_rem(g1h->g1_rem_set()),
 44.1119 -      _hash_seed(17), _queue_num(queue_num),
 44.1120 -      _term_attempts(0),
 44.1121 -      _age_table(false),
 44.1122 -#if G1_DETAILED_STATS
 44.1123 -      _pushes(0), _pops(0), _steals(0),
 44.1124 -      _steal_attempts(0),  _overflow_pushes(0),
 44.1125 -#endif
 44.1126 -      _strong_roots_time(0), _term_time(0),
 44.1127 -      _alloc_buffer_waste(0), _undo_waste(0)
 44.1128 -  {
 44.1129 -    // we allocate G1YoungSurvRateNumRegions plus one entries, since
 44.1130 -    // we "sacrifice" entry 0 to keep track of surviving bytes for
 44.1131 -    // non-young regions (where the age is -1)
 44.1132 -    // We also add a few elements at the beginning and at the end in
 44.1133 -    // an attempt to eliminate cache contention
 44.1134 -    size_t real_length = 1 + _g1h->g1_policy()->young_cset_length();
 44.1135 -    size_t array_length = PADDING_ELEM_NUM +
 44.1136 -                          real_length +
 44.1137 -                          PADDING_ELEM_NUM;
 44.1138 -    _surviving_young_words_base = NEW_C_HEAP_ARRAY(size_t, array_length);
 44.1139 -    if (_surviving_young_words_base == NULL)
 44.1140 -      vm_exit_out_of_memory(array_length * sizeof(size_t),
 44.1141 -                            "Not enough space for young surv histo.");
 44.1142 -    _surviving_young_words = _surviving_young_words_base + PADDING_ELEM_NUM;
 44.1143 -    memset(_surviving_young_words, 0, real_length * sizeof(size_t));
 44.1144 -
 44.1145 -    _overflowed_refs = new OverflowQueue(10);
 44.1146 -
 44.1147 -    _start = os::elapsedTime();
 44.1148 -  }
 44.1149 -
 44.1150 -  ~G1ParScanThreadState() {
 44.1151 -    FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_base);
 44.1152 -  }
 44.1153 -
 44.1154 -  RefToScanQueue*   refs()            { return _refs;             }
 44.1155 -  OverflowQueue*    overflowed_refs() { return _overflowed_refs;  }
 44.1156 -  ageTable*         age_table()       { return &_age_table;       }
 44.1157 -
 44.1158 -  G1ParGCAllocBuffer* alloc_buffer(GCAllocPurpose purpose) {
 44.1159 -    return &_alloc_buffers[purpose];
 44.1160 -  }
 44.1161 -
 44.1162 -  size_t alloc_buffer_waste()                    { return _alloc_buffer_waste; }
 44.1163 -  size_t undo_waste()                            { return _undo_waste; }
 44.1164 -
 44.1165 -  void push_on_queue(oop* ref) {
 44.1166 -    assert(ref != NULL, "invariant");
 44.1167 -    assert(has_partial_array_mask(ref) || _g1h->obj_in_cs(*ref), "invariant");
 44.1168 -
 44.1169 -    if (!refs()->push(ref)) {
 44.1170 -      overflowed_refs()->push(ref);
 44.1171 -      IF_G1_DETAILED_STATS(note_overflow_push());
 44.1172 -    } else {
 44.1173 -      IF_G1_DETAILED_STATS(note_push());
 44.1174 -    }
 44.1175 -  }
 44.1176 -
 44.1177 -  void pop_from_queue(oop*& ref) {
 44.1178 -    if (!refs()->pop_local(ref)) {
 44.1179 -      ref = NULL;
 44.1180 -    } else {
 44.1181 -      assert(ref != NULL, "invariant");
 44.1182 -      assert(has_partial_array_mask(ref) || _g1h->obj_in_cs(*ref),
 44.1183 -             "invariant");
 44.1184 -
 44.1185 -      IF_G1_DETAILED_STATS(note_pop());
 44.1186 -    }
 44.1187 -  }
 44.1188 -
 44.1189 -  void pop_from_overflow_queue(oop*& ref) {
 44.1190 -    ref = overflowed_refs()->pop();
 44.1191 -  }
 44.1192 -
 44.1193 -  int refs_to_scan()                             { return refs()->size();                 }
 44.1194 -  int overflowed_refs_to_scan()                  { return overflowed_refs()->length();    }
 44.1195 -
 44.1196 -  void update_rs(HeapRegion* from, oop* p, int tid) {
 44.1197 -    if (G1DeferredRSUpdate) {
 44.1198 -      deferred_rs_update(from, p, tid);
 44.1199 -    } else {
 44.1200 -      immediate_rs_update(from, p, tid);
 44.1201 -    }
 44.1202 -  }
 44.1203 -
 44.1204 -  HeapWord* allocate_slow(GCAllocPurpose purpose, size_t word_sz) {
 44.1205 -
 44.1206 -    HeapWord* obj = NULL;
 44.1207 -    if (word_sz * 100 <
 44.1208 -        (size_t)(G1ParallelGCAllocBufferSize / HeapWordSize) *
 44.1209 -                                                  ParallelGCBufferWastePct) {
 44.1210 -      G1ParGCAllocBuffer* alloc_buf = alloc_buffer(purpose);
 44.1211 -      add_to_alloc_buffer_waste(alloc_buf->words_remaining());
 44.1212 -      alloc_buf->retire(false, false);
 44.1213 -
 44.1214 -      HeapWord* buf =
 44.1215 -        _g1h->par_allocate_during_gc(purpose, G1ParallelGCAllocBufferSize / HeapWordSize);
 44.1216 -      if (buf == NULL) return NULL; // Let caller handle allocation failure.
 44.1217 -      // Otherwise.
 44.1218 -      alloc_buf->set_buf(buf);
 44.1219 -
 44.1220 -      obj = alloc_buf->allocate(word_sz);
 44.1221 -      assert(obj != NULL, "buffer was definitely big enough...");
 44.1222 -    } else {
 44.1223 -      obj = _g1h->par_allocate_during_gc(purpose, word_sz);
 44.1224 -    }
 44.1225 -    return obj;
 44.1226 -  }
 44.1227 -
 44.1228 -  HeapWord* allocate(GCAllocPurpose purpose, size_t word_sz) {
 44.1229 -    HeapWord* obj = alloc_buffer(purpose)->allocate(word_sz);
 44.1230 -    if (obj != NULL) return obj;
 44.1231 -    return allocate_slow(purpose, word_sz);
 44.1232 -  }
 44.1233 -
 44.1234 -  void undo_allocation(GCAllocPurpose purpose, HeapWord* obj, size_t word_sz) {
 44.1235 -    if (alloc_buffer(purpose)->contains(obj)) {
 44.1236 -      guarantee(alloc_buffer(purpose)->contains(obj + word_sz - 1),
 44.1237 -                "should contain whole object");
 44.1238 -      alloc_buffer(purpose)->undo_allocation(obj, word_sz);
 44.1239 -    } else {
 44.1240 -      CollectedHeap::fill_with_object(obj, word_sz);
 44.1241 -      add_to_undo_waste(word_sz);
 44.1242 -    }
 44.1243 -  }
 44.1244 -
 44.1245 -  void set_evac_failure_closure(OopsInHeapRegionClosure* evac_failure_cl) {
 44.1246 -    _evac_failure_cl = evac_failure_cl;
 44.1247 -  }
 44.1248 -  OopsInHeapRegionClosure* evac_failure_closure() {
 44.1249 -    return _evac_failure_cl;
 44.1250 -  }
 44.1251 -
 44.1252 -  void set_evac_closure(G1ParScanHeapEvacClosure* evac_cl) {
 44.1253 -    _evac_cl = evac_cl;
 44.1254 -  }
 44.1255 -
 44.1256 -  void set_partial_scan_closure(G1ParScanPartialArrayClosure* partial_scan_cl) {
 44.1257 -    _partial_scan_cl = partial_scan_cl;
 44.1258 -  }
 44.1259 -
 44.1260 -  int* hash_seed() { return &_hash_seed; }
 44.1261 -  int  queue_num() { return _queue_num; }
 44.1262 -
 44.1263 -  int term_attempts()   { return _term_attempts; }
 44.1264 -  void note_term_attempt()  { _term_attempts++; }
 44.1265 -
 44.1266 -#if G1_DETAILED_STATS
 44.1267 -  int pushes()          { return _pushes; }
 44.1268 -  int pops()            { return _pops; }
 44.1269 -  int steals()          { return _steals; }
 44.1270 -  int steal_attempts()  { return _steal_attempts; }
 44.1271 -  int overflow_pushes() { return _overflow_pushes; }
 44.1272 -
 44.1273 -  void note_push()          { _pushes++; }
 44.1274 -  void note_pop()           { _pops++; }
 44.1275 -  void note_steal()         { _steals++; }
 44.1276 -  void note_steal_attempt() { _steal_attempts++; }
 44.1277 -  void note_overflow_push() { _overflow_pushes++; }
 44.1278 -#endif
 44.1279 -
 44.1280 -  void start_strong_roots() {
 44.1281 -    _start_strong_roots = os::elapsedTime();
 44.1282 -  }
 44.1283 -  void end_strong_roots() {
 44.1284 -    _strong_roots_time += (os::elapsedTime() - _start_strong_roots);
 44.1285 -  }
 44.1286 -  double strong_roots_time() { return _strong_roots_time; }
 44.1287 -
 44.1288 -  void start_term_time() {
 44.1289 -    note_term_attempt();
 44.1290 -    _start_term = os::elapsedTime();
 44.1291 -  }
 44.1292 -  void end_term_time() {
 44.1293 -    _term_time += (os::elapsedTime() - _start_term);
 44.1294 -  }
 44.1295 -  double term_time() { return _term_time; }
 44.1296 -
 44.1297 -  double elapsed() {
 44.1298 -    return os::elapsedTime() - _start;
 44.1299 -  }
 44.1300 -
 44.1301 -  size_t* surviving_young_words() {
 44.1302 -    // We add on to hide entry 0 which accumulates surviving words for
 44.1303 -    // age -1 regions (i.e. non-young ones)
 44.1304 -    return _surviving_young_words;
 44.1305 -  }
 44.1306 -
 44.1307 -  void retire_alloc_buffers() {
 44.1308 -    for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {
 44.1309 -      size_t waste = _alloc_buffers[ap].words_remaining();
 44.1310 -      add_to_alloc_buffer_waste(waste);
 44.1311 -      _alloc_buffers[ap].retire(true, false);
 44.1312 -    }
 44.1313 -  }
 44.1314 -
 44.1315 -private:
 44.1316 -  void deal_with_reference(oop* ref_to_scan) {
 44.1317 -    if (has_partial_array_mask(ref_to_scan)) {
 44.1318 -      _partial_scan_cl->do_oop_nv(ref_to_scan);
 44.1319 -    } else {
 44.1320 -      // Note: we can use "raw" versions of "region_containing" because
 44.1321 -      // "obj_to_scan" is definitely in the heap, and is not in a
 44.1322 -      // humongous region.
 44.1323 -      HeapRegion* r = _g1h->heap_region_containing_raw(ref_to_scan);
 44.1324 -      _evac_cl->set_region(r);
 44.1325 -      _evac_cl->do_oop_nv(ref_to_scan);
 44.1326 -    }
 44.1327 -  }
 44.1328 -
 44.1329 -public:
 44.1330 -  void trim_queue() {
 44.1331 -    // I've replicated the loop twice, first to drain the overflow
 44.1332 -    // queue, second to drain the task queue. This is better than
 44.1333 -    // having a single loop, which checks both conditions and, inside
 44.1334 -    // it, either pops the overflow queue or the task queue, as each
 44.1335 -    // loop is tighter. Also, the decision to drain the overflow queue
 44.1336 -    // first is not arbitrary, as the overflow queue is not visible
 44.1337 -    // to the other workers, whereas the task queue is. So, we want to
 44.1338 -    // drain the "invisible" entries first, while allowing the other
 44.1339 -    // workers to potentially steal the "visible" entries.
 44.1340 -
 44.1341 -    while (refs_to_scan() > 0 || overflowed_refs_to_scan() > 0) {
 44.1342 -      while (overflowed_refs_to_scan() > 0) {
 44.1343 -        oop *ref_to_scan = NULL;
 44.1344 -        pop_from_overflow_queue(ref_to_scan);
 44.1345 -        assert(ref_to_scan != NULL, "invariant");
 44.1346 -        // We shouldn't have pushed it on the queue if it was not
 44.1347 -        // pointing into the CSet.
 44.1348 -        assert(ref_to_scan != NULL, "sanity");
 44.1349 -        assert(has_partial_array_mask(ref_to_scan) ||
 44.1350 -                                      _g1h->obj_in_cs(*ref_to_scan), "sanity");
 44.1351 -
 44.1352 -        deal_with_reference(ref_to_scan);
 44.1353 -      }
 44.1354 -
 44.1355 -      while (refs_to_scan() > 0) {
 44.1356 -        oop *ref_to_scan = NULL;
 44.1357 -        pop_from_queue(ref_to_scan);
 44.1358 -
 44.1359 -        if (ref_to_scan != NULL) {
 44.1360 -          // We shouldn't have pushed it on the queue if it was not
 44.1361 -          // pointing into the CSet.
 44.1362 -          assert(has_partial_array_mask(ref_to_scan) ||
 44.1363 -                                      _g1h->obj_in_cs(*ref_to_scan), "sanity");
 44.1364 -
 44.1365 -          deal_with_reference(ref_to_scan);
 44.1366 -        }
 44.1367 -      }
 44.1368 -    }
 44.1369 -  }
 44.1370 -};
 44.1371 +    _strong_roots_time(0), _term_time(0),
 44.1372 +    _alloc_buffer_waste(0), _undo_waste(0)
 44.1373 +{
 44.1374 +  // we allocate G1YoungSurvRateNumRegions plus one entries, since
 44.1375 +  // we "sacrifice" entry 0 to keep track of surviving bytes for
 44.1376 +  // non-young regions (where the age is -1)
 44.1377 +  // We also add a few elements at the beginning and at the end in
 44.1378 +  // an attempt to eliminate cache contention
 44.1379 +  size_t real_length = 1 + _g1h->g1_policy()->young_cset_length();
 44.1380 +  size_t array_length = PADDING_ELEM_NUM +
 44.1381 +                        real_length +
 44.1382 +                        PADDING_ELEM_NUM;
 44.1383 +  _surviving_young_words_base = NEW_C_HEAP_ARRAY(size_t, array_length);
 44.1384 +  if (_surviving_young_words_base == NULL)
 44.1385 +    vm_exit_out_of_memory(array_length * sizeof(size_t),
 44.1386 +                          "Not enough space for young surv histo.");
 44.1387 +  _surviving_young_words = _surviving_young_words_base + PADDING_ELEM_NUM;
 44.1388 +  memset(_surviving_young_words, 0, real_length * sizeof(size_t));
 44.1389 +
 44.1390 +  _overflowed_refs = new OverflowQueue(10);
 44.1391 +
 44.1392 +  _start = os::elapsedTime();
 44.1393 +}
 44.1394  
 44.1395  G1ParClosureSuper::G1ParClosureSuper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) :
 44.1396    _g1(g1), _g1_rem(_g1->g1_rem_set()), _cm(_g1->concurrent_mark()),
 44.1397    _par_scan_state(par_scan_state) { }
 44.1398  
 44.1399 -// This closure is applied to the fields of the objects that have just been copied.
 44.1400 -// Should probably be made inline and moved in g1OopClosures.inline.hpp.
 44.1401 -void G1ParScanClosure::do_oop_nv(oop* p) {
 44.1402 -  oop obj = *p;
 44.1403 -
 44.1404 -  if (obj != NULL) {
 44.1405 -    if (_g1->in_cset_fast_test(obj)) {
 44.1406 -      // We're not going to even bother checking whether the object is
 44.1407 -      // already forwarded or not, as this usually causes an immediate
 44.1408 -      // stall. We'll try to prefetch the object (for write, given that
 44.1409 -      // we might need to install the forwarding reference) and we'll
 44.1410 -      // get back to it when pop it from the queue
 44.1411 -      Prefetch::write(obj->mark_addr(), 0);
 44.1412 -      Prefetch::read(obj->mark_addr(), (HeapWordSize*2));
 44.1413 -
 44.1414 -      // slightly paranoid test; I'm trying to catch potential
 44.1415 -      // problems before we go into push_on_queue to know where the
 44.1416 -      // problem is coming from
 44.1417 -      assert(obj == *p, "the value of *p should not have changed");
 44.1418 -      _par_scan_state->push_on_queue(p);
 44.1419 -    } else {
 44.1420 -      _par_scan_state->update_rs(_from, p, _par_scan_state->queue_num());
 44.1421 -    }
 44.1422 -  }
 44.1423 -}
 44.1424 -
 44.1425 -void G1ParCopyHelper::mark_forwardee(oop* p) {
 44.1426 +template <class T> void G1ParCopyHelper::mark_forwardee(T* p) {
 44.1427    // This is called _after_ do_oop_work has been called, hence after
 44.1428    // the object has been relocated to its new location and *p points
 44.1429    // to its new location.
 44.1430  
 44.1431 -  oop thisOop = *p;
 44.1432 -  if (thisOop != NULL) {
 44.1433 -    assert((_g1->evacuation_failed()) || (!_g1->obj_in_cs(thisOop)),
 44.1434 +  T heap_oop = oopDesc::load_heap_oop(p);
 44.1435 +  if (!oopDesc::is_null(heap_oop)) {
 44.1436 +    oop obj = oopDesc::decode_heap_oop(heap_oop);
 44.1437 +    assert((_g1->evacuation_failed()) || (!_g1->obj_in_cs(obj)),
 44.1438             "shouldn't still be in the CSet if evacuation didn't fail.");
 44.1439 -    HeapWord* addr = (HeapWord*)thisOop;
 44.1440 +    HeapWord* addr = (HeapWord*)obj;
 44.1441      if (_g1->is_in_g1_reserved(addr))
 44.1442        _cm->grayRoot(oop(addr));
 44.1443    }
 44.1444 @@ -4191,7 +3694,8 @@
 44.1445  
 44.1446      if (obj->is_objArray() && arrayOop(obj)->length() >= ParGCArrayScanChunk) {
 44.1447        arrayOop(old)->set_length(0);
 44.1448 -      _par_scan_state->push_on_queue(set_partial_array_mask(old));
 44.1449 +      oop* old_p = set_partial_array_mask(old);
 44.1450 +      _par_scan_state->push_on_queue(old_p);
 44.1451      } else {
 44.1452        // No point in using the slower heap_region_containing() method,
 44.1453        // given that we know obj is in the heap.
 44.1454 @@ -4205,11 +3709,11 @@
 44.1455    return obj;
 44.1456  }
 44.1457  
 44.1458 -template<bool do_gen_barrier, G1Barrier barrier,
 44.1459 -         bool do_mark_forwardee, bool skip_cset_test>
 44.1460 -void G1ParCopyClosure<do_gen_barrier, barrier,
 44.1461 -                      do_mark_forwardee, skip_cset_test>::do_oop_work(oop* p) {
 44.1462 -  oop obj = *p;
 44.1463 +template <bool do_gen_barrier, G1Barrier barrier, bool do_mark_forwardee, bool skip_cset_test>
 44.1464 +template <class T>
 44.1465 +void G1ParCopyClosure <do_gen_barrier, barrier, do_mark_forwardee, skip_cset_test>
 44.1466 +::do_oop_work(T* p) {
 44.1467 +  oop obj = oopDesc::load_decode_heap_oop(p);
 44.1468    assert(barrier != G1BarrierRS || obj != NULL,
 44.1469           "Precondition: G1BarrierRS implies obj is nonNull");
 44.1470  
 44.1471 @@ -4226,9 +3730,10 @@
 44.1472                             "into CS.", p, (void*) obj);
 44.1473  #endif
 44.1474      if (obj->is_forwarded()) {
 44.1475 -      *p = obj->forwardee();
 44.1476 +      oopDesc::encode_store_heap_oop(p, obj->forwardee());
 44.1477      } else {
 44.1478 -      *p = copy_to_survivor_space(obj);
 44.1479 +      oop copy_oop = copy_to_survivor_space(obj);
 44.1480 +      oopDesc::encode_store_heap_oop(p, copy_oop);
 44.1481      }
 44.1482      // When scanning the RS, we only care about objs in CS.
 44.1483      if (barrier == G1BarrierRS) {
 44.1484 @@ -4247,21 +3752,9 @@
 44.1485  }
 44.1486  
 44.1487  template void G1ParCopyClosure<false, G1BarrierEvac, false, true>::do_oop_work(oop* p);
 44.1488 -
 44.1489 -template<class T> void G1ParScanPartialArrayClosure::process_array_chunk(
 44.1490 -  oop obj, int start, int end) {
 44.1491 -  // process our set of indices (include header in first chunk)
 44.1492 -  assert(start < end, "invariant");
 44.1493 -  T* const base      = (T*)objArrayOop(obj)->base();
 44.1494 -  T* const start_addr = (start == 0) ? (T*) obj : base + start;
 44.1495 -  T* const end_addr   = base + end;
 44.1496 -  MemRegion mr((HeapWord*)start_addr, (HeapWord*)end_addr);
 44.1497 -  _scanner.set_region(_g1->heap_region_containing(obj));
 44.1498 -  obj->oop_iterate(&_scanner, mr);
 44.1499 -}
 44.1500 -
 44.1501 -void G1ParScanPartialArrayClosure::do_oop_nv(oop* p) {
 44.1502 -  assert(!UseCompressedOops, "Needs to be fixed to work with compressed oops");
 44.1503 +template void G1ParCopyClosure<false, G1BarrierEvac, false, true>::do_oop_work(narrowOop* p);
 44.1504 +
 44.1505 +template <class T> void G1ParScanPartialArrayClosure::do_oop_nv(T* p) {
 44.1506    assert(has_partial_array_mask(p), "invariant");
 44.1507    oop old = clear_partial_array_mask(p);
 44.1508    assert(old->is_objArray(), "must be obj array");
 44.1509 @@ -4281,19 +3774,19 @@
 44.1510      end = start + ParGCArrayScanChunk;
 44.1511      arrayOop(old)->set_length(end);
 44.1512      // Push remainder.
 44.1513 -    _par_scan_state->push_on_queue(set_partial_array_mask(old));
 44.1514 +    oop* old_p = set_partial_array_mask(old);
 44.1515 +    assert(arrayOop(old)->length() < obj->length(), "Empty push?");
 44.1516 +    _par_scan_state->push_on_queue(old_p);
 44.1517    } else {
 44.1518      // Restore length so that the heap remains parsable in
 44.1519      // case of evacuation failure.
 44.1520      arrayOop(old)->set_length(end);
 44.1521    }
 44.1522 -
 44.1523 +  _scanner.set_region(_g1->heap_region_containing_raw(obj));
 44.1524    // process our set of indices (include header in first chunk)
 44.1525 -  process_array_chunk<oop>(obj, start, end);
 44.1526 +  obj->oop_iterate_range(&_scanner, start, end);
 44.1527  }
 44.1528  
 44.1529 -int G1ScanAndBalanceClosure::_nq = 0;
 44.1530 -
 44.1531  class G1ParEvacuateFollowersClosure : public VoidClosure {
 44.1532  protected:
 44.1533    G1CollectedHeap*              _g1h;
 44.1534 @@ -4316,21 +3809,28 @@
 44.1535    void do_void() {
 44.1536      G1ParScanThreadState* pss = par_scan_state();
 44.1537      while (true) {
 44.1538 -      oop* ref_to_scan;
 44.1539        pss->trim_queue();
 44.1540        IF_G1_DETAILED_STATS(pss->note_steal_attempt());
 44.1541 -      if (queues()->steal(pss->queue_num(),
 44.1542 -                          pss->hash_seed(),
 44.1543 -                          ref_to_scan)) {
 44.1544 +
 44.1545 +      StarTask stolen_task;
 44.1546 +      if (queues()->steal(pss->queue_num(), pss->hash_seed(), stolen_task)) {
 44.1547          IF_G1_DETAILED_STATS(pss->note_steal());
 44.1548  
 44.1549          // slightly paranoid tests; I'm trying to catch potential
 44.1550          // problems before we go into push_on_queue to know where the
 44.1551          // problem is coming from
 44.1552 -        assert(ref_to_scan != NULL, "invariant");
 44.1553 -        assert(has_partial_array_mask(ref_to_scan) ||
 44.1554 -                                   _g1h->obj_in_cs(*ref_to_scan), "invariant");
 44.1555 -        pss->push_on_queue(ref_to_scan);
 44.1556 +        assert((oop*)stolen_task != NULL, "Error");
 44.1557 +        if (stolen_task.is_narrow()) {
 44.1558 +          assert(UseCompressedOops, "Error");
 44.1559 +          narrowOop* p = (narrowOop*) stolen_task;
 44.1560 +          assert(has_partial_array_mask(p) ||
 44.1561 +                 _g1h->obj_in_cs(oopDesc::load_decode_heap_oop(p)), "Error");
 44.1562 +          pss->push_on_queue(p);
 44.1563 +        } else {
 44.1564 +          oop* p = (oop*) stolen_task;
 44.1565 +          assert(has_partial_array_mask(p) || _g1h->obj_in_cs(*p), "Error");
 44.1566 +          pss->push_on_queue(p);
 44.1567 +        }
 44.1568          continue;
 44.1569        }
 44.1570        pss->start_term_time();
 44.1571 @@ -4347,6 +3847,7 @@
 44.1572    G1CollectedHeap*       _g1h;
 44.1573    RefToScanQueueSet      *_queues;
 44.1574    ParallelTaskTerminator _terminator;
 44.1575 +  int _n_workers;
 44.1576  
 44.1577    Mutex _stats_lock;
 44.1578    Mutex* stats_lock() { return &_stats_lock; }
 44.1579 @@ -4362,7 +3863,8 @@
 44.1580        _g1h(g1h),
 44.1581        _queues(task_queues),
 44.1582        _terminator(workers, _queues),
 44.1583 -      _stats_lock(Mutex::leaf, "parallel G1 stats lock", true)
 44.1584 +      _stats_lock(Mutex::leaf, "parallel G1 stats lock", true),
 44.1585 +      _n_workers(workers)
 44.1586    {}
 44.1587  
 44.1588    RefToScanQueueSet* queues() { return _queues; }
 44.1589 @@ -4372,6 +3874,7 @@
 44.1590    }
 44.1591  
 44.1592    void work(int i) {
 44.1593 +    if (i >= _n_workers) return;  // no work needed this round
 44.1594      ResourceMark rm;
 44.1595      HandleMark   hm;
 44.1596  
 44.1597 @@ -4469,23 +3972,6 @@
 44.1598  
 44.1599  // *** Common G1 Evacuation Stuff
 44.1600  
 44.1601 -class G1CountClosure: public OopsInHeapRegionClosure {
 44.1602 -public:
 44.1603 -  int n;
 44.1604 -  G1CountClosure() : n(0) {}
 44.1605 -  void do_oop(narrowOop* p) {
 44.1606 -    guarantee(false, "NYI");
 44.1607 -  }
 44.1608 -  void do_oop(oop* p) {
 44.1609 -    oop obj = *p;
 44.1610 -    assert(obj != NULL && G1CollectedHeap::heap()->obj_in_cs(obj),
 44.1611 -           "Rem set closure called on non-rem-set pointer.");
 44.1612 -    n++;
 44.1613 -  }
 44.1614 -};
 44.1615 -
 44.1616 -static G1CountClosure count_closure;
 44.1617 -
 44.1618  void
 44.1619  G1CollectedHeap::
 44.1620  g1_process_strong_roots(bool collecting_perm_gen,
 44.1621 @@ -5357,7 +4843,7 @@
 44.1622    assert(_free_region_list == NULL, "Postcondition of loop.");
 44.1623    if (_free_region_list_size != 0) {
 44.1624      gclog_or_tty->print_cr("Size is %d.", _free_region_list_size);
 44.1625 -    print();
 44.1626 +    print_on(gclog_or_tty, true /* extended */);
 44.1627    }
 44.1628    assert(_free_region_list_size == 0, "Postconditions of loop.");
 44.1629  }
 44.1630 @@ -5535,8 +5021,3 @@
 44.1631  void G1CollectedHeap::g1_unimplemented() {
 44.1632    // Unimplemented();
 44.1633  }
 44.1634 -
 44.1635 -
 44.1636 -// Local Variables: ***
 44.1637 -// c-indentation-style: gnu ***
 44.1638 -// End: ***

    45.1 --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp	Mon Jul 27 09:06:22 2009 -0700
    45.2 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp	Mon Jul 27 17:23:52 2009 -0400
    45.3 @@ -56,8 +56,8 @@
    45.4  #  define IF_G1_DETAILED_STATS(code)
    45.5  #endif
    45.6  
    45.7 -typedef GenericTaskQueue<oop*>    RefToScanQueue;
    45.8 -typedef GenericTaskQueueSet<oop*> RefToScanQueueSet;
    45.9 +typedef GenericTaskQueue<StarTask>    RefToScanQueue;
   45.10 +typedef GenericTaskQueueSet<StarTask> RefToScanQueueSet;
   45.11  
   45.12  typedef int RegionIdx_t;   // needs to hold [ 0..max_regions() )
   45.13  typedef int CardIdx_t;     // needs to hold [ 0..CardsPerRegion )
   45.14 @@ -700,6 +700,9 @@
   45.15    size_t g1_reserved_obj_bytes() { return _g1_reserved.byte_size(); }
   45.16    virtual size_t capacity() const;
   45.17    virtual size_t used() const;
   45.18 +  // This should be called when we're not holding the heap lock. The
   45.19 +  // result might be a bit inaccurate.
   45.20 +  size_t used_unlocked() const;
   45.21    size_t recalculate_used() const;
   45.22  #ifndef PRODUCT
   45.23    size_t recalculate_used_regions() const;
   45.24 @@ -1061,8 +1064,14 @@
   45.25  
   45.26    // Override; it uses the "prev" marking information
   45.27    virtual void verify(bool allow_dirty, bool silent);
   45.28 +  // Default behavior by calling print(tty);
   45.29    virtual void print() const;
   45.30 +  // This calls print_on(st, PrintHeapAtGCExtended).
   45.31    virtual void print_on(outputStream* st) const;
   45.32 +  // If extended is true, it will print out information for all
   45.33 +  // regions in the heap by calling print_on_extended(st).
   45.34 +  virtual void print_on(outputStream* st, bool extended) const;
   45.35 +  virtual void print_on_extended(outputStream* st) const;
   45.36  
   45.37    virtual void print_gc_threads_on(outputStream* st) const;
   45.38    virtual void gc_threads_do(ThreadClosure* tc) const;
   45.39 @@ -1265,6 +1274,552 @@
   45.40  
   45.41  };
   45.42  
   45.43 -// Local Variables: ***
   45.44 -// c-indentation-style: gnu ***
   45.45 -// End: ***
   45.46 +#define use_local_bitmaps         1
   45.47 +#define verify_local_bitmaps      0
   45.48 +#define oop_buffer_length       256
   45.49 +
   45.50 +#ifndef PRODUCT
   45.51 +class GCLabBitMap;
   45.52 +class GCLabBitMapClosure: public BitMapClosure {
   45.53 +private:
   45.54 +  ConcurrentMark* _cm;
   45.55 +  GCLabBitMap*    _bitmap;
   45.56 +
   45.57 +public:
   45.58 +  GCLabBitMapClosure(ConcurrentMark* cm,
   45.59 +                     GCLabBitMap* bitmap) {
   45.60 +    _cm     = cm;
   45.61 +    _bitmap = bitmap;
   45.62 +  }
   45.63 +
   45.64 +  virtual bool do_bit(size_t offset);
   45.65 +};
   45.66 +#endif // !PRODUCT
   45.67 +
   45.68 +class GCLabBitMap: public BitMap {
   45.69 +private:
   45.70 +  ConcurrentMark* _cm;
   45.71 +
   45.72 +  int       _shifter;
   45.73 +  size_t    _bitmap_word_covers_words;
   45.74 +
   45.75 +  // beginning of the heap
   45.76 +  HeapWord* _heap_start;
   45.77 +
   45.78 +  // this is the actual start of the GCLab
   45.79 +  HeapWord* _real_start_word;
   45.80 +
   45.81 +  // this is the actual end of the GCLab
   45.82 +  HeapWord* _real_end_word;
   45.83 +
   45.84 +  // this is the first word, possibly located before the actual start
   45.85 +  // of the GCLab, that corresponds to the first bit of the bitmap
   45.86 +  HeapWord* _start_word;
   45.87 +
   45.88 +  // size of a GCLab in words
   45.89 +  size_t _gclab_word_size;
   45.90 +
   45.91 +  static int shifter() {
   45.92 +    return MinObjAlignment - 1;
   45.93 +  }
   45.94 +
   45.95 +  // how many heap words does a single bitmap word corresponds to?
   45.96 +  static size_t bitmap_word_covers_words() {
   45.97 +    return BitsPerWord << shifter();
   45.98 +  }
   45.99 +
  45.100 +  static size_t gclab_word_size() {
  45.101 +    return G1ParallelGCAllocBufferSize / HeapWordSize;
  45.102 +  }
  45.103 +
  45.104 +  static size_t bitmap_size_in_bits() {
  45.105 +    size_t bits_in_bitmap = gclab_word_size() >> shifter();
  45.106 +    // We are going to ensure that the beginning of a word in this
  45.107 +    // bitmap also corresponds to the beginning of a word in the
  45.108 +    // global marking bitmap. To handle the case where a GCLab
  45.109 +    // starts from the middle of the bitmap, we need to add enough
  45.110 +    // space (i.e. up to a bitmap word) to ensure that we have
  45.111 +    // enough bits in the bitmap.
  45.112 +    return bits_in_bitmap + BitsPerWord - 1;
  45.113 +  }
  45.114 +public:
  45.115 +  GCLabBitMap(HeapWord* heap_start)
  45.116 +    : BitMap(bitmap_size_in_bits()),
  45.117 +      _cm(G1CollectedHeap::heap()->concurrent_mark()),
  45.118 +      _shifter(shifter()),
  45.119 +      _bitmap_word_covers_words(bitmap_word_covers_words()),
  45.120 +      _heap_start(heap_start),
  45.121 +      _gclab_word_size(gclab_word_size()),
  45.122 +      _real_start_word(NULL),
  45.123 +      _real_end_word(NULL),
  45.124 +      _start_word(NULL)
  45.125 +  {
  45.126 +    guarantee( size_in_words() >= bitmap_size_in_words(),
  45.127 +               "just making sure");
  45.128 +  }
  45.129 +
  45.130 +  inline unsigned heapWordToOffset(HeapWord* addr) {
  45.131 +    unsigned offset = (unsigned) pointer_delta(addr, _start_word) >> _shifter;
  45.132 +    assert(offset < size(), "offset should be within bounds");
  45.133 +    return offset;
  45.134 +  }
  45.135 +
  45.136 +  inline HeapWord* offsetToHeapWord(size_t offset) {
  45.137 +    HeapWord* addr =  _start_word + (offset << _shifter);
  45.138 +    assert(_real_start_word <= addr && addr < _real_end_word, "invariant");
  45.139 +    return addr;
  45.140 +  }
  45.141 +
  45.142 +  bool fields_well_formed() {
  45.143 +    bool ret1 = (_real_start_word == NULL) &&
  45.144 +                (_real_end_word == NULL) &&
  45.145 +                (_start_word == NULL);
  45.146 +    if (ret1)
  45.147 +      return true;
  45.148 +
  45.149 +    bool ret2 = _real_start_word >= _start_word &&
  45.150 +      _start_word < _real_end_word &&
  45.151 +      (_real_start_word + _gclab_word_size) == _real_end_word &&
  45.152 +      (_start_word + _gclab_word_size + _bitmap_word_covers_words)
  45.153 +                                                              > _real_end_word;
  45.154 +    return ret2;
  45.155 +  }
  45.156 +
  45.157 +  inline bool mark(HeapWord* addr) {
  45.158 +    guarantee(use_local_bitmaps, "invariant");
  45.159 +    assert(fields_well_formed(), "invariant");
  45.160 +
  45.161 +    if (addr >= _real_start_word && addr < _real_end_word) {
  45.162 +      assert(!isMarked(addr), "should not have already been marked");
  45.163 +
  45.164 +      // first mark it on the bitmap
  45.165 +      at_put(heapWordToOffset(addr), true);
  45.166 +
  45.167 +      return true;
  45.168 +    } else {
  45.169 +      return false;
  45.170 +    }
  45.171 +  }
  45.172 +
  45.173 +  inline bool isMarked(HeapWord* addr) {
  45.174 +    guarantee(use_local_bitmaps, "invariant");
  45.175 +    assert(fields_well_formed(), "invariant");
  45.176 +
  45.177 +    return at(heapWordToOffset(addr));
  45.178 +  }
  45.179 +
  45.180 +  void set_buffer(HeapWord* start) {
  45.181 +    guarantee(use_local_bitmaps, "invariant");
  45.182 +    clear();
  45.183 +
  45.184 +    assert(start != NULL, "invariant");
  45.185 +    _real_start_word = start;
  45.186 +    _real_end_word   = start + _gclab_word_size;
  45.187 +
  45.188 +    size_t diff =
  45.189 +      pointer_delta(start, _heap_start) % _bitmap_word_covers_words;
  45.190 +    _start_word = start - diff;
  45.191 +
  45.192 +    assert(fields_well_formed(), "invariant");
  45.193 +  }
  45.194 +
  45.195 +#ifndef PRODUCT
  45.196 +  void verify() {
  45.197 +    // verify that the marks have been propagated
  45.198 +    GCLabBitMapClosure cl(_cm, this);
  45.199 +    iterate(&cl);
  45.200 +  }
  45.201 +#endif // PRODUCT
  45.202 +
  45.203 +  void retire() {
  45.204 +    guarantee(use_local_bitmaps, "invariant");
  45.205 +    assert(fields_well_formed(), "invariant");
  45.206 +
  45.207 +    if (_start_word != NULL) {
  45.208 +      CMBitMap*       mark_bitmap = _cm->nextMarkBitMap();
  45.209 +
  45.210 +      // this means that the bitmap was set up for the GCLab
  45.211 +      assert(_real_start_word != NULL && _real_end_word != NULL, "invariant");
  45.212 +
  45.213 +      mark_bitmap->mostly_disjoint_range_union(this,
  45.214 +                                0, // always start from the start of the bitmap
  45.215 +                                _start_word,
  45.216 +                                size_in_words());
  45.217 +      _cm->grayRegionIfNecessary(MemRegion(_real_start_word, _real_end_word));
  45.218 +
  45.219 +#ifndef PRODUCT
  45.220 +      if (use_local_bitmaps && verify_local_bitmaps)
  45.221 +        verify();
  45.222 +#endif // PRODUCT
  45.223 +    } else {
  45.224 +      assert(_real_start_word == NULL && _real_end_word == NULL, "invariant");
  45.225 +    }
  45.226 +  }
  45.227 +
  45.228 +  static size_t bitmap_size_in_words() {
  45.229 +    return (bitmap_size_in_bits() + BitsPerWord - 1) / BitsPerWord;
  45.230 +  }
  45.231 +};
  45.232 +
  45.233 +class G1ParGCAllocBuffer: public ParGCAllocBuffer {
  45.234 +private:
  45.235 +  bool        _retired;
  45.236 +  bool        _during_marking;
  45.237 +  GCLabBitMap _bitmap;
  45.238 +
  45.239 +public:
  45.240 +  G1ParGCAllocBuffer() :
  45.241 +    ParGCAllocBuffer(G1ParallelGCAllocBufferSize / HeapWordSize),
  45.242 +    _during_marking(G1CollectedHeap::heap()->mark_in_progress()),
  45.243 +    _bitmap(G1CollectedHeap::heap()->reserved_region().start()),
  45.244 +    _retired(false)
  45.245 +  { }
  45.246 +
  45.247 +  inline bool mark(HeapWord* addr) {
  45.248 +    guarantee(use_local_bitmaps, "invariant");
  45.249 +    assert(_during_marking, "invariant");
  45.250 +    return _bitmap.mark(addr);
  45.251 +  }
  45.252 +
  45.253 +  inline void set_buf(HeapWord* buf) {
  45.254 +    if (use_local_bitmaps && _during_marking)
  45.255 +      _bitmap.set_buffer(buf);
  45.256 +    ParGCAllocBuffer::set_buf(buf);
  45.257 +    _retired = false;
  45.258 +  }
  45.259 +
  45.260 +  inline void retire(bool end_of_gc, bool retain) {
  45.261 +    if (_retired)
  45.262 +      return;
  45.263 +    if (use_local_bitmaps && _during_marking) {
  45.264 +      _bitmap.retire();
  45.265 +    }
  45.266 +    ParGCAllocBuffer::retire(end_of_gc, retain);
  45.267 +    _retired = true;
  45.268 +  }
  45.269 +};
  45.270 +
  45.271 +class G1ParScanThreadState : public StackObj {
  45.272 +protected:
  45.273 +  G1CollectedHeap* _g1h;
  45.274 +  RefToScanQueue*  _refs;
  45.275 +  DirtyCardQueue   _dcq;
  45.276 +  CardTableModRefBS* _ct_bs;
  45.277 +  G1RemSet* _g1_rem;
  45.278 +
  45.279 +  typedef GrowableArray<StarTask> OverflowQueue;
  45.280 +  OverflowQueue* _overflowed_refs;
  45.281 +
  45.282 +  G1ParGCAllocBuffer _alloc_buffers[GCAllocPurposeCount];
  45.283 +  ageTable           _age_table;
  45.284 +
  45.285 +  size_t           _alloc_buffer_waste;
  45.286 +  size_t           _undo_waste;
  45.287 +
  45.288 +  OopsInHeapRegionClosure*      _evac_failure_cl;
  45.289 +  G1ParScanHeapEvacClosure*     _evac_cl;
  45.290 +  G1ParScanPartialArrayClosure* _partial_scan_cl;
  45.291 +
  45.292 +  int _hash_seed;
  45.293 +  int _queue_num;
  45.294 +
  45.295 +  int _term_attempts;
  45.296 +#if G1_DETAILED_STATS
  45.297 +  int _pushes, _pops, _steals, _steal_attempts;
  45.298 +  int _overflow_pushes;
  45.299 +#endif
  45.300 +
  45.301 +  double _start;
  45.302 +  double _start_strong_roots;
  45.303 +  double _strong_roots_time;
  45.304 +  double _start_term;
  45.305 +  double _term_time;
  45.306 +
  45.307 +  // Map from young-age-index (0 == not young, 1 is youngest) to
  45.308 +  // surviving words. base is what we get back from the malloc call
  45.309 +  size_t* _surviving_young_words_base;
  45.310 +  // this points into the array, as we use the first few entries for padding
  45.311 +  size_t* _surviving_young_words;
  45.312 +
  45.313 +#define PADDING_ELEM_NUM (64 / sizeof(size_t))
  45.314 +
  45.315 +  void   add_to_alloc_buffer_waste(size_t waste) { _alloc_buffer_waste += waste; }
  45.316 +
  45.317 +  void   add_to_undo_waste(size_t waste)         { _undo_waste += waste; }
  45.318 +
  45.319 +  DirtyCardQueue& dirty_card_queue()             { return _dcq;  }
  45.320 +  CardTableModRefBS* ctbs()                      { return _ct_bs; }
  45.321 +
  45.322 +  template <class T> void immediate_rs_update(HeapRegion* from, T* p, int tid) {
  45.323 +    if (!from->is_survivor()) {
  45.324 +      _g1_rem->par_write_ref(from, p, tid);
  45.325 +    }
  45.326 +  }
  45.327 +
  45.328 +  template <class T> void deferred_rs_update(HeapRegion* from, T* p, int tid) {
  45.329 +    // If the new value of the field points to the same region or
  45.330 +    // is the to-space, we don't need to include it in the Rset updates.
  45.331 +    if (!from->is_in_reserved(oopDesc::load_decode_heap_oop(p)) && !from->is_survivor()) {
  45.332 +      size_t card_index = ctbs()->index_for(p);
  45.333 +      // If the card hasn't been added to the buffer, do it.
  45.334 +      if (ctbs()->mark_card_deferred(card_index)) {
  45.335 +        dirty_card_queue().enqueue((jbyte*)ctbs()->byte_for_index(card_index));
  45.336 +      }
  45.337 +    }
  45.338 +  }
  45.339 +
  45.340 +public:
  45.341 +  G1ParScanThreadState(G1CollectedHeap* g1h, int queue_num);
  45.342 +
  45.343 +  ~G1ParScanThreadState() {
  45.344 +    FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_base);
  45.345 +  }
  45.346 +
  45.347 +  RefToScanQueue*   refs()            { return _refs;             }
  45.348 +  OverflowQueue*    overflowed_refs() { return _overflowed_refs;  }
  45.349 +  ageTable*         age_table()       { return &_age_table;       }
  45.350 +
  45.351 +  G1ParGCAllocBuffer* alloc_buffer(GCAllocPurpose purpose) {
  45.352 +    return &_alloc_buffers[purpose];
  45.353 +  }
  45.354 +
  45.355 +  size_t alloc_buffer_waste()                    { return _alloc_buffer_waste; }
  45.356 +  size_t undo_waste()                            { return _undo_waste; }
  45.357 +
  45.358 +  template <class T> void push_on_queue(T* ref) {
  45.359 +    assert(ref != NULL, "invariant");
  45.360 +    assert(has_partial_array_mask(ref) ||
  45.361 +           _g1h->obj_in_cs(oopDesc::load_decode_heap_oop(ref)), "invariant");
  45.362 +#ifdef ASSERT
  45.363 +    if (has_partial_array_mask(ref)) {
  45.364 +      oop p = clear_partial_array_mask(ref);
  45.365 +      // Verify that we point into the CS
  45.366 +      assert(_g1h->obj_in_cs(p), "Should be in CS");
  45.367 +    }
  45.368 +#endif
  45.369 +    if (!refs()->push(ref)) {
  45.370 +      overflowed_refs()->push(ref);
  45.371 +      IF_G1_DETAILED_STATS(note_overflow_push());
  45.372 +    } else {
  45.373 +      IF_G1_DETAILED_STATS(note_push());
  45.374 +    }
  45.375 +  }
  45.376 +
  45.377 +  void pop_from_queue(StarTask& ref) {
  45.378 +    if (refs()->pop_local(ref)) {
  45.379 +      assert((oop*)ref != NULL, "pop_local() returned true");
  45.380 +      assert(UseCompressedOops || !ref.is_narrow(), "Error");
  45.381 +      assert(has_partial_array_mask((oop*)ref) ||
  45.382 +             _g1h->obj_in_cs(ref.is_narrow() ? oopDesc::load_decode_heap_oop((narrowOop*)ref)
  45.383 +                                             : oopDesc::load_decode_heap_oop((oop*)ref)),
  45.384 +             "invariant");
  45.385 +      IF_G1_DETAILED_STATS(note_pop());
  45.386 +    } else {
  45.387 +      StarTask null_task;
  45.388 +      ref = null_task;
  45.389 +    }
  45.390 +  }
  45.391 +
  45.392 +  void pop_from_overflow_queue(StarTask& ref) {
  45.393 +    StarTask new_ref = overflowed_refs()->pop();
  45.394 +    assert((oop*)new_ref != NULL, "pop() from a local non-empty stack");
  45.395 +    assert(UseCompressedOops || !new_ref.is_narrow(), "Error");
  45.396 +    assert(has_partial_array_mask((oop*)new_ref) ||
  45.397 +           _g1h->obj_in_cs(new_ref.is_narrow() ? oopDesc::load_decode_heap_oop((narrowOop*)new_ref)
  45.398 +                                               : oopDesc::load_decode_heap_oop((oop*)new_ref)),
  45.399 +             "invariant");
  45.400 +    ref = new_ref;
  45.401 +  }
  45.402 +
  45.403 +  int refs_to_scan()                             { return refs()->size();                 }
  45.404 +  int overflowed_refs_to_scan()                  { return overflowed_refs()->length();    }
  45.405 +
  45.406 +  template <class T> void update_rs(HeapRegion* from, T* p, int tid) {
  45.407 +    if (G1DeferredRSUpdate) {
  45.408 +      deferred_rs_update(from, p, tid);
  45.409 +    } else {
  45.410 +      immediate_rs_update(from, p, tid);
  45.411 +    }
  45.412 +  }
  45.413 +
  45.414 +  HeapWord* allocate_slow(GCAllocPurpose purpose, size_t word_sz) {
  45.415 +
  45.416 +    HeapWord* obj = NULL;
  45.417 +    if (word_sz * 100 <
  45.418 +        (size_t)(G1ParallelGCAllocBufferSize / HeapWordSize) *
  45.419 +                                                  ParallelGCBufferWastePct) {
  45.420 +      G1ParGCAllocBuffer* alloc_buf = alloc_buffer(purpose);
  45.421 +      add_to_alloc_buffer_waste(alloc_buf->words_remaining());
  45.422 +      alloc_buf->retire(false, false);
  45.423 +
  45.424 +      HeapWord* buf =
  45.425 +        _g1h->par_allocate_during_gc(purpose, G1ParallelGCAllocBufferSize / HeapWordSize);
  45.426 +      if (buf == NULL) return NULL; // Let caller handle allocation failure.
  45.427 +      // Otherwise.
  45.428 +      alloc_buf->set_buf(buf);
  45.429 +
  45.430 +      obj = alloc_buf->allocate(word_sz);
  45.431 +      assert(obj != NULL, "buffer was definitely big enough...");
  45.432 +    } else {
  45.433 +      obj = _g1h->par_allocate_during_gc(purpose, word_sz);
  45.434 +    }
  45.435 +    return obj;
  45.436 +  }
  45.437 +
  45.438 +  HeapWord* allocate(GCAllocPurpose purpose, size_t word_sz) {
  45.439 +    HeapWord* obj = alloc_buffer(purpose)->allocate(word_sz);
  45.440 +    if (obj != NULL) return obj;
  45.441 +    return allocate_slow(purpose, word_sz);
  45.442 +  }
  45.443 +
  45.444 +  void undo_allocation(GCAllocPurpose purpose, HeapWord* obj, size_t word_sz) {
  45.445 +    if (alloc_buffer(purpose)->contains(obj)) {
  45.446 +      assert(alloc_buffer(purpose)->contains(obj + word_sz - 1),
  45.447 +             "should contain whole object");
  45.448 +      alloc_buffer(purpose)->undo_allocation(obj, word_sz);
  45.449 +    } else {
  45.450 +      CollectedHeap::fill_with_object(obj, word_sz);
  45.451 +      add_to_undo_waste(word_sz);
  45.452 +    }
  45.453 +  }
  45.454 +
  45.455 +  void set_evac_failure_closure(OopsInHeapRegionClosure* evac_failure_cl) {
  45.456 +    _evac_failure_cl = evac_failure_cl;
  45.457 +  }
  45.458 +  OopsInHeapRegionClosure* evac_failure_closure() {
  45.459 +    return _evac_failure_cl;
  45.460 +  }
  45.461 +
  45.462 +  void set_evac_closure(G1ParScanHeapEvacClosure* evac_cl) {
  45.463 +    _evac_cl = evac_cl;
  45.464 +  }
  45.465 +
  45.466 +  void set_partial_scan_closure(G1ParScanPartialArrayClosure* partial_scan_cl) {
  45.467 +    _partial_scan_cl = partial_scan_cl;
  45.468 +  }
  45.469 +
  45.470 +  int* hash_seed() { return &_hash_seed; }
  45.471 +  int  queue_num() { return _queue_num; }
  45.472 +
  45.473 +  int term_attempts()   { return _term_attempts; }
  45.474 +  void note_term_attempt()  { _term_attempts++; }
  45.475 +
  45.476 +#if G1_DETAILED_STATS
  45.477 +  int pushes()          { return _pushes; }
  45.478 +  int pops()            { return _pops; }
  45.479 +  int steals()          { return _steals; }
  45.480 +  int steal_attempts()  { return _steal_attempts; }
  45.481 +  int overflow_pushes() { return _overflow_pushes; }
  45.482 +
  45.483 +  void note_push()          { _pushes++; }
  45.484 +  void note_pop()           { _pops++; }
  45.485 +  void note_steal()         { _steals++; }
  45.486 +  void note_steal_attempt() { _steal_attempts++; }
  45.487 +  void note_overflow_push() { _overflow_pushes++; }
  45.488 +#endif
  45.489 +
  45.490 +  void start_strong_roots() {
  45.491 +    _start_strong_roots = os::elapsedTime();
  45.492 +  }
  45.493 +  void end_strong_roots() {
  45.494 +    _strong_roots_time += (os::elapsedTime() - _start_strong_roots);
  45.495 +  }
  45.496 +  double strong_roots_time() { return _strong_roots_time; }
  45.497 +
  45.498 +  void start_term_time() {
  45.499 +    note_term_attempt();
  45.500 +    _start_term = os::elapsedTime();
  45.501 +  }
  45.502 +  void end_term_time() {
  45.503 +    _term_time += (os::elapsedTime() - _start_term);
  45.504 +  }
  45.505 +  double term_time() { return _term_time; }
  45.506 +
  45.507 +  double elapsed() {
  45.508 +    return os::elapsedTime() - _start;
  45.509 +  }
  45.510 +
  45.511 +  size_t* surviving_young_words() {
  45.512 +    // We add on to hide entry 0 which accumulates surviving words for
  45.513 +    // age -1 regions (i.e. non-young ones)
  45.514 +    return _surviving_young_words;
  45.515 +  }
  45.516 +
  45.517 +  void retire_alloc_buffers() {
  45.518 +    for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {
  45.519 +      size_t waste = _alloc_buffers[ap].words_remaining();
  45.520 +      add_to_alloc_buffer_waste(waste);
  45.521 +      _alloc_buffers[ap].retire(true, false);
  45.522 +    }
  45.523 +  }
  45.524 +
  45.525 +private:
  45.526 +  template <class T> void deal_with_reference(T* ref_to_scan) {
  45.527 +    if (has_partial_array_mask(ref_to_scan)) {
  45.528 +      _partial_scan_cl->do_oop_nv(ref_to_scan);
  45.529 +    } else {
  45.530 +      // Note: we can use "raw" versions of "region_containing" because
  45.531 +      // "obj_to_scan" is definitely in the heap, and is not in a
  45.532 +      // humongous region.
  45.533 +      HeapRegion* r = _g1h->heap_region_containing_raw(ref_to_scan);
  45.534 +      _evac_cl->set_region(r);
  45.535 +      _evac_cl->do_oop_nv(ref_to_scan);
  45.536 +    }
  45.537 +  }
  45.538 +
  45.539 +public:
  45.540 +  void trim_queue() {
  45.541 +    // I've replicated the loop twice, first to drain the overflow
  45.542 +    // queue, second to drain the task queue. This is better than
  45.543 +    // having a single loop, which checks both conditions and, inside
  45.544 +    // it, either pops the overflow queue or the task queue, as each
  45.545 +    // loop is tighter. Also, the decision to drain the overflow queue
  45.546 +    // first is not arbitrary, as the overflow queue is not visible
  45.547 +    // to the other workers, whereas the task queue is. So, we want to
  45.548 +    // drain the "invisible" entries first, while allowing the other
  45.549 +    // workers to potentially steal the "visible" entries.
  45.550 +
  45.551 +    while (refs_to_scan() > 0 || overflowed_refs_to_scan() > 0) {
  45.552 +      while (overflowed_refs_to_scan() > 0) {
  45.553 +        StarTask ref_to_scan;
  45.554 +        assert((oop*)ref_to_scan == NULL, "Constructed above");
  45.555 +        pop_from_overflow_queue(ref_to_scan);
  45.556 +        // We shouldn't have pushed it on the queue if it was not
  45.557 +        // pointing into the CSet.
  45.558 +        assert((oop*)ref_to_scan != NULL, "Follows from inner loop invariant");
  45.559 +        if (ref_to_scan.is_narrow()) {
  45.560 +          assert(UseCompressedOops, "Error");
  45.561 +          narrowOop* p = (narrowOop*)ref_to_scan;
  45.562 +          assert(!has_partial_array_mask(p) &&
  45.563 +                 _g1h->obj_in_cs(oopDesc::load_decode_heap_oop(p)), "sanity");
  45.564 +          deal_with_reference(p);
  45.565 +        } else {
  45.566 +          oop* p = (oop*)ref_to_scan;
  45.567 +          assert((has_partial_array_mask(p) && _g1h->obj_in_cs(clear_partial_array_mask(p))) ||
  45.568 +                 _g1h->obj_in_cs(oopDesc::load_decode_heap_oop(p)), "sanity");
  45.569 +          deal_with_reference(p);
  45.570 +        }
  45.571 +      }
  45.572 +
  45.573 +      while (refs_to_scan() > 0) {
  45.574 +        StarTask ref_to_scan;
  45.575 +        assert((oop*)ref_to_scan == NULL, "Constructed above");
  45.576 +        pop_from_queue(ref_to_scan);
  45.577 +        if ((oop*)ref_to_scan != NULL) {
  45.578 +          if (ref_to_scan.is_narrow()) {
  45.579 +            assert(UseCompressedOops, "Error");
  45.580 +            narrowOop* p = (narrowOop*)ref_to_scan;
  45.581 +            assert(!has_partial_array_mask(p) &&
  45.582 +                   _g1h->obj_in_cs(oopDesc::load_decode_heap_oop(p)), "sanity");
  45.583 +            deal_with_reference(p);
  45.584 +          } else {
  45.585 +            oop* p = (oop*)ref_to_scan;
  45.586 +            assert((has_partial_array_mask(p) && _g1h->obj_in_cs(clear_partial_array_mask(p))) ||
  45.587 +                  _g1h->obj_in_cs(oopDesc::load_decode_heap_oop(p)), "sanity");
  45.588 +            deal_with_reference(p);
  45.589 +          }
  45.590 +        }
  45.591 +      }
  45.592 +    }
  45.593 +  }
  45.594 +};

    46.1 --- a/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp	Mon Jul 27 09:06:22 2009 -0700
    46.2 +++ b/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp	Mon Jul 27 17:23:52 2009 -0400
    46.3 @@ -293,10 +293,6 @@
    46.4    if (G1SteadyStateUsed < 50) {
    46.5      vm_exit_during_initialization("G1SteadyStateUsed must be at least 50%.");
    46.6    }
    46.7 -  if (UseConcMarkSweepGC) {
    46.8 -    vm_exit_during_initialization("-XX:+UseG1GC is incompatible with "
    46.9 -                                  "-XX:+UseConcMarkSweepGC.");
   46.10 -  }
   46.11  
   46.12    initialize_gc_policy_counters();
   46.13

    47.1 --- a/src/share/vm/gc_implementation/g1/g1CollectorPolicy.hpp	Mon Jul 27 09:06:22 2009 -0700
    47.2 +++ b/src/share/vm/gc_implementation/g1/g1CollectorPolicy.hpp	Mon Jul 27 17:23:52 2009 -0400
    47.3 @@ -1097,6 +1097,10 @@
    47.4      _recorded_survivor_tail    = tail;
    47.5    }
    47.6  
    47.7 +  size_t recorded_survivor_regions() {
    47.8 +    return _recorded_survivor_regions;
    47.9 +  }
   47.10 +
   47.11    void record_thread_age_table(ageTable* age_table)
   47.12    {
   47.13      _survivors_age_table.merge_par(age_table);

    48.1 --- a/src/share/vm/gc_implementation/g1/g1OopClosures.hpp	Mon Jul 27 09:06:22 2009 -0700
    48.2 +++ b/src/share/vm/gc_implementation/g1/g1OopClosures.hpp	Mon Jul 27 17:23:52 2009 -0400
    48.3 @@ -42,18 +42,6 @@
    48.4    virtual void set_region(HeapRegion* from) { _from = from; }
    48.5  };
    48.6  
    48.7 -
    48.8 -class G1ScanAndBalanceClosure : public OopClosure {
    48.9 -  G1CollectedHeap* _g1;
   48.10 -  static int _nq;
   48.11 -public:
   48.12 -  G1ScanAndBalanceClosure(G1CollectedHeap* g1) : _g1(g1) { }
   48.13 -  inline  void do_oop_nv(oop* p);
   48.14 -  inline  void do_oop_nv(narrowOop* p) { guarantee(false, "NYI"); }
   48.15 -  virtual void do_oop(oop* p);
   48.16 -  virtual void do_oop(narrowOop* p)    { guarantee(false, "NYI"); }
   48.17 -};
   48.18 -
   48.19  class G1ParClosureSuper : public OopsInHeapRegionClosure {
   48.20  protected:
   48.21    G1CollectedHeap* _g1;
   48.22 @@ -69,34 +57,32 @@
   48.23  public:
   48.24    G1ParScanClosure(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) :
   48.25      G1ParClosureSuper(g1, par_scan_state) { }
   48.26 -  void do_oop_nv(oop* p);   // should be made inline
   48.27 -  inline  void do_oop_nv(narrowOop* p) { guarantee(false, "NYI"); }
   48.28 +  template <class T> void do_oop_nv(T* p);
   48.29    virtual void do_oop(oop* p)          { do_oop_nv(p); }
   48.30    virtual void do_oop(narrowOop* p)    { do_oop_nv(p); }
   48.31  };
   48.32  
   48.33 -#define G1_PARTIAL_ARRAY_MASK 1
   48.34 +#define G1_PARTIAL_ARRAY_MASK 0x2
   48.35  
   48.36 -inline bool has_partial_array_mask(oop* ref) {
   48.37 -  return (intptr_t) ref & G1_PARTIAL_ARRAY_MASK;
   48.38 +template <class T> inline bool has_partial_array_mask(T* ref) {
   48.39 +  return ((uintptr_t)ref & G1_PARTIAL_ARRAY_MASK) == G1_PARTIAL_ARRAY_MASK;
   48.40  }
   48.41  
   48.42 -inline oop* set_partial_array_mask(oop obj) {
   48.43 -  return (oop*) ((intptr_t) obj | G1_PARTIAL_ARRAY_MASK);
   48.44 +template <class T> inline T* set_partial_array_mask(T obj) {
   48.45 +  assert(((uintptr_t)obj & G1_PARTIAL_ARRAY_MASK) == 0, "Information loss!");
   48.46 +  return (T*) ((uintptr_t)obj | G1_PARTIAL_ARRAY_MASK);
   48.47  }
   48.48  
   48.49 -inline oop clear_partial_array_mask(oop* ref) {
   48.50 -  return oop((intptr_t) ref & ~G1_PARTIAL_ARRAY_MASK);
   48.51 +template <class T> inline oop clear_partial_array_mask(T* ref) {
   48.52 +  return oop((intptr_t)ref & ~G1_PARTIAL_ARRAY_MASK);
   48.53  }
   48.54  
   48.55  class G1ParScanPartialArrayClosure : public G1ParClosureSuper {
   48.56    G1ParScanClosure _scanner;
   48.57 -  template <class T> void process_array_chunk(oop obj, int start, int end);
   48.58  public:
   48.59    G1ParScanPartialArrayClosure(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) :
   48.60      G1ParClosureSuper(g1, par_scan_state), _scanner(g1, par_scan_state) { }
   48.61 -  void do_oop_nv(oop* p);
   48.62 -  void do_oop_nv(narrowOop* p)      { guarantee(false, "NYI"); }
   48.63 +  template <class T> void do_oop_nv(T* p);
   48.64    virtual void do_oop(oop* p)       { do_oop_nv(p); }
   48.65    virtual void do_oop(narrowOop* p) { do_oop_nv(p); }
   48.66  };
   48.67 @@ -105,7 +91,7 @@
   48.68  class G1ParCopyHelper : public G1ParClosureSuper {
   48.69    G1ParScanClosure *_scanner;
   48.70  protected:
   48.71 -  void mark_forwardee(oop* p);
   48.72 +  template <class T> void mark_forwardee(T* p);
   48.73    oop copy_to_survivor_space(oop obj);
   48.74  public:
   48.75    G1ParCopyHelper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state,
   48.76 @@ -117,36 +103,35 @@
   48.77           bool do_mark_forwardee, bool skip_cset_test>
   48.78  class G1ParCopyClosure : public G1ParCopyHelper {
   48.79    G1ParScanClosure _scanner;
   48.80 -  void do_oop_work(oop* p);
   48.81 -  void do_oop_work(narrowOop* p) { guarantee(false, "NYI"); }
   48.82 +  template <class T> void do_oop_work(T* p);
   48.83  public:
   48.84    G1ParCopyClosure(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) :
   48.85      _scanner(g1, par_scan_state), G1ParCopyHelper(g1, par_scan_state, &_scanner) { }
   48.86 -  inline void do_oop_nv(oop* p) {
   48.87 +  template <class T> void do_oop_nv(T* p) {
   48.88      do_oop_work(p);
   48.89      if (do_mark_forwardee)
   48.90        mark_forwardee(p);
   48.91    }
   48.92 -  inline void do_oop_nv(narrowOop* p) { guarantee(false, "NYI"); }
   48.93    virtual void do_oop(oop* p)       { do_oop_nv(p); }
   48.94    virtual void do_oop(narrowOop* p) { do_oop_nv(p); }
   48.95  };
   48.96  
   48.97  typedef G1ParCopyClosure<false, G1BarrierNone, false, false> G1ParScanExtRootClosure;
   48.98  typedef G1ParCopyClosure<true,  G1BarrierNone, false, false> G1ParScanPermClosure;
   48.99 +typedef G1ParCopyClosure<false, G1BarrierRS,   false, false> G1ParScanHeapRSClosure;
  48.100  typedef G1ParCopyClosure<false, G1BarrierNone, true,  false> G1ParScanAndMarkExtRootClosure;
  48.101  typedef G1ParCopyClosure<true,  G1BarrierNone, true,  false> G1ParScanAndMarkPermClosure;
  48.102 -typedef G1ParCopyClosure<false, G1BarrierRS,   false, false> G1ParScanHeapRSClosure;
  48.103  typedef G1ParCopyClosure<false, G1BarrierRS,   true,  false> G1ParScanAndMarkHeapRSClosure;
  48.104  // This is the only case when we set skip_cset_test. Basically, this
  48.105  // closure is (should?) only be called directly while we're draining
  48.106  // the overflow and task queues. In that case we know that the
  48.107  // reference in question points into the collection set, otherwise we
  48.108 -// would not have pushed it on the queue.
  48.109 -typedef G1ParCopyClosure<false, G1BarrierEvac, false, true> G1ParScanHeapEvacClosure;
  48.110 +// would not have pushed it on the queue. The following is defined in
  48.111 +// g1_specialized_oop_closures.hpp.
  48.112 +// typedef G1ParCopyClosure<false, G1BarrierEvac, false, true> G1ParScanHeapEvacClosure;
  48.113  // We need a separate closure to handle references during evacuation
  48.114 -// failure processing, as it cannot asume that the reference already
  48.115 - // points to the collection set (like G1ParScanHeapEvacClosure does).
  48.116 +// failure processing, as we cannot asume that the reference already
  48.117 +// points into the collection set (like G1ParScanHeapEvacClosure does).
  48.118  typedef G1ParCopyClosure<false, G1BarrierEvac, false, false> G1ParScanHeapEvacFailureClosure;
  48.119  
  48.120  class FilterIntoCSClosure: public OopClosure {
  48.121 @@ -158,10 +143,9 @@
  48.122                          G1CollectedHeap* g1, OopClosure* oc) :
  48.123      _dcto_cl(dcto_cl), _g1(g1), _oc(oc)
  48.124    {}
  48.125 -  inline void do_oop_nv(oop* p);
  48.126 -  inline void do_oop_nv(narrowOop* p) { guarantee(false, "NYI"); }
  48.127 -  virtual void do_oop(oop* p);
  48.128 -  virtual void do_oop(narrowOop* p)   { guarantee(false, "NYI"); }
  48.129 +  template <class T> void do_oop_nv(T* p);
  48.130 +  virtual void do_oop(oop* p)        { do_oop_nv(p); }
  48.131 +  virtual void do_oop(narrowOop* p)  { do_oop_nv(p); }
  48.132    bool apply_to_weak_ref_discovered_field() { return true; }
  48.133    bool do_header() { return false; }
  48.134  };
  48.135 @@ -174,10 +158,9 @@
  48.136                                       OopsInHeapRegionClosure* oc) :
  48.137      _g1(g1), _oc(oc)
  48.138    {}
  48.139 -  inline  void do_oop_nv(oop* p);
  48.140 -  inline  void do_oop_nv(narrowOop* p) { guarantee(false, "NYI"); }
  48.141 -  virtual void do_oop(oop* p);
  48.142 -  virtual void do_oop(narrowOop* p)    { guarantee(false, "NYI"); }
  48.143 +  template <class T> void do_oop_nv(T* p);
  48.144 +  virtual void do_oop(oop* p) { do_oop_nv(p); }
  48.145 +  virtual void do_oop(narrowOop* p) { do_oop_nv(p); }
  48.146    bool apply_to_weak_ref_discovered_field() { return true; }
  48.147    bool do_header() { return false; }
  48.148    void set_region(HeapRegion* from) {
  48.149 @@ -195,10 +178,9 @@
  48.150                                              ConcurrentMark* cm)
  48.151    : _g1(g1), _oc(oc), _cm(cm) { }
  48.152  
  48.153 -  inline  void do_oop_nv(oop* p);
  48.154 -  inline  void do_oop_nv(narrowOop* p) { guarantee(false, "NYI"); }
  48.155 -  virtual void do_oop(oop* p);
  48.156 -  virtual void do_oop(narrowOop* p)    { guarantee(false, "NYI"); }
  48.157 +  template <class T> void do_oop_nv(T* p);
  48.158 +  virtual void do_oop(oop* p) { do_oop_nv(p); }
  48.159 +  virtual void do_oop(narrowOop* p) { do_oop_nv(p); }
  48.160    bool apply_to_weak_ref_discovered_field() { return true; }
  48.161    bool do_header() { return false; }
  48.162    void set_region(HeapRegion* from) {
  48.163 @@ -213,10 +195,9 @@
  48.164    int _out_of_region;
  48.165  public:
  48.166    FilterOutOfRegionClosure(HeapRegion* r, OopClosure* oc);
  48.167 -  inline  void do_oop_nv(oop* p);
  48.168 -  inline  void do_oop_nv(narrowOop* p) { guarantee(false, "NYI"); }
  48.169 -  virtual void do_oop(oop* p);
  48.170 -  virtual void do_oop(narrowOop* p)   { guarantee(false, "NYI"); }
  48.171 +  template <class T> void do_oop_nv(T* p);
  48.172 +  virtual void do_oop(oop* p) { do_oop_nv(p); }
  48.173 +  virtual void do_oop(narrowOop* p) { do_oop_nv(p); }
  48.174    bool apply_to_weak_ref_discovered_field() { return true; }
  48.175    bool do_header() { return false; }
  48.176    int out_of_region() { return _out_of_region; }

    49.1 --- a/src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp	Mon Jul 27 09:06:22 2009 -0700
    49.2 +++ b/src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp	Mon Jul 27 17:23:52 2009 -0400
    49.3 @@ -31,9 +31,10 @@
    49.4  // perf-critical inner loop.
    49.5  #define FILTERINTOCSCLOSURE_DOHISTOGRAMCOUNT 0
    49.6  
    49.7 -inline void FilterIntoCSClosure::do_oop_nv(oop* p) {
    49.8 -  oop obj = *p;
    49.9 -  if (obj != NULL && _g1->obj_in_cs(obj)) {
   49.10 +template <class T> inline void FilterIntoCSClosure::do_oop_nv(T* p) {
   49.11 +  T heap_oop = oopDesc::load_heap_oop(p);
   49.12 +  if (!oopDesc::is_null(heap_oop) &&
   49.13 +      _g1->obj_in_cs(oopDesc::decode_heap_oop_not_null(heap_oop))) {
   49.14      _oc->do_oop(p);
   49.15  #if FILTERINTOCSCLOSURE_DOHISTOGRAMCOUNT
   49.16      _dcto_cl->incr_count();
   49.17 @@ -41,44 +42,32 @@
   49.18    }
   49.19  }
   49.20  
   49.21 -inline void FilterIntoCSClosure::do_oop(oop* p)
   49.22 -{
   49.23 -  do_oop_nv(p);
   49.24 -}
   49.25 -
   49.26  #define FILTEROUTOFREGIONCLOSURE_DOHISTOGRAMCOUNT 0
   49.27  
   49.28 -inline void FilterOutOfRegionClosure::do_oop_nv(oop* p) {
   49.29 -  oop obj = *p;
   49.30 -  HeapWord* obj_hw = (HeapWord*)obj;
   49.31 -  if (obj_hw != NULL && (obj_hw < _r_bottom || obj_hw >= _r_end)) {
   49.32 -    _oc->do_oop(p);
   49.33 +template <class T> inline void FilterOutOfRegionClosure::do_oop_nv(T* p) {
   49.34 +  T heap_oop = oopDesc::load_heap_oop(p);
   49.35 +  if (!oopDesc::is_null(heap_oop)) {
   49.36 +    HeapWord* obj_hw = (HeapWord*)oopDesc::decode_heap_oop_not_null(heap_oop);
   49.37 +    if (obj_hw < _r_bottom || obj_hw >= _r_end) {
   49.38 +      _oc->do_oop(p);
   49.39  #if FILTEROUTOFREGIONCLOSURE_DOHISTOGRAMCOUNT
   49.40 -    _out_of_region++;
   49.41 +      _out_of_region++;
   49.42  #endif
   49.43 +    }
   49.44    }
   49.45  }
   49.46  
   49.47 -inline void FilterOutOfRegionClosure::do_oop(oop* p)
   49.48 -{
   49.49 -  do_oop_nv(p);
   49.50 -}
   49.51 -
   49.52 -inline void FilterInHeapRegionAndIntoCSClosure::do_oop_nv(oop* p) {
   49.53 -  oop obj = *p;
   49.54 -  if (obj != NULL && _g1->obj_in_cs(obj))
   49.55 +template <class T> inline void FilterInHeapRegionAndIntoCSClosure::do_oop_nv(T* p) {
   49.56 +  T heap_oop = oopDesc::load_heap_oop(p);
   49.57 +  if (!oopDesc::is_null(heap_oop) &&
   49.58 +      _g1->obj_in_cs(oopDesc::decode_heap_oop_not_null(heap_oop)))
   49.59      _oc->do_oop(p);
   49.60  }
   49.61  
   49.62 -inline void FilterInHeapRegionAndIntoCSClosure::do_oop(oop* p)
   49.63 -{
   49.64 -  do_oop_nv(p);
   49.65 -}
   49.66 -
   49.67 -
   49.68 -inline void FilterAndMarkInHeapRegionAndIntoCSClosure::do_oop_nv(oop* p) {
   49.69 -  oop obj = *p;
   49.70 -  if (obj != NULL) {
   49.71 +template <class T> inline void FilterAndMarkInHeapRegionAndIntoCSClosure::do_oop_nv(T* p) {
   49.72 +  T heap_oop = oopDesc::load_heap_oop(p);
   49.73 +  if (!oopDesc::is_null(heap_oop)) {
   49.74 +    oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
   49.75      HeapRegion* hr = _g1->heap_region_containing((HeapWord*) obj);
   49.76      if (hr != NULL) {
   49.77        if (hr->in_collection_set())
   49.78 @@ -89,24 +78,29 @@
   49.79    }
   49.80  }
   49.81  
   49.82 -inline void FilterAndMarkInHeapRegionAndIntoCSClosure::do_oop(oop* p)
   49.83 -{
   49.84 -  do_oop_nv(p);
   49.85 +// This closure is applied to the fields of the objects that have just been copied.
   49.86 +template <class T> inline void G1ParScanClosure::do_oop_nv(T* p) {
   49.87 +  T heap_oop = oopDesc::load_heap_oop(p);
   49.88 +
   49.89 +  if (!oopDesc::is_null(heap_oop)) {
   49.90 +    oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
   49.91 +    if (_g1->in_cset_fast_test(obj)) {
   49.92 +      // We're not going to even bother checking whether the object is
   49.93 +      // already forwarded or not, as this usually causes an immediate
   49.94 +      // stall. We'll try to prefetch the object (for write, given that
   49.95 +      // we might need to install the forwarding reference) and we'll
   49.96 +      // get back to it when pop it from the queue
   49.97 +      Prefetch::write(obj->mark_addr(), 0);
   49.98 +      Prefetch::read(obj->mark_addr(), (HeapWordSize*2));
   49.99 +
  49.100 +      // slightly paranoid test; I'm trying to catch potential
  49.101 +      // problems before we go into push_on_queue to know where the
  49.102 +      // problem is coming from
  49.103 +      assert(obj == oopDesc::load_decode_heap_oop(p),
  49.104 +             "p should still be pointing to obj");
  49.105 +      _par_scan_state->push_on_queue(p);
  49.106 +    } else {
  49.107 +      _par_scan_state->update_rs(_from, p, _par_scan_state->queue_num());
  49.108 +    }
  49.109 +  }
  49.110  }
  49.111 -
  49.112 -inline void G1ScanAndBalanceClosure::do_oop_nv(oop* p) {
  49.113 -  RefToScanQueue* q;
  49.114 -  if (ParallelGCThreads > 0) {
  49.115 -    // Deal the work out equally.
  49.116 -    _nq = (_nq + 1) % ParallelGCThreads;
  49.117 -    q = _g1->task_queue(_nq);
  49.118 -  } else {
  49.119 -    q = _g1->task_queue(0);
  49.120 -  }
  49.121 -  bool nooverflow = q->push(p);
  49.122 -  guarantee(nooverflow, "Overflow during poplularity region processing");
  49.123 -}
  49.124 -
  49.125 -inline void G1ScanAndBalanceClosure::do_oop(oop* p) {
  49.126 -  do_oop_nv(p);
  49.127 -}

    50.1 --- a/src/share/vm/gc_implementation/g1/g1RemSet.cpp	Mon Jul 27 09:06:22 2009 -0700
    50.2 +++ b/src/share/vm/gc_implementation/g1/g1RemSet.cpp	Mon Jul 27 17:23:52 2009 -0400
    50.3 @@ -65,11 +65,10 @@
    50.4    void set_region(HeapRegion* from) {
    50.5      _blk->set_region(from);
    50.6    }
    50.7 -  virtual void do_oop(narrowOop* p) {
    50.8 -    guarantee(false, "NYI");
    50.9 -  }
   50.10 -  virtual void do_oop(oop* p) {
   50.11 -    oop obj = *p;
   50.12 +  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
   50.13 +  virtual void do_oop(      oop* p) { do_oop_work(p); }
   50.14 +  template <class T> void do_oop_work(T* p) {
   50.15 +    oop obj = oopDesc::load_decode_heap_oop(p);
   50.16      if (_g1->obj_in_cs(obj)) _blk->do_oop(p);
   50.17    }
   50.18    bool apply_to_weak_ref_discovered_field() { return true; }
   50.19 @@ -110,11 +109,10 @@
   50.20  public:
   50.21    VerifyRSCleanCardOopClosure(G1CollectedHeap* g1) : _g1(g1) {}
   50.22  
   50.23 -  virtual void do_oop(narrowOop* p) {
   50.24 -    guarantee(false, "NYI");
   50.25 -  }
   50.26 -  virtual void do_oop(oop* p) {
   50.27 -    oop obj = *p;
   50.28 +  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
   50.29 +  virtual void do_oop(      oop* p) { do_oop_work(p); }
   50.30 +  template <class T> void do_oop_work(T* p) {
   50.31 +    oop obj = oopDesc::load_decode_heap_oop(p);
   50.32      HeapRegion* to = _g1->heap_region_containing(obj);
   50.33      guarantee(to == NULL || !to->in_collection_set(),
   50.34                "Missed a rem set member.");
   50.35 @@ -129,9 +127,9 @@
   50.36  {
   50.37    _seq_task = new SubTasksDone(NumSeqTasks);
   50.38    guarantee(n_workers() > 0, "There should be some workers");
   50.39 -  _new_refs = NEW_C_HEAP_ARRAY(GrowableArray<oop*>*, n_workers());
   50.40 +  _new_refs = NEW_C_HEAP_ARRAY(GrowableArray<OopOrNarrowOopStar>*, n_workers());
   50.41    for (uint i = 0; i < n_workers(); i++) {
   50.42 -    _new_refs[i] = new (ResourceObj::C_HEAP) GrowableArray<oop*>(8192,true);
   50.43 +    _new_refs[i] = new (ResourceObj::C_HEAP) GrowableArray<OopOrNarrowOopStar>(8192,true);
   50.44    }
   50.45  }
   50.46  
   50.47 @@ -140,7 +138,7 @@
   50.48    for (uint i = 0; i < n_workers(); i++) {
   50.49      delete _new_refs[i];
   50.50    }
   50.51 -  FREE_C_HEAP_ARRAY(GrowableArray<oop*>*, _new_refs);
   50.52 +  FREE_C_HEAP_ARRAY(GrowableArray<OopOrNarrowOopStar>*, _new_refs);
   50.53  }
   50.54  
   50.55  void CountNonCleanMemRegionClosure::do_MemRegion(MemRegion mr) {
   50.56 @@ -428,15 +426,15 @@
   50.57    }
   50.58  };
   50.59  
   50.60 -void
   50.61 -HRInto_G1RemSet::scanNewRefsRS(OopsInHeapRegionClosure* oc,
   50.62 -                                             int worker_i) {
   50.63 +template <class T> void
   50.64 +HRInto_G1RemSet::scanNewRefsRS_work(OopsInHeapRegionClosure* oc,
   50.65 +                                    int worker_i) {
   50.66    double scan_new_refs_start_sec = os::elapsedTime();
   50.67    G1CollectedHeap* g1h = G1CollectedHeap::heap();
   50.68    CardTableModRefBS* ct_bs = (CardTableModRefBS*) (g1h->barrier_set());
   50.69    for (int i = 0; i < _new_refs[worker_i]->length(); i++) {
   50.70 -    oop* p = _new_refs[worker_i]->at(i);
   50.71 -    oop obj = *p;
   50.72 +    T* p = (T*) _new_refs[worker_i]->at(i);
   50.73 +    oop obj = oopDesc::load_decode_heap_oop(p);
   50.74      // *p was in the collection set when p was pushed on "_new_refs", but
   50.75      // another thread may have processed this location from an RS, so it
   50.76      // might not point into the CS any longer.  If so, it's obviously been
   50.77 @@ -549,11 +547,10 @@
   50.78    G1CollectedHeap* _g1;
   50.79  public:
   50.80    UpdateRSetOopsIntoCSImmediate(G1CollectedHeap* g1) : _g1(g1) { }
   50.81 -  virtual void do_oop(narrowOop* p) {
   50.82 -    guarantee(false, "NYI");
   50.83 -  }
   50.84 -  virtual void do_oop(oop* p) {
   50.85 -    HeapRegion* to = _g1->heap_region_containing(*p);
   50.86 +  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
   50.87 +  virtual void do_oop(      oop* p) { do_oop_work(p); }
   50.88 +  template <class T> void do_oop_work(T* p) {
   50.89 +    HeapRegion* to = _g1->heap_region_containing(oopDesc::load_decode_heap_oop(p));
   50.90      if (to->in_collection_set()) {
   50.91        to->rem_set()->add_reference(p, 0);
   50.92      }
   50.93 @@ -567,11 +564,10 @@
   50.94  public:
   50.95    UpdateRSetOopsIntoCSDeferred(G1CollectedHeap* g1, DirtyCardQueue* dcq) :
   50.96      _g1(g1), _ct_bs((CardTableModRefBS*)_g1->barrier_set()), _dcq(dcq) { }
   50.97 -  virtual void do_oop(narrowOop* p) {
   50.98 -    guarantee(false, "NYI");
   50.99 -  }
  50.100 -  virtual void do_oop(oop* p) {
  50.101 -    oop obj = *p;
  50.102 +  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
  50.103 +  virtual void do_oop(      oop* p) { do_oop_work(p); }
  50.104 +  template <class T> void do_oop_work(T* p) {
  50.105 +    oop obj = oopDesc::load_decode_heap_oop(p);
  50.106      if (_g1->obj_in_cs(obj)) {
  50.107        size_t card_index = _ct_bs->index_for(p);
  50.108        if (_ct_bs->mark_card_deferred(card_index)) {
  50.109 @@ -581,10 +577,10 @@
  50.110    }
  50.111  };
  50.112  
  50.113 -void HRInto_G1RemSet::new_refs_iterate(OopClosure* cl) {
  50.114 +template <class T> void HRInto_G1RemSet::new_refs_iterate_work(OopClosure* cl) {
  50.115    for (size_t i = 0; i < n_workers(); i++) {
  50.116      for (int j = 0; j < _new_refs[i]->length(); j++) {
  50.117 -      oop* p = _new_refs[i]->at(j);
  50.118 +      T* p = (T*) _new_refs[i]->at(j);
  50.119        cl->do_oop(p);
  50.120      }
  50.121    }

    51.1 --- a/src/share/vm/gc_implementation/g1/g1RemSet.hpp	Mon Jul 27 09:06:22 2009 -0700
    51.2 +++ b/src/share/vm/gc_implementation/g1/g1RemSet.hpp	Mon Jul 27 17:23:52 2009 -0400
    51.3 @@ -62,10 +62,12 @@
    51.4    // If "this" is of the given subtype, return "this", else "NULL".
    51.5    virtual HRInto_G1RemSet* as_HRInto_G1RemSet() { return NULL; }
    51.6  
    51.7 -  // Record, if necessary, the fact that *p (where "p" is in region "from")
    51.8 -  // has changed to its new value.
    51.9 +  // Record, if necessary, the fact that *p (where "p" is in region "from",
   51.10 +  // and is, a fortiori, required to be non-NULL) has changed to its new value.
   51.11    virtual void write_ref(HeapRegion* from, oop* p) = 0;
   51.12 +  virtual void write_ref(HeapRegion* from, narrowOop* p) = 0;
   51.13    virtual void par_write_ref(HeapRegion* from, oop* p, int tid) = 0;
   51.14 +  virtual void par_write_ref(HeapRegion* from, narrowOop* p, int tid) = 0;
   51.15  
   51.16    // Requires "region_bm" and "card_bm" to be bitmaps with 1 bit per region
   51.17    // or card, respectively, such that a region or card with a corresponding
   51.18 @@ -105,7 +107,9 @@
   51.19  
   51.20    // Nothing is necessary in the version below.
   51.21    void write_ref(HeapRegion* from, oop* p) {}
   51.22 +  void write_ref(HeapRegion* from, narrowOop* p) {}
   51.23    void par_write_ref(HeapRegion* from, oop* p, int tid) {}
   51.24 +  void par_write_ref(HeapRegion* from, narrowOop* p, int tid) {}
   51.25  
   51.26    void scrub(BitMap* region_bm, BitMap* card_bm) {}
   51.27    void scrub_par(BitMap* region_bm, BitMap* card_bm,
   51.28 @@ -143,8 +147,19 @@
   51.29    // their references into the collection summarized in "_new_refs".
   51.30    bool _par_traversal_in_progress;
   51.31    void set_par_traversal(bool b) { _par_traversal_in_progress = b; }
   51.32 -  GrowableArray<oop*>** _new_refs;
   51.33 -  void new_refs_iterate(OopClosure* cl);
   51.34 +  GrowableArray<OopOrNarrowOopStar>** _new_refs;
   51.35 +  template <class T> void new_refs_iterate_work(OopClosure* cl);
   51.36 +  void new_refs_iterate(OopClosure* cl) {
   51.37 +    if (UseCompressedOops) {
   51.38 +      new_refs_iterate_work<narrowOop>(cl);
   51.39 +    } else {
   51.40 +      new_refs_iterate_work<oop>(cl);
   51.41 +    }
   51.42 +  }
   51.43 +
   51.44 +protected:
   51.45 +  template <class T> void write_ref_nv(HeapRegion* from, T* p);
   51.46 +  template <class T> void par_write_ref_nv(HeapRegion* from, T* p, int tid);
   51.47  
   51.48  public:
   51.49    // This is called to reset dual hash tables after the gc pause
   51.50 @@ -161,7 +176,14 @@
   51.51    void prepare_for_oops_into_collection_set_do();
   51.52    void cleanup_after_oops_into_collection_set_do();
   51.53    void scanRS(OopsInHeapRegionClosure* oc, int worker_i);
   51.54 -  void scanNewRefsRS(OopsInHeapRegionClosure* oc, int worker_i);
   51.55 +  template <class T> void scanNewRefsRS_work(OopsInHeapRegionClosure* oc, int worker_i);
   51.56 +  void scanNewRefsRS(OopsInHeapRegionClosure* oc, int worker_i) {
   51.57 +    if (UseCompressedOops) {
   51.58 +      scanNewRefsRS_work<narrowOop>(oc, worker_i);
   51.59 +    } else {
   51.60 +      scanNewRefsRS_work<oop>(oc, worker_i);
   51.61 +    }
   51.62 +  }
   51.63    void updateRS(int worker_i);
   51.64    HeapRegion* calculateStartRegion(int i);
   51.65  
   51.66 @@ -172,12 +194,22 @@
   51.67  
   51.68    // Record, if necessary, the fact that *p (where "p" is in region "from",
   51.69    // which is required to be non-NULL) has changed to a new non-NULL value.
   51.70 -  inline void write_ref(HeapRegion* from, oop* p);
   51.71 -  // The "_nv" version is the same; it exists just so that it is not virtual.
   51.72 -  inline void write_ref_nv(HeapRegion* from, oop* p);
   51.73 +  // [Below the virtual version calls a non-virtual protected
   51.74 +  // workhorse that is templatified for narrow vs wide oop.]
   51.75 +  inline void write_ref(HeapRegion* from, oop* p) {
   51.76 +    write_ref_nv(from, p);
   51.77 +  }
   51.78 +  inline void write_ref(HeapRegion* from, narrowOop* p) {
   51.79 +    write_ref_nv(from, p);
   51.80 +  }
   51.81 +  inline void par_write_ref(HeapRegion* from, oop* p, int tid) {
   51.82 +    par_write_ref_nv(from, p, tid);
   51.83 +  }
   51.84 +  inline void par_write_ref(HeapRegion* from, narrowOop* p, int tid) {
   51.85 +    par_write_ref_nv(from, p, tid);
   51.86 +  }
   51.87  
   51.88 -  inline bool self_forwarded(oop obj);
   51.89 -  inline void par_write_ref(HeapRegion* from, oop* p, int tid);
   51.90 +  bool self_forwarded(oop obj);
   51.91  
   51.92    void scrub(BitMap* region_bm, BitMap* card_bm);
   51.93    void scrub_par(BitMap* region_bm, BitMap* card_bm,
   51.94 @@ -208,6 +240,9 @@
   51.95    HeapRegion* _from;
   51.96    HRInto_G1RemSet* _rs;
   51.97    int _worker_i;
   51.98 +
   51.99 +  template <class T> void do_oop_work(T* p);
  51.100 +
  51.101  public:
  51.102    UpdateRSOopClosure(HRInto_G1RemSet* rs, int worker_i = 0) :
  51.103      _from(NULL), _rs(rs), _worker_i(worker_i) {
  51.104 @@ -219,11 +254,10 @@
  51.105      _from = from;
  51.106    }
  51.107  
  51.108 -  virtual void do_oop(narrowOop* p);
  51.109 -  virtual void do_oop(oop* p);
  51.110 +  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
  51.111 +  virtual void do_oop(oop* p)       { do_oop_work(p); }
  51.112  
  51.113    // Override: this closure is idempotent.
  51.114    //  bool idempotent() { return true; }
  51.115    bool apply_to_weak_ref_discovered_field() { return true; }
  51.116  };
  51.117 -

    52.1 --- a/src/share/vm/gc_implementation/g1/g1RemSet.inline.hpp	Mon Jul 27 09:06:22 2009 -0700
    52.2 +++ b/src/share/vm/gc_implementation/g1/g1RemSet.inline.hpp	Mon Jul 27 17:23:52 2009 -0400
    52.3 @@ -30,12 +30,8 @@
    52.4    }
    52.5  }
    52.6  
    52.7 -inline void HRInto_G1RemSet::write_ref_nv(HeapRegion* from, oop* p) {
    52.8 -  par_write_ref(from, p, 0);
    52.9 -}
   52.10 -
   52.11 -inline void HRInto_G1RemSet::write_ref(HeapRegion* from, oop* p) {
   52.12 -  write_ref_nv(from, p);
   52.13 +template <class T> inline void HRInto_G1RemSet::write_ref_nv(HeapRegion* from, T* p) {
   52.14 +  par_write_ref_nv(from, p, 0);
   52.15  }
   52.16  
   52.17  inline bool HRInto_G1RemSet::self_forwarded(oop obj) {
   52.18 @@ -43,8 +39,8 @@
   52.19    return result;
   52.20  }
   52.21  
   52.22 -inline void HRInto_G1RemSet::par_write_ref(HeapRegion* from, oop* p, int tid) {
   52.23 -  oop obj = *p;
   52.24 +template <class T> inline void HRInto_G1RemSet::par_write_ref_nv(HeapRegion* from, T* p, int tid) {
   52.25 +  oop obj = oopDesc::load_decode_heap_oop(p);
   52.26  #ifdef ASSERT
   52.27    // can't do because of races
   52.28    // assert(obj == NULL || obj->is_oop(), "expected an oop");
   52.29 @@ -71,7 +67,7 @@
   52.30      // false during the evacuation failure handing.
   52.31      if (_par_traversal_in_progress &&
   52.32          to->in_collection_set() && !self_forwarded(obj)) {
   52.33 -      _new_refs[tid]->push(p);
   52.34 +      _new_refs[tid]->push((void*)p);
   52.35        // Deferred updates to the Cset are either discarded (in the normal case),
   52.36        // or processed (if an evacuation failure occurs) at the end
   52.37        // of the collection.
   52.38 @@ -89,11 +85,7 @@
   52.39    }
   52.40  }
   52.41  
   52.42 -inline void UpdateRSOopClosure::do_oop(narrowOop* p) {
   52.43 -  guarantee(false, "NYI");
   52.44 -}
   52.45 -
   52.46 -inline void UpdateRSOopClosure::do_oop(oop* p) {
   52.47 +template <class T> inline void UpdateRSOopClosure::do_oop_work(T* p) {
   52.48    assert(_from != NULL, "from region must be non-NULL");
   52.49    _rs->par_write_ref(_from, p, _worker_i);
   52.50  }

    53.1 --- a/src/share/vm/gc_implementation/g1/g1SATBCardTableModRefBS.cpp	Mon Jul 27 09:06:22 2009 -0700
    53.2 +++ b/src/share/vm/gc_implementation/g1/g1SATBCardTableModRefBS.cpp	Mon Jul 27 17:23:52 2009 -0400
    53.3 @@ -34,6 +34,7 @@
    53.4  
    53.5  
    53.6  void G1SATBCardTableModRefBS::enqueue(oop pre_val) {
    53.7 +  assert(pre_val->is_oop_or_null(true), "Error");
    53.8    if (!JavaThread::satb_mark_queue_set().active()) return;
    53.9    Thread* thr = Thread::current();
   53.10    if (thr->is_Java_thread()) {
   53.11 @@ -46,32 +47,31 @@
   53.12  }
   53.13  
   53.14  // When we know the current java thread:
   53.15 -void
   53.16 -G1SATBCardTableModRefBS::write_ref_field_pre_static(void* field,
   53.17 -                                                    oop newVal,
   53.18 +template <class T> void
   53.19 +G1SATBCardTableModRefBS::write_ref_field_pre_static(T* field,
   53.20 +                                                    oop new_val,
   53.21                                                      JavaThread* jt) {
   53.22    if (!JavaThread::satb_mark_queue_set().active()) return;
   53.23 -  assert(!UseCompressedOops, "Else will need to modify this to deal with narrowOop");
   53.24 -  oop preVal = *(oop*)field;
   53.25 -  if (preVal != NULL) {
   53.26 -    jt->satb_mark_queue().enqueue(preVal);
   53.27 +  T heap_oop = oopDesc::load_heap_oop(field);
   53.28 +  if (!oopDesc::is_null(heap_oop)) {
   53.29 +    oop pre_val = oopDesc::decode_heap_oop_not_null(heap_oop);
   53.30 +    assert(pre_val->is_oop(true /* ignore mark word */), "Error");
   53.31 +    jt->satb_mark_queue().enqueue(pre_val);
   53.32    }
   53.33  }
   53.34  
   53.35 -void
   53.36 -G1SATBCardTableModRefBS::write_ref_array_pre(MemRegion mr) {
   53.37 +template <class T> void
   53.38 +G1SATBCardTableModRefBS::write_ref_array_pre_work(T* dst, int count) {
   53.39    if (!JavaThread::satb_mark_queue_set().active()) return;
   53.40 -  assert(!UseCompressedOops, "Else will need to modify this to deal with narrowOop");
   53.41 -  oop* elem_ptr = (oop*)mr.start();
   53.42 -  while ((HeapWord*)elem_ptr < mr.end()) {
   53.43 -    oop elem = *elem_ptr;
   53.44 -    if (elem != NULL) enqueue(elem);
   53.45 -    elem_ptr++;
   53.46 +  T* elem_ptr = dst;
   53.47 +  for (int i = 0; i < count; i++, elem_ptr++) {
   53.48 +    T heap_oop = oopDesc::load_heap_oop(elem_ptr);
   53.49 +    if (!oopDesc::is_null(heap_oop)) {
   53.50 +      enqueue(oopDesc::decode_heap_oop_not_null(heap_oop));
   53.51 +    }
   53.52    }
   53.53  }
   53.54  
   53.55 -
   53.56 -
   53.57  G1SATBCardTableLoggingModRefBS::
   53.58  G1SATBCardTableLoggingModRefBS(MemRegion whole_heap,
   53.59                                 int max_covered_regions) :

    54.1 --- a/src/share/vm/gc_implementation/g1/g1SATBCardTableModRefBS.hpp	Mon Jul 27 09:06:22 2009 -0700
    54.2 +++ b/src/share/vm/gc_implementation/g1/g1SATBCardTableModRefBS.hpp	Mon Jul 27 17:23:52 2009 -0400
    54.3 @@ -47,31 +47,41 @@
    54.4  
    54.5    // This notes that we don't need to access any BarrierSet data
    54.6    // structures, so this can be called from a static context.
    54.7 -  static void write_ref_field_pre_static(void* field, oop newVal) {
    54.8 -    assert(!UseCompressedOops, "Else needs to be templatized");
    54.9 -    oop preVal = *((oop*)field);
   54.10 -    if (preVal != NULL) {
   54.11 -      enqueue(preVal);
   54.12 +  template <class T> static void write_ref_field_pre_static(T* field, oop newVal) {
   54.13 +    T heap_oop = oopDesc::load_heap_oop(field);
   54.14 +    if (!oopDesc::is_null(heap_oop)) {
   54.15 +      enqueue(oopDesc::decode_heap_oop(heap_oop));
   54.16      }
   54.17    }
   54.18  
   54.19    // When we know the current java thread:
   54.20 -  static void write_ref_field_pre_static(void* field, oop newVal,
   54.21 -                                         JavaThread* jt);
   54.22 +  template <class T> static void write_ref_field_pre_static(T* field, oop newVal,
   54.23 +                                                            JavaThread* jt);
   54.24  
   54.25    // We export this to make it available in cases where the static
   54.26    // type of the barrier set is known.  Note that it is non-virtual.
   54.27 -  inline void inline_write_ref_field_pre(void* field, oop newVal) {
   54.28 +  template <class T> inline void inline_write_ref_field_pre(T* field, oop newVal) {
   54.29      write_ref_field_pre_static(field, newVal);
   54.30    }
   54.31  
   54.32 -  // This is the more general virtual version.
   54.33 -  void write_ref_field_pre_work(void* field, oop new_val) {
   54.34 +  // These are the more general virtual versions.
   54.35 +  virtual void write_ref_field_pre_work(oop* field, oop new_val) {
   54.36      inline_write_ref_field_pre(field, new_val);
   54.37    }
   54.38 +  virtual void write_ref_field_pre_work(narrowOop* field, oop new_val) {
   54.39 +    inline_write_ref_field_pre(field, new_val);
   54.40 +  }
   54.41 +  virtual void write_ref_field_pre_work(void* field, oop new_val) {
   54.42 +    guarantee(false, "Not needed");
   54.43 +  }
   54.44  
   54.45 -  virtual void write_ref_array_pre(MemRegion mr);
   54.46 -
   54.47 +  template <class T> void write_ref_array_pre_work(T* dst, int count);
   54.48 +  virtual void write_ref_array_pre(oop* dst, int count) {
   54.49 +    write_ref_array_pre_work(dst, count);
   54.50 +  }
   54.51 +  virtual void write_ref_array_pre(narrowOop* dst, int count) {
   54.52 +    write_ref_array_pre_work(dst, count);
   54.53 +  }
   54.54  };
   54.55  
   54.56  // Adds card-table logging to the post-barrier.

    55.1 --- a/src/share/vm/gc_implementation/g1/g1_globals.hpp	Mon Jul 27 09:06:22 2009 -0700
    55.2 +++ b/src/share/vm/gc_implementation/g1/g1_globals.hpp	Mon Jul 27 17:23:52 2009 -0400
    55.3 @@ -80,9 +80,6 @@
    55.4    develop(bool, G1TraceConcurrentRefinement, false,                         \
    55.5            "Trace G1 concurrent refinement")                                 \
    55.6                                                                              \
    55.7 -  develop(bool, G1ConcMark, true,                                           \
    55.8 -          "If true, run concurrent marking for G1")                         \
    55.9 -                                                                            \
   55.10    product(intx, G1MarkStackSize, 2 * 1024 * 1024,                           \
   55.11            "Size of the mark stack for concurrent marking.")                 \
   55.12                                                                              \

    56.1 --- a/src/share/vm/gc_implementation/g1/g1_specialized_oop_closures.hpp	Mon Jul 27 09:06:22 2009 -0700
    56.2 +++ b/src/share/vm/gc_implementation/g1/g1_specialized_oop_closures.hpp	Mon Jul 27 17:23:52 2009 -0400
    56.3 @@ -37,14 +37,12 @@
    56.4  class G1ParCopyClosure;
    56.5  class G1ParScanClosure;
    56.6  
    56.7 -typedef G1ParCopyClosure<false, G1BarrierEvac, false, true>
    56.8 -                                                      G1ParScanHeapEvacClosure;
    56.9 +typedef G1ParCopyClosure<false, G1BarrierEvac, false, true> G1ParScanHeapEvacClosure;
   56.10  
   56.11  class FilterIntoCSClosure;
   56.12  class FilterOutOfRegionClosure;
   56.13  class FilterInHeapRegionAndIntoCSClosure;
   56.14  class FilterAndMarkInHeapRegionAndIntoCSClosure;
   56.15 -class G1ScanAndBalanceClosure;
   56.16  
   56.17  #ifdef FURTHER_SPECIALIZED_OOP_OOP_ITERATE_CLOSURES
   56.18  #error "FURTHER_SPECIALIZED_OOP_OOP_ITERATE_CLOSURES already defined."
   56.19 @@ -56,8 +54,7 @@
   56.20        f(FilterIntoCSClosure,_nv)                        \
   56.21        f(FilterOutOfRegionClosure,_nv)                   \
   56.22        f(FilterInHeapRegionAndIntoCSClosure,_nv)         \
   56.23 -      f(FilterAndMarkInHeapRegionAndIntoCSClosure,_nv)  \
   56.24 -      f(G1ScanAndBalanceClosure,_nv)
   56.25 +      f(FilterAndMarkInHeapRegionAndIntoCSClosure,_nv)
   56.26  
   56.27  #ifdef FURTHER_SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES
   56.28  #error "FURTHER_SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES already defined."

    57.1 --- a/src/share/vm/gc_implementation/g1/heapRegion.cpp	Mon Jul 27 09:06:22 2009 -0700
    57.2 +++ b/src/share/vm/gc_implementation/g1/heapRegion.cpp	Mon Jul 27 17:23:52 2009 -0400
    57.3 @@ -66,16 +66,16 @@
    57.4    bool failures() { return _failures; }
    57.5    int n_failures() { return _n_failures; }
    57.6  
    57.7 -  virtual void do_oop(narrowOop* p) {
    57.8 -    guarantee(false, "NYI");
    57.9 -  }
   57.10 +  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
   57.11 +  virtual void do_oop(      oop* p) { do_oop_work(p); }
   57.12  
   57.13 -  void do_oop(oop* p) {
   57.14 +  template <class T> void do_oop_work(T* p) {
   57.15      assert(_containing_obj != NULL, "Precondition");
   57.16      assert(!_g1h->is_obj_dead_cond(_containing_obj, _use_prev_marking),
   57.17             "Precondition");
   57.18 -    oop obj = *p;
   57.19 -    if (obj != NULL) {
   57.20 +    T heap_oop = oopDesc::load_heap_oop(p);
   57.21 +    if (!oopDesc::is_null(heap_oop)) {
   57.22 +      oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
   57.23        bool failed = false;
   57.24        if (!_g1h->is_in_closed_subset(obj) ||
   57.25            _g1h->is_obj_dead_cond(obj, _use_prev_marking)) {
   57.26 @@ -106,8 +106,8 @@
   57.27        }
   57.28  
   57.29        if (!_g1h->full_collection()) {
   57.30 -        HeapRegion* from = _g1h->heap_region_containing(p);
   57.31 -        HeapRegion* to   = _g1h->heap_region_containing(*p);
   57.32 +        HeapRegion* from = _g1h->heap_region_containing((HeapWord*)p);
   57.33 +        HeapRegion* to   = _g1h->heap_region_containing(obj);
   57.34          if (from != NULL && to != NULL &&
   57.35              from != to &&
   57.36              !to->isHumongous()) {
   57.37 @@ -534,13 +534,13 @@
   57.38    // Otherwise, find the obj that extends onto mr.start().
   57.39  
   57.40    assert(cur <= mr.start()
   57.41 -         && (oop(cur)->klass() == NULL ||
   57.42 +         && (oop(cur)->klass_or_null() == NULL ||
   57.43               cur + oop(cur)->size() > mr.start()),
   57.44           "postcondition of block_start");
   57.45    oop obj;
   57.46    while (cur < mr.end()) {
   57.47      obj = oop(cur);
   57.48 -    if (obj->klass() == NULL) {
   57.49 +    if (obj->klass_or_null() == NULL) {
   57.50        // Ran into an unparseable point.
   57.51        return cur;
   57.52      } else if (!g1h->is_obj_dead(obj)) {
   57.53 @@ -577,7 +577,7 @@
   57.54    assert(cur <= mr.start(), "Postcondition");
   57.55  
   57.56    while (cur <= mr.start()) {
   57.57 -    if (oop(cur)->klass() == NULL) {
   57.58 +    if (oop(cur)->klass_or_null() == NULL) {
   57.59        // Ran into an unparseable point.
   57.60        return cur;
   57.61      }
   57.62 @@ -591,7 +591,7 @@
   57.63    obj = oop(cur);
   57.64    // If we finish this loop...
   57.65    assert(cur <= mr.start()
   57.66 -         && obj->klass() != NULL
   57.67 +         && obj->klass_or_null() != NULL
   57.68           && cur + obj->size() > mr.start(),
   57.69           "Loop postcondition");
   57.70    if (!g1h->is_obj_dead(obj)) {
   57.71 @@ -601,7 +601,7 @@
   57.72    HeapWord* next;
   57.73    while (cur < mr.end()) {
   57.74      obj = oop(cur);
   57.75 -    if (obj->klass() == NULL) {
   57.76 +    if (obj->klass_or_null() == NULL) {
   57.77        // Ran into an unparseable point.
   57.78        return cur;
   57.79      };
   57.80 @@ -703,7 +703,7 @@
   57.81    }
   57.82    if (vl_cl.failures()) {
   57.83      gclog_or_tty->print_cr("Heap:");
   57.84 -    G1CollectedHeap::heap()->print();
   57.85 +    G1CollectedHeap::heap()->print_on(gclog_or_tty, true /* extended */);
   57.86      gclog_or_tty->print_cr("");
   57.87    }
   57.88    if (VerifyDuringGC &&
   57.89 @@ -781,8 +781,13 @@
   57.90      // will pick up the right saved_mark_word() as the high water mark
   57.91      // of the region. Either way, the behaviour will be correct.
   57.92      ContiguousSpace::set_saved_mark();
   57.93 +    OrderAccess::storestore();
   57.94      _gc_time_stamp = curr_gc_time_stamp;
   57.95 -    OrderAccess::fence();
   57.96 +    // The following fence is to force a flush of the writes above, but
   57.97 +    // is strictly not needed because when an allocating worker thread
   57.98 +    // calls set_saved_mark() it does so under the ParGCRareEvent_lock;
   57.99 +    // when the lock is released, the write will be flushed.
  57.100 +    // OrderAccess::fence();
  57.101    }
  57.102  }
  57.103

    58.1 --- a/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp	Mon Jul 27 09:06:22 2009 -0700
    58.2 +++ b/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp	Mon Jul 27 17:23:52 2009 -0400
    58.3 @@ -126,7 +126,7 @@
    58.4      }
    58.5    }
    58.6  
    58.7 -  void add_reference_work(oop* from, bool par) {
    58.8 +  void add_reference_work(OopOrNarrowOopStar from, bool par) {
    58.9      // Must make this robust in case "from" is not in "_hr", because of
   58.10      // concurrency.
   58.11  
   58.12 @@ -173,11 +173,11 @@
   58.13      _bm.clear();
   58.14    }
   58.15  
   58.16 -  void add_reference(oop* from) {
   58.17 +  void add_reference(OopOrNarrowOopStar from) {
   58.18      add_reference_work(from, /*parallel*/ true);
   58.19    }
   58.20  
   58.21 -  void seq_add_reference(oop* from) {
   58.22 +  void seq_add_reference(OopOrNarrowOopStar from) {
   58.23      add_reference_work(from, /*parallel*/ false);
   58.24    }
   58.25  
   58.26 @@ -220,7 +220,7 @@
   58.27    }
   58.28  
   58.29    // Requires "from" to be in "hr()".
   58.30 -  bool contains_reference(oop* from) const {
   58.31 +  bool contains_reference(OopOrNarrowOopStar from) const {
   58.32      assert(hr()->is_in_reserved(from), "Precondition.");
   58.33      size_t card_ind = pointer_delta(from, hr()->bottom(),
   58.34                                      CardTableModRefBS::card_size);
   58.35 @@ -394,7 +394,7 @@
   58.36    void set_next(PosParPRT* nxt) { _next = nxt; }
   58.37    PosParPRT** next_addr() { return &_next; }
   58.38  
   58.39 -  void add_reference(oop* from, int tid) {
   58.40 +  void add_reference(OopOrNarrowOopStar from, int tid) {
   58.41      // Expand if necessary.
   58.42      PerRegionTable** pt = par_tables();
   58.43      if (par_tables() == NULL && tid > 0 && hr()->is_gc_alloc_region()) {
   58.44 @@ -447,7 +447,7 @@
   58.45      return res;
   58.46    }
   58.47  
   58.48 -  bool contains_reference(oop* from) const {
   58.49 +  bool contains_reference(OopOrNarrowOopStar from) const {
   58.50      if (PerRegionTable::contains_reference(from)) return true;
   58.51      if (_par_tables != NULL) {
   58.52        for (int i = 0; i < HeapRegionRemSet::num_par_rem_sets()-1; i++) {
   58.53 @@ -564,12 +564,15 @@
   58.54  }
   58.55  #endif
   58.56  
   58.57 -void OtherRegionsTable::add_reference(oop* from, int tid) {
   58.58 +void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, int tid) {
   58.59    size_t cur_hrs_ind = hr()->hrs_index();
   58.60  
   58.61  #if HRRS_VERBOSE
   58.62    gclog_or_tty->print_cr("ORT::add_reference_work(" PTR_FORMAT "->" PTR_FORMAT ").",
   58.63 -                                                  from, *from);
   58.64 +                                                  from,
   58.65 +                                                  UseCompressedOops
   58.66 +                                                  ? oopDesc::load_decode_heap_oop((narrowOop*)from)
   58.67 +                                                  : oopDesc::load_decode_heap_oop((oop*)from));
   58.68  #endif
   58.69  
   58.70    int from_card = (int)(uintptr_t(from) >> CardTableModRefBS::card_shift);
   58.71 @@ -1021,13 +1024,13 @@
   58.72    }
   58.73  }
   58.74  
   58.75 -bool OtherRegionsTable::contains_reference(oop* from) const {
   58.76 +bool OtherRegionsTable::contains_reference(OopOrNarrowOopStar from) const {
   58.77    // Cast away const in this case.
   58.78    MutexLockerEx x((Mutex*)&_m, Mutex::_no_safepoint_check_flag);
   58.79    return contains_reference_locked(from);
   58.80  }
   58.81  
   58.82 -bool OtherRegionsTable::contains_reference_locked(oop* from) const {
   58.83 +bool OtherRegionsTable::contains_reference_locked(OopOrNarrowOopStar from) const {
   58.84    HeapRegion* hr = _g1h->heap_region_containing_raw(from);
   58.85    if (hr == NULL) return false;
   58.86    RegionIdx_t hr_ind = (RegionIdx_t) hr->hrs_index();
   58.87 @@ -1288,24 +1291,24 @@
   58.88  
   58.89  
   58.90  
   58.91 -oop**        HeapRegionRemSet::_recorded_oops = NULL;
   58.92 -HeapWord**   HeapRegionRemSet::_recorded_cards = NULL;
   58.93 -HeapRegion** HeapRegionRemSet::_recorded_regions = NULL;
   58.94 -int          HeapRegionRemSet::_n_recorded = 0;
   58.95 +OopOrNarrowOopStar* HeapRegionRemSet::_recorded_oops = NULL;
   58.96 +HeapWord**          HeapRegionRemSet::_recorded_cards = NULL;
   58.97 +HeapRegion**        HeapRegionRemSet::_recorded_regions = NULL;
   58.98 +int                 HeapRegionRemSet::_n_recorded = 0;
   58.99  
  58.100  HeapRegionRemSet::Event* HeapRegionRemSet::_recorded_events = NULL;
  58.101  int*         HeapRegionRemSet::_recorded_event_index = NULL;
  58.102  int          HeapRegionRemSet::_n_recorded_events = 0;
  58.103  
  58.104 -void HeapRegionRemSet::record(HeapRegion* hr, oop* f) {
  58.105 +void HeapRegionRemSet::record(HeapRegion* hr, OopOrNarrowOopStar f) {
  58.106    if (_recorded_oops == NULL) {
  58.107      assert(_n_recorded == 0
  58.108             && _recorded_cards == NULL
  58.109             && _recorded_regions == NULL,
  58.110             "Inv");
  58.111 -    _recorded_oops = NEW_C_HEAP_ARRAY(oop*, MaxRecorded);
  58.112 -    _recorded_cards = NEW_C_HEAP_ARRAY(HeapWord*, MaxRecorded);
  58.113 -    _recorded_regions = NEW_C_HEAP_ARRAY(HeapRegion*, MaxRecorded);
  58.114 +    _recorded_oops    = NEW_C_HEAP_ARRAY(OopOrNarrowOopStar, MaxRecorded);
  58.115 +    _recorded_cards   = NEW_C_HEAP_ARRAY(HeapWord*,          MaxRecorded);
  58.116 +    _recorded_regions = NEW_C_HEAP_ARRAY(HeapRegion*,        MaxRecorded);
  58.117    }
  58.118    if (_n_recorded == MaxRecorded) {
  58.119      gclog_or_tty->print_cr("Filled up 'recorded' (%d).", MaxRecorded);
  58.120 @@ -1408,21 +1411,21 @@
  58.121    HeapRegionRemSet* hrrs = hr0->rem_set();
  58.122  
  58.123    // Make three references from region 0x101...
  58.124 -  hrrs->add_reference((oop*)hr1_start);
  58.125 -  hrrs->add_reference((oop*)hr1_mid);
  58.126 -  hrrs->add_reference((oop*)hr1_last);
  58.127 +  hrrs->add_reference((OopOrNarrowOopStar)hr1_start);
  58.128 +  hrrs->add_reference((OopOrNarrowOopStar)hr1_mid);
  58.129 +  hrrs->add_reference((OopOrNarrowOopStar)hr1_last);
  58.130  
  58.131 -  hrrs->add_reference((oop*)hr2_start);
  58.132 -  hrrs->add_reference((oop*)hr2_mid);
  58.133 -  hrrs->add_reference((oop*)hr2_last);
  58.134 +  hrrs->add_reference((OopOrNarrowOopStar)hr2_start);
  58.135 +  hrrs->add_reference((OopOrNarrowOopStar)hr2_mid);
  58.136 +  hrrs->add_reference((OopOrNarrowOopStar)hr2_last);
  58.137  
  58.138 -  hrrs->add_reference((oop*)hr3_start);
  58.139 -  hrrs->add_reference((oop*)hr3_mid);
  58.140 -  hrrs->add_reference((oop*)hr3_last);
  58.141 +  hrrs->add_reference((OopOrNarrowOopStar)hr3_start);
  58.142 +  hrrs->add_reference((OopOrNarrowOopStar)hr3_mid);
  58.143 +  hrrs->add_reference((OopOrNarrowOopStar)hr3_last);
  58.144  
  58.145    // Now cause a coarsening.
  58.146 -  hrrs->add_reference((oop*)hr4->bottom());
  58.147 -  hrrs->add_reference((oop*)hr5->bottom());
  58.148 +  hrrs->add_reference((OopOrNarrowOopStar)hr4->bottom());
  58.149 +  hrrs->add_reference((OopOrNarrowOopStar)hr5->bottom());
  58.150  
  58.151    // Now, does iteration yield these three?
  58.152    HeapRegionRemSetIterator iter;

    59.1 --- a/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp	Mon Jul 27 09:06:22 2009 -0700
    59.2 +++ b/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp	Mon Jul 27 17:23:52 2009 -0400
    59.3 @@ -116,9 +116,9 @@
    59.4  
    59.5    // For now.  Could "expand" some tables in the future, so that this made
    59.6    // sense.
    59.7 -  void add_reference(oop* from, int tid);
    59.8 +  void add_reference(OopOrNarrowOopStar from, int tid);
    59.9  
   59.10 -  void add_reference(oop* from) {
   59.11 +  void add_reference(OopOrNarrowOopStar from) {
   59.12      return add_reference(from, 0);
   59.13    }
   59.14  
   59.15 @@ -140,8 +140,8 @@
   59.16    static size_t static_mem_size();
   59.17    static size_t fl_mem_size();
   59.18  
   59.19 -  bool contains_reference(oop* from) const;
   59.20 -  bool contains_reference_locked(oop* from) const;
   59.21 +  bool contains_reference(OopOrNarrowOopStar from) const;
   59.22 +  bool contains_reference_locked(OopOrNarrowOopStar from) const;
   59.23  
   59.24    void clear();
   59.25  
   59.26 @@ -192,10 +192,10 @@
   59.27    // Unused unless G1RecordHRRSOops is true.
   59.28  
   59.29    static const int MaxRecorded = 1000000;
   59.30 -  static oop**        _recorded_oops;
   59.31 -  static HeapWord**   _recorded_cards;
   59.32 -  static HeapRegion** _recorded_regions;
   59.33 -  static int          _n_recorded;
   59.34 +  static OopOrNarrowOopStar* _recorded_oops;
   59.35 +  static HeapWord**          _recorded_cards;
   59.36 +  static HeapRegion**        _recorded_regions;
   59.37 +  static int                 _n_recorded;
   59.38  
   59.39    static const int MaxRecordedEvents = 1000;
   59.40    static Event*       _recorded_events;
   59.41 @@ -231,13 +231,13 @@
   59.42  
   59.43    /* Used in the sequential case.  Returns "true" iff this addition causes
   59.44       the size limit to be reached. */
   59.45 -  void add_reference(oop* from) {
   59.46 +  void add_reference(OopOrNarrowOopStar from) {
   59.47      _other_regions.add_reference(from);
   59.48    }
   59.49  
   59.50    /* Used in the parallel case.  Returns "true" iff this addition causes
   59.51       the size limit to be reached. */
   59.52 -  void add_reference(oop* from, int tid) {
   59.53 +  void add_reference(OopOrNarrowOopStar from, int tid) {
   59.54      _other_regions.add_reference(from, tid);
   59.55    }
   59.56  
   59.57 @@ -301,7 +301,7 @@
   59.58      return OtherRegionsTable::fl_mem_size();
   59.59    }
   59.60  
   59.61 -  bool contains_reference(oop* from) const {
   59.62 +  bool contains_reference(OopOrNarrowOopStar from) const {
   59.63      return _other_regions.contains_reference(from);
   59.64    }
   59.65    void print() const;
   59.66 @@ -329,7 +329,7 @@
   59.67    }
   59.68  #endif
   59.69  
   59.70 -  static void record(HeapRegion* hr, oop* f);
   59.71 +  static void record(HeapRegion* hr, OopOrNarrowOopStar f);
   59.72    static void print_recorded();
   59.73    static void record_event(Event evnt);
   59.74

    60.1 --- a/src/share/vm/gc_implementation/g1/satbQueue.cpp	Mon Jul 27 09:06:22 2009 -0700
    60.2 +++ b/src/share/vm/gc_implementation/g1/satbQueue.cpp	Mon Jul 27 17:23:52 2009 -0400
    60.3 @@ -43,6 +43,18 @@
    60.4      }
    60.5    }
    60.6  }
    60.7 +
    60.8 +#ifdef ASSERT
    60.9 +void ObjPtrQueue::verify_oops_in_buffer() {
   60.10 +  if (_buf == NULL) return;
   60.11 +  for (size_t i = _index; i < _sz; i += oopSize) {
   60.12 +    oop obj = (oop)_buf[byte_index_to_index((int)i)];
   60.13 +    assert(obj != NULL && obj->is_oop(true /* ignore mark word */),
   60.14 +           "Not an oop");
   60.15 +  }
   60.16 +}
   60.17 +#endif
   60.18 +
   60.19  #ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away
   60.20  #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
   60.21  #endif // _MSC_VER
   60.22 @@ -66,6 +78,7 @@
   60.23  
   60.24  
   60.25  void SATBMarkQueueSet::handle_zero_index_for_thread(JavaThread* t) {
   60.26 +  DEBUG_ONLY(t->satb_mark_queue().verify_oops_in_buffer();)
   60.27    t->satb_mark_queue().handle_zero_index();
   60.28  }
   60.29  
   60.30 @@ -143,7 +156,7 @@
   60.31      }
   60.32      _completed_buffers_tail = NULL;
   60.33      _n_completed_buffers = 0;
   60.34 -    debug_only(assert_completed_buffer_list_len_correct_locked());
   60.35 +    DEBUG_ONLY(assert_completed_buffer_list_len_correct_locked());
   60.36    }
   60.37    while (buffers_to_delete != NULL) {
   60.38      CompletedBufferNode* nd = buffers_to_delete;

    61.1 --- a/src/share/vm/gc_implementation/g1/satbQueue.hpp	Mon Jul 27 09:06:22 2009 -0700
    61.2 +++ b/src/share/vm/gc_implementation/g1/satbQueue.hpp	Mon Jul 27 17:23:52 2009 -0400
    61.3 @@ -39,6 +39,7 @@
    61.4    static void apply_closure_to_buffer(ObjectClosure* cl,
    61.5                                        void** buf, size_t index, size_t sz);
    61.6  
    61.7 +  void verify_oops_in_buffer() NOT_DEBUG_RETURN;
    61.8  };
    61.9  
   61.10

    62.1 --- a/src/share/vm/gc_implementation/includeDB_gc_g1	Mon Jul 27 09:06:22 2009 -0700
    62.2 +++ b/src/share/vm/gc_implementation/includeDB_gc_g1	Mon Jul 27 17:23:52 2009 -0400
    62.3 @@ -27,6 +27,7 @@
    62.4  bufferingOopClosure.hpp			genOopClosures.hpp
    62.5  bufferingOopClosure.hpp			generation.hpp
    62.6  bufferingOopClosure.hpp			os.hpp
    62.7 +bufferingOopClosure.hpp			taskqueue.hpp
    62.8  
    62.9  cardTableRS.cpp				concurrentMark.hpp
   62.10  cardTableRS.cpp				g1SATBCardTableModRefBS.hpp
   62.11 @@ -139,7 +140,7 @@
   62.12  g1CollectedHeap.cpp                     g1CollectedHeap.inline.hpp
   62.13  g1CollectedHeap.cpp                     g1CollectorPolicy.hpp
   62.14  g1CollectedHeap.cpp                     g1MarkSweep.hpp
   62.15 -g1CollectedHeap.cpp                     g1RemSet.hpp
   62.16 +g1CollectedHeap.cpp                     g1RemSet.inline.hpp
   62.17  g1CollectedHeap.cpp                     g1OopClosures.inline.hpp
   62.18  g1CollectedHeap.cpp                     genOopClosures.inline.hpp
   62.19  g1CollectedHeap.cpp                     gcLocker.inline.hpp
   62.20 @@ -151,13 +152,14 @@
   62.21  g1CollectedHeap.cpp                     isGCActiveMark.hpp
   62.22  g1CollectedHeap.cpp			oop.inline.hpp
   62.23  g1CollectedHeap.cpp			oop.pcgc.inline.hpp
   62.24 -g1CollectedHeap.cpp			parGCAllocBuffer.hpp
   62.25  g1CollectedHeap.cpp                     vm_operations_g1.hpp
   62.26  g1CollectedHeap.cpp                     vmThread.hpp
   62.27  
   62.28  g1CollectedHeap.hpp                     barrierSet.hpp
   62.29 +g1CollectedHeap.hpp                     g1RemSet.hpp
   62.30  g1CollectedHeap.hpp                     heapRegion.hpp
   62.31  g1CollectedHeap.hpp                     memRegion.hpp
   62.32 +g1CollectedHeap.hpp			parGCAllocBuffer.hpp
   62.33  g1CollectedHeap.hpp                     sharedHeap.hpp
   62.34  
   62.35  g1CollectedHeap.inline.hpp              concurrentMark.hpp
   62.36 @@ -245,6 +247,7 @@
   62.37  g1RemSet.cpp				iterator.hpp
   62.38  g1RemSet.cpp				oop.inline.hpp
   62.39  
   62.40 +g1RemSet.inline.hpp			oop.inline.hpp
   62.41  g1RemSet.inline.hpp			g1RemSet.hpp
   62.42  g1RemSet.inline.hpp			heapRegionRemSet.hpp
   62.43  
   62.44 @@ -255,6 +258,7 @@
   62.45  g1SATBCardTableModRefBS.cpp		thread_<os_family>.inline.hpp
   62.46  g1SATBCardTableModRefBS.cpp		satbQueue.hpp
   62.47  
   62.48 +g1SATBCardTableModRefBS.hpp		oop.inline.hpp
   62.49  g1SATBCardTableModRefBS.hpp		cardTableModRefBS.hpp
   62.50  g1SATBCardTableModRefBS.hpp		memRegion.hpp
   62.51

    63.1 --- a/src/share/vm/gc_implementation/parNew/parCardTableModRefBS.cpp	Mon Jul 27 09:06:22 2009 -0700
    63.2 +++ b/src/share/vm/gc_implementation/parNew/parCardTableModRefBS.cpp	Mon Jul 27 17:23:52 2009 -0400
    63.3 @@ -31,9 +31,10 @@
    63.4                                                          bool clear,
    63.5                                                          int n_threads) {
    63.6    if (n_threads > 0) {
    63.7 -    assert(n_threads == (int)ParallelGCThreads, "# worker threads != # requested!");
    63.8 -
    63.9 -      // Make sure the LNC array is valid for the space.
   63.10 +    assert((n_threads == 1 && ParallelGCThreads == 0) ||
   63.11 +           n_threads <= (int)ParallelGCThreads,
   63.12 +           "# worker threads != # requested!");
   63.13 +    // Make sure the LNC array is valid for the space.
   63.14      jbyte**   lowest_non_clean;
   63.15      uintptr_t lowest_non_clean_base_chunk_index;
   63.16      size_t    lowest_non_clean_chunk_size;

    64.1 --- a/src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.cpp	Mon Jul 27 09:06:22 2009 -0700
    64.2 +++ b/src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.cpp	Mon Jul 27 17:23:52 2009 -0400
    64.3 @@ -885,7 +885,7 @@
    64.4  }
    64.5  
    64.6  
    64.7 -void ParallelScavengeHeap::verify(bool allow_dirty, bool silent) {
    64.8 +void ParallelScavengeHeap::verify(bool allow_dirty, bool silent, bool option /* ignored */) {
    64.9    // Why do we need the total_collections()-filter below?
   64.10    if (total_collections() > 0) {
   64.11      if (!silent) {

    65.1 --- a/src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.hpp	Mon Jul 27 09:06:22 2009 -0700
    65.2 +++ b/src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.hpp	Mon Jul 27 17:23:52 2009 -0400
    65.3 @@ -217,7 +217,7 @@
    65.4    virtual void gc_threads_do(ThreadClosure* tc) const;
    65.5    virtual void print_tracing_info() const;
    65.6  
    65.7 -  void verify(bool allow_dirty, bool silent);
    65.8 +  void verify(bool allow_dirty, bool silent, bool /* option */);
    65.9  
   65.10    void print_heap_change(size_t prev_used);
   65.11

    66.1 --- a/src/share/vm/gc_implementation/parallelScavenge/psPromotionManager.inline.hpp	Mon Jul 27 09:06:22 2009 -0700
    66.2 +++ b/src/share/vm/gc_implementation/parallelScavenge/psPromotionManager.inline.hpp	Mon Jul 27 17:23:52 2009 -0400
    66.3 @@ -117,6 +117,7 @@
    66.4      process_array_chunk(old);
    66.5    } else {
    66.6      if (p.is_narrow()) {
    66.7 +      assert(UseCompressedOops, "Error");
    66.8        PSScavenge::copy_and_push_safe_barrier(this, (narrowOop*)p);
    66.9      } else {
   66.10        PSScavenge::copy_and_push_safe_barrier(this, (oop*)p);

    67.1 --- a/src/share/vm/gc_interface/collectedHeap.hpp	Mon Jul 27 09:06:22 2009 -0700
    67.2 +++ b/src/share/vm/gc_interface/collectedHeap.hpp	Mon Jul 27 17:23:52 2009 -0400
    67.3 @@ -533,7 +533,7 @@
    67.4    virtual void print_tracing_info() const = 0;
    67.5  
    67.6    // Heap verification
    67.7 -  virtual void verify(bool allow_dirty, bool silent) = 0;
    67.8 +  virtual void verify(bool allow_dirty, bool silent, bool option) = 0;
    67.9  
   67.10    // Non product verification and debugging.
   67.11  #ifndef PRODUCT

    68.1 --- a/src/share/vm/includeDB_core	Mon Jul 27 09:06:22 2009 -0700
    68.2 +++ b/src/share/vm/includeDB_core	Mon Jul 27 17:23:52 2009 -0400
    68.3 @@ -554,7 +554,6 @@
    68.4  ciEnv.cpp                               linkResolver.hpp
    68.5  ciEnv.cpp                               methodDataOop.hpp
    68.6  ciEnv.cpp                               objArrayKlass.hpp
    68.7 -ciEnv.cpp                               oop.hpp
    68.8  ciEnv.cpp                               oop.inline.hpp
    68.9  ciEnv.cpp                               oop.inline2.hpp
   68.10  ciEnv.cpp                               oopFactory.hpp
   68.11 @@ -785,7 +784,6 @@
   68.12  ciSignature.cpp                         allocation.inline.hpp
   68.13  ciSignature.cpp                         ciSignature.hpp
   68.14  ciSignature.cpp                         ciUtilities.hpp
   68.15 -ciSignature.cpp                         oop.hpp
   68.16  ciSignature.cpp                         oop.inline.hpp
   68.17  ciSignature.cpp                         signature.hpp
   68.18  
   68.19 @@ -952,7 +950,6 @@
   68.20  classify.cpp                            classify.hpp
   68.21  classify.cpp                            systemDictionary.hpp
   68.22  
   68.23 -classify.hpp                            oop.hpp
   68.24  classify.hpp                            oop.inline.hpp
   68.25  
   68.26  codeBlob.cpp                            allocation.inline.hpp
   68.27 @@ -1187,7 +1184,6 @@
   68.28  compilerOracle.cpp                      jniHandles.hpp
   68.29  compilerOracle.cpp                      klass.hpp
   68.30  compilerOracle.cpp                      methodOop.hpp
   68.31 -compilerOracle.cpp                      oop.hpp
   68.32  compilerOracle.cpp                      oop.inline.hpp
   68.33  compilerOracle.cpp                      oopFactory.hpp
   68.34  compilerOracle.cpp                      resourceArea.hpp
   68.35 @@ -1631,7 +1627,6 @@
   68.36  frame.cpp                               methodOop.hpp
   68.37  frame.cpp                               monitorChunk.hpp
   68.38  frame.cpp                               nativeInst_<arch>.hpp
   68.39 -frame.cpp                               oop.hpp
   68.40  frame.cpp                               oop.inline.hpp
   68.41  frame.cpp                               oop.inline2.hpp
   68.42  frame.cpp                               oopMapCache.hpp
   68.43 @@ -1799,7 +1794,6 @@
   68.44  generation.cpp                          generation.hpp
   68.45  generation.cpp                          generation.inline.hpp
   68.46  generation.cpp                          java.hpp
   68.47 -generation.cpp                          oop.hpp
   68.48  generation.cpp                          oop.inline.hpp
   68.49  generation.cpp                          spaceDecorator.hpp
   68.50  generation.cpp                          space.inline.hpp
   68.51 @@ -2272,7 +2266,6 @@
   68.52  java.cpp                                memprofiler.hpp
   68.53  java.cpp                                methodOop.hpp
   68.54  java.cpp                                objArrayOop.hpp
   68.55 -java.cpp                                oop.hpp
   68.56  java.cpp                                oop.inline.hpp
   68.57  java.cpp                                oopFactory.hpp
   68.58  java.cpp                                sharedRuntime.hpp
   68.59 @@ -2949,7 +2942,7 @@
   68.60  nativeInst_<arch>.cpp                   assembler_<arch>.inline.hpp
   68.61  nativeInst_<arch>.cpp                   handles.hpp
   68.62  nativeInst_<arch>.cpp                   nativeInst_<arch>.hpp
   68.63 -nativeInst_<arch>.cpp                   oop.hpp
   68.64 +nativeInst_<arch>.cpp                   oop.inline.hpp
   68.65  nativeInst_<arch>.cpp                   ostream.hpp
   68.66  nativeInst_<arch>.cpp                   resourceArea.hpp
   68.67  nativeInst_<arch>.cpp                   sharedRuntime.hpp
   68.68 @@ -3844,7 +3837,7 @@
   68.69  stackValue.cpp                          debugInfo.hpp
   68.70  stackValue.cpp                          frame.inline.hpp
   68.71  stackValue.cpp                          handles.inline.hpp
   68.72 -stackValue.cpp                          oop.hpp
   68.73 +stackValue.cpp                          oop.inline.hpp
   68.74  stackValue.cpp                          stackValue.hpp
   68.75  
   68.76  stackValue.hpp                          handles.hpp
   68.77 @@ -4332,7 +4325,6 @@
   68.78  unhandledOops.cpp                       collectedHeap.hpp
   68.79  unhandledOops.cpp                       gcLocker.inline.hpp
   68.80  unhandledOops.cpp                       globalDefinitions.hpp
   68.81 -unhandledOops.cpp                       oop.hpp
   68.82  unhandledOops.cpp                       oop.inline.hpp
   68.83  unhandledOops.cpp                       thread.hpp
   68.84  unhandledOops.cpp                       unhandledOops.hpp
   68.85 @@ -4468,7 +4460,6 @@
   68.86  vframe.cpp                              nmethod.hpp
   68.87  vframe.cpp                              objectMonitor.hpp
   68.88  vframe.cpp                              objectMonitor.inline.hpp
   68.89 -vframe.cpp                              oop.hpp
   68.90  vframe.cpp                              oop.inline.hpp
   68.91  vframe.cpp                              oopMapCache.hpp
   68.92  vframe.cpp                              pcDesc.hpp
   68.93 @@ -4580,7 +4571,6 @@
   68.94  vmThread.cpp                            interfaceSupport.hpp
   68.95  vmThread.cpp                            methodOop.hpp
   68.96  vmThread.cpp                            mutexLocker.hpp
   68.97 -vmThread.cpp                            oop.hpp
   68.98  vmThread.cpp                            oop.inline.hpp
   68.99  vmThread.cpp                            os.hpp
  68.100  vmThread.cpp                            resourceArea.hpp

    69.1 --- a/src/share/vm/includeDB_features	Mon Jul 27 09:06:22 2009 -0700
    69.2 +++ b/src/share/vm/includeDB_features	Mon Jul 27 17:23:52 2009 -0400
    69.3 @@ -47,7 +47,7 @@
    69.4  dump.cpp                                javaClasses.hpp
    69.5  dump.cpp                                loaderConstraints.hpp
    69.6  dump.cpp                                methodDataOop.hpp
    69.7 -dump.cpp                                oop.hpp
    69.8 +dump.cpp                                oop.inline.hpp
    69.9  dump.cpp                                oopFactory.hpp
   69.10  dump.cpp                                resourceArea.hpp
   69.11  dump.cpp                                signature.hpp
   69.12 @@ -237,7 +237,7 @@
   69.13  serialize.cpp                           compiledICHolderOop.hpp
   69.14  serialize.cpp                           methodDataOop.hpp
   69.15  serialize.cpp                           objArrayOop.hpp
   69.16 -serialize.cpp                           oop.hpp
   69.17 +serialize.cpp                           oop.inline.hpp
   69.18  serialize.cpp                           symbolTable.hpp
   69.19  serialize.cpp                           systemDictionary.hpp
   69.20  
   69.21 @@ -295,7 +295,7 @@
   69.22  vmStructs.cpp                           objArrayKlass.hpp
   69.23  vmStructs.cpp                           objArrayKlassKlass.hpp
   69.24  vmStructs.cpp                           objArrayOop.hpp
   69.25 -vmStructs.cpp                           oop.hpp
   69.26 +vmStructs.cpp                           oop.inline.hpp
   69.27  vmStructs.cpp                           oopMap.hpp
   69.28  vmStructs.cpp                           pcDesc.hpp
   69.29  vmStructs.cpp                           perfMemory.hpp

    70.1 --- a/src/share/vm/interpreter/rewriter.cpp	Mon Jul 27 09:06:22 2009 -0700
    70.2 +++ b/src/share/vm/interpreter/rewriter.cpp	Mon Jul 27 17:23:52 2009 -0400
    70.3 @@ -273,6 +273,7 @@
    70.4    compute_index_maps();
    70.5  
    70.6    if (RegisterFinalizersAtInit && _klass->name() == vmSymbols::java_lang_Object()) {
    70.7 +    bool did_rewrite = false;
    70.8      int i = _methods->length();
    70.9      while (i-- > 0) {
   70.10        methodOop method = (methodOop)_methods->obj_at(i);
   70.11 @@ -281,9 +282,11 @@
   70.12          // object for finalization if needed.
   70.13          methodHandle m(THREAD, method);
   70.14          rewrite_Object_init(m, CHECK);
   70.15 +        did_rewrite = true;
   70.16          break;
   70.17        }
   70.18      }
   70.19 +    assert(did_rewrite, "must find Object::<init> to rewrite it");
   70.20    }
   70.21  
   70.22    // rewrite methods, in two passes

    71.1 --- a/src/share/vm/memory/barrierSet.cpp	Mon Jul 27 09:06:22 2009 -0700
    71.2 +++ b/src/share/vm/memory/barrierSet.cpp	Mon Jul 27 17:23:52 2009 -0400
    71.3 @@ -25,12 +25,27 @@
    71.4  # include "incls/_precompiled.incl"
    71.5  # include "incls/_barrierSet.cpp.incl"
    71.6  
    71.7 -// count is in HeapWord's
    71.8 +// count is number of array elements being written
    71.9  void BarrierSet::static_write_ref_array_pre(HeapWord* start, size_t count) {
   71.10 -   Universe::heap()->barrier_set()->write_ref_array_pre(MemRegion(start, start + count));
   71.11 +  assert(count <= (size_t)max_intx, "count too large");
   71.12 +#if 0
   71.13 +  warning("Pre: \t" INTPTR_FORMAT "[" SIZE_FORMAT "]\t",
   71.14 +                   start,            count);
   71.15 +#endif
   71.16 +  if (UseCompressedOops) {
   71.17 +    Universe::heap()->barrier_set()->write_ref_array_pre((narrowOop*)start, (int)count);
   71.18 +  } else {
   71.19 +    Universe::heap()->barrier_set()->write_ref_array_pre(      (oop*)start, (int)count);
   71.20 +  }
   71.21  }
   71.22  
   71.23 -// count is in HeapWord's
   71.24 +// count is number of array elements being written
   71.25  void BarrierSet::static_write_ref_array_post(HeapWord* start, size_t count) {
   71.26 -   Universe::heap()->barrier_set()->write_ref_array_work(MemRegion(start, start + count));
   71.27 +  assert(count <= (size_t)max_intx, "count too large");
   71.28 +  HeapWord* end = start + objArrayOopDesc::array_size((int)count);
   71.29 +#if 0
   71.30 +  warning("Post:\t" INTPTR_FORMAT "[" SIZE_FORMAT "] : [" INTPTR_FORMAT","INTPTR_FORMAT")\t",
   71.31 +                   start,            count,              start,          end);
   71.32 +#endif
   71.33 +  Universe::heap()->barrier_set()->write_ref_array_work(MemRegion(start, end));
   71.34  }

    72.1 --- a/src/share/vm/memory/barrierSet.hpp	Mon Jul 27 09:06:22 2009 -0700
    72.2 +++ b/src/share/vm/memory/barrierSet.hpp	Mon Jul 27 17:23:52 2009 -0400
    72.3 @@ -81,9 +81,13 @@
    72.4    // barrier types.  Semantically, it should be thought of as a call to the
    72.5    // virtual "_work" function below, which must implement the barrier.)
    72.6    // First the pre-write versions...
    72.7 -  inline void write_ref_field_pre(void* field, oop new_val);
    72.8 +  template <class T> inline void write_ref_field_pre(T* field, oop new_val);
    72.9 +private:
   72.10 +  // Keep this private so as to catch violations at build time.
   72.11 +  virtual void write_ref_field_pre_work(     void* field, oop new_val) { guarantee(false, "Not needed"); };
   72.12  protected:
   72.13 -  virtual void write_ref_field_pre_work(void* field, oop new_val) {};
   72.14 +  virtual void write_ref_field_pre_work(      oop* field, oop new_val) {};
   72.15 +  virtual void write_ref_field_pre_work(narrowOop* field, oop new_val) {};
   72.16  public:
   72.17  
   72.18    // ...then the post-write version.
   72.19 @@ -117,12 +121,17 @@
   72.20    virtual void read_ref_array(MemRegion mr) = 0;
   72.21    virtual void read_prim_array(MemRegion mr) = 0;
   72.22  
   72.23 -  virtual void write_ref_array_pre(MemRegion mr) {}
   72.24 +  virtual void write_ref_array_pre(      oop* dst, int length) {}
   72.25 +  virtual void write_ref_array_pre(narrowOop* dst, int length) {}
   72.26    inline void write_ref_array(MemRegion mr);
   72.27  
   72.28    // Static versions, suitable for calling from generated code.
   72.29    static void static_write_ref_array_pre(HeapWord* start, size_t count);
   72.30    static void static_write_ref_array_post(HeapWord* start, size_t count);
   72.31 +  // Narrow oop versions of the above; count is # of array elements being written,
   72.32 +  // starting with "start", which is HeapWord-aligned.
   72.33 +  static void static_write_ref_array_pre_narrow(HeapWord* start, size_t count);
   72.34 +  static void static_write_ref_array_post_narrow(HeapWord* start, size_t count);
   72.35  
   72.36  protected:
   72.37    virtual void write_ref_array_work(MemRegion mr) = 0;

    73.1 --- a/src/share/vm/memory/barrierSet.inline.hpp	Mon Jul 27 09:06:22 2009 -0700
    73.2 +++ b/src/share/vm/memory/barrierSet.inline.hpp	Mon Jul 27 17:23:52 2009 -0400
    73.3 @@ -23,10 +23,10 @@
    73.4   */
    73.5  
    73.6  // Inline functions of BarrierSet, which de-virtualize certain
    73.7 -// performance-critical calls when when the barrier is the most common
    73.8 +// performance-critical calls when the barrier is the most common
    73.9  // card-table kind.
   73.10  
   73.11 -void BarrierSet::write_ref_field_pre(void* field, oop new_val) {
   73.12 +template <class T> void BarrierSet::write_ref_field_pre(T* field, oop new_val) {
   73.13    if (kind() == CardTableModRef) {
   73.14      ((CardTableModRefBS*)this)->inline_write_ref_field_pre(field, new_val);
   73.15    } else {

    74.1 --- a/src/share/vm/memory/cardTableModRefBS.hpp	Mon Jul 27 09:06:22 2009 -0700
    74.2 +++ b/src/share/vm/memory/cardTableModRefBS.hpp	Mon Jul 27 17:23:52 2009 -0400
    74.3 @@ -287,7 +287,7 @@
    74.4    // these functions here for performance.
    74.5  protected:
    74.6    void write_ref_field_work(oop obj, size_t offset, oop newVal);
    74.7 -  void write_ref_field_work(void* field, oop newVal);
    74.8 +  virtual void write_ref_field_work(void* field, oop newVal);
    74.9  public:
   74.10  
   74.11    bool has_write_ref_array_opt() { return true; }
   74.12 @@ -317,10 +317,10 @@
   74.13  
   74.14    // *** Card-table-barrier-specific things.
   74.15  
   74.16 -  inline void inline_write_ref_field_pre(void* field, oop newVal) {}
   74.17 +  template <class T> inline void inline_write_ref_field_pre(T* field, oop newVal) {}
   74.18  
   74.19 -  inline void inline_write_ref_field(void* field, oop newVal) {
   74.20 -    jbyte* byte = byte_for(field);
   74.21 +  template <class T> inline void inline_write_ref_field(T* field, oop newVal) {
   74.22 +    jbyte* byte = byte_for((void*)field);
   74.23      *byte = dirty_card;
   74.24    }
   74.25

    75.1 --- a/src/share/vm/memory/genCollectedHeap.cpp	Mon Jul 27 09:06:22 2009 -0700
    75.2 +++ b/src/share/vm/memory/genCollectedHeap.cpp	Mon Jul 27 17:23:52 2009 -0400
    75.3 @@ -1194,7 +1194,7 @@
    75.4    return _gens[level]->gc_stats();
    75.5  }
    75.6  
    75.7 -void GenCollectedHeap::verify(bool allow_dirty, bool silent) {
    75.8 +void GenCollectedHeap::verify(bool allow_dirty, bool silent, bool option /* ignored */) {
    75.9    if (!silent) {
   75.10      gclog_or_tty->print("permgen ");
   75.11    }

    76.1 --- a/src/share/vm/memory/genCollectedHeap.hpp	Mon Jul 27 09:06:22 2009 -0700
    76.2 +++ b/src/share/vm/memory/genCollectedHeap.hpp	Mon Jul 27 17:23:52 2009 -0400
    76.3 @@ -325,7 +325,7 @@
    76.4    void prepare_for_verify();
    76.5  
    76.6    // Override.
    76.7 -  void verify(bool allow_dirty, bool silent);
    76.8 +  void verify(bool allow_dirty, bool silent, bool /* option */);
    76.9  
   76.10    // Override.
   76.11    void print() const;

    77.1 --- a/src/share/vm/memory/genOopClosures.hpp	Mon Jul 27 09:06:22 2009 -0700
    77.2 +++ b/src/share/vm/memory/genOopClosures.hpp	Mon Jul 27 17:23:52 2009 -0400
    77.3 @@ -57,7 +57,7 @@
    77.4    template <class T> void do_barrier(T* p);
    77.5  
    77.6    // Version for use by closures that may be called in parallel code.
    77.7 -  void par_do_barrier(oop* p);
    77.8 +  template <class T> void par_do_barrier(T* p);
    77.9  
   77.10   public:
   77.11    OopsInGenClosure() : OopClosure(NULL),

    78.1 --- a/src/share/vm/memory/genOopClosures.inline.hpp	Mon Jul 27 09:06:22 2009 -0700
    78.2 +++ b/src/share/vm/memory/genOopClosures.inline.hpp	Mon Jul 27 17:23:52 2009 -0400
    78.3 @@ -40,18 +40,20 @@
    78.4  
    78.5  template <class T> inline void OopsInGenClosure::do_barrier(T* p) {
    78.6    assert(generation()->is_in_reserved(p), "expected ref in generation");
    78.7 -  assert(!oopDesc::is_null(*p), "expected non-null object");
    78.8 -  oop obj = oopDesc::load_decode_heap_oop_not_null(p);
    78.9 +  T heap_oop = oopDesc::load_heap_oop(p);
   78.10 +  assert(!oopDesc::is_null(heap_oop), "expected non-null oop");
   78.11 +  oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
   78.12    // If p points to a younger generation, mark the card.
   78.13    if ((HeapWord*)obj < _gen_boundary) {
   78.14      _rs->inline_write_ref_field_gc(p, obj);
   78.15    }
   78.16  }
   78.17  
   78.18 -inline void OopsInGenClosure::par_do_barrier(oop* p) {
   78.19 +template <class T> inline void OopsInGenClosure::par_do_barrier(T* p) {
   78.20    assert(generation()->is_in_reserved(p), "expected ref in generation");
   78.21 -  oop obj = *p;
   78.22 -  assert(obj != NULL, "expected non-null object");
   78.23 +  T heap_oop = oopDesc::load_heap_oop(p);
   78.24 +  assert(!oopDesc::is_null(heap_oop), "expected non-null oop");
   78.25 +  oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
   78.26    // If p points to a younger generation, mark the card.
   78.27    if ((HeapWord*)obj < gen_boundary()) {
   78.28      rs()->write_ref_field_gc_par(p, obj);

    79.1 --- a/src/share/vm/memory/referenceProcessor.cpp	Mon Jul 27 09:06:22 2009 -0700
    79.2 +++ b/src/share/vm/memory/referenceProcessor.cpp	Mon Jul 27 17:23:52 2009 -0400
    79.3 @@ -1013,12 +1013,19 @@
    79.4    // discovered_addr.
    79.5    oop current_head = refs_list.head();
    79.6  
    79.7 -  // Note: In the case of G1, this pre-barrier is strictly
    79.8 +  // Note: In the case of G1, this specific pre-barrier is strictly
    79.9    // not necessary because the only case we are interested in
   79.10 -  // here is when *discovered_addr is NULL, so this will expand to
   79.11 -  // nothing. As a result, I am just manually eliding this out for G1.
   79.12 +  // here is when *discovered_addr is NULL (see the CAS further below),
   79.13 +  // so this will expand to nothing. As a result, we have manually
   79.14 +  // elided this out for G1, but left in the test for some future
   79.15 +  // collector that might have need for a pre-barrier here.
   79.16    if (_discovered_list_needs_barrier && !UseG1GC) {
   79.17 -    _bs->write_ref_field_pre((void*)discovered_addr, current_head); guarantee(false, "Needs to be fixed: YSR");
   79.18 +    if (UseCompressedOops) {
   79.19 +      _bs->write_ref_field_pre((narrowOop*)discovered_addr, current_head);
   79.20 +    } else {
   79.21 +      _bs->write_ref_field_pre((oop*)discovered_addr, current_head);
   79.22 +    }
   79.23 +    guarantee(false, "Need to check non-G1 collector");
   79.24    }
   79.25    oop retest = oopDesc::atomic_compare_exchange_oop(current_head, discovered_addr,
   79.26                                                      NULL);
   79.27 @@ -1029,9 +1036,8 @@
   79.28      refs_list.set_head(obj);
   79.29      refs_list.inc_length(1);
   79.30      if (_discovered_list_needs_barrier) {
   79.31 -      _bs->write_ref_field((void*)discovered_addr, current_head); guarantee(false, "Needs to be fixed: YSR");
   79.32 +      _bs->write_ref_field((void*)discovered_addr, current_head);
   79.33      }
   79.34 -
   79.35    } else {
   79.36      // If retest was non NULL, another thread beat us to it:
   79.37      // The reference has already been discovered...
   79.38 @@ -1177,11 +1183,16 @@
   79.39      // pre-value, we can safely elide the pre-barrier here for the case of G1.
   79.40      assert(discovered == NULL, "control point invariant");
   79.41      if (_discovered_list_needs_barrier && !UseG1GC) { // safe to elide for G1
   79.42 -      _bs->write_ref_field_pre((oop*)discovered_addr, current_head);
   79.43 +      if (UseCompressedOops) {
   79.44 +        _bs->write_ref_field_pre((narrowOop*)discovered_addr, current_head);
   79.45 +      } else {
   79.46 +        _bs->write_ref_field_pre((oop*)discovered_addr, current_head);
   79.47 +      }
   79.48 +      guarantee(false, "Need to check non-G1 collector");
   79.49      }
   79.50      oop_store_raw(discovered_addr, current_head);
   79.51      if (_discovered_list_needs_barrier) {
   79.52 -      _bs->write_ref_field((oop*)discovered_addr, current_head);
   79.53 +      _bs->write_ref_field((void*)discovered_addr, current_head);
   79.54      }
   79.55      list->set_head(obj);
   79.56      list->inc_length(1);

    80.1 --- a/src/share/vm/memory/space.hpp	Mon Jul 27 09:06:22 2009 -0700
    80.2 +++ b/src/share/vm/memory/space.hpp	Mon Jul 27 17:23:52 2009 -0400
    80.3 @@ -106,6 +106,7 @@
    80.4    virtual void set_end(HeapWord* value)    { _end = value; }
    80.5  
    80.6    virtual HeapWord* saved_mark_word() const  { return _saved_mark_word; }
    80.7 +
    80.8    void set_saved_mark_word(HeapWord* p) { _saved_mark_word = p; }
    80.9  
   80.10    MemRegionClosure* preconsumptionDirtyCardClosure() const {

    81.1 --- a/src/share/vm/memory/universe.cpp	Mon Jul 27 09:06:22 2009 -0700
    81.2 +++ b/src/share/vm/memory/universe.cpp	Mon Jul 27 17:23:52 2009 -0400
    81.3 @@ -1170,7 +1170,7 @@
    81.4    st->print_cr("}");
    81.5  }
    81.6  
    81.7 -void Universe::verify(bool allow_dirty, bool silent) {
    81.8 +void Universe::verify(bool allow_dirty, bool silent, bool option) {
    81.9    if (SharedSkipVerify) {
   81.10      return;
   81.11    }
   81.12 @@ -1194,7 +1194,7 @@
   81.13    if (!silent) gclog_or_tty->print("[Verifying ");
   81.14    if (!silent) gclog_or_tty->print("threads ");
   81.15    Threads::verify();
   81.16 -  heap()->verify(allow_dirty, silent);
   81.17 +  heap()->verify(allow_dirty, silent, option);
   81.18  
   81.19    if (!silent) gclog_or_tty->print("syms ");
   81.20    SymbolTable::verify();

    82.1 --- a/src/share/vm/memory/universe.hpp	Mon Jul 27 09:06:22 2009 -0700
    82.2 +++ b/src/share/vm/memory/universe.hpp	Mon Jul 27 17:23:52 2009 -0400
    82.3 @@ -343,6 +343,7 @@
    82.4    // For UseCompressedOops
    82.5    static address* narrow_oop_base_addr()              { return &_narrow_oop._base; }
    82.6    static address  narrow_oop_base()                   { return  _narrow_oop._base; }
    82.7 +  static bool  is_narrow_oop_base(void* addr)         { return (narrow_oop_base() == (address)addr); }
    82.8    static int      narrow_oop_shift()                  { return  _narrow_oop._shift; }
    82.9    static void     set_narrow_oop_base(address base)   { _narrow_oop._base  = base; }
   82.10    static void     set_narrow_oop_shift(int shift)     { _narrow_oop._shift = shift; }
   82.11 @@ -398,7 +399,7 @@
   82.12  
   82.13    // Debugging
   82.14    static bool verify_in_progress() { return _verify_in_progress; }
   82.15 -  static void verify(bool allow_dirty = true, bool silent = false);
   82.16 +  static void verify(bool allow_dirty = true, bool silent = false, bool option = true);
   82.17    static int  verify_count()                  { return _verify_count; }
   82.18    static void print();
   82.19    static void print_on(outputStream* st);

    83.1 --- a/src/share/vm/oops/instanceRefKlass.cpp	Mon Jul 27 09:06:22 2009 -0700
    83.2 +++ b/src/share/vm/oops/instanceRefKlass.cpp	Mon Jul 27 17:23:52 2009 -0400
    83.3 @@ -28,13 +28,14 @@
    83.4  template <class T>
    83.5  static void specialized_oop_follow_contents(instanceRefKlass* ref, oop obj) {
    83.6    T* referent_addr = (T*)java_lang_ref_Reference::referent_addr(obj);
    83.7 -  oop referent = oopDesc::load_decode_heap_oop(referent_addr);
    83.8 +  T heap_oop = oopDesc::load_heap_oop(referent_addr);
    83.9    debug_only(
   83.10      if(TraceReferenceGC && PrintGCDetails) {
   83.11        gclog_or_tty->print_cr("instanceRefKlass::oop_follow_contents " INTPTR_FORMAT, obj);
   83.12      }
   83.13    )
   83.14 -  if (referent != NULL) {
   83.15 +  if (!oopDesc::is_null(heap_oop)) {
   83.16 +    oop referent = oopDesc::decode_heap_oop_not_null(heap_oop);
   83.17      if (!referent->is_gc_marked() &&
   83.18          MarkSweep::ref_processor()->
   83.19            discover_reference(obj, ref->reference_type())) {
   83.20 @@ -81,13 +82,14 @@
   83.21                                              ParCompactionManager* cm,
   83.22                                              oop obj) {
   83.23    T* referent_addr = (T*)java_lang_ref_Reference::referent_addr(obj);
   83.24 -  oop referent = oopDesc::load_decode_heap_oop(referent_addr);
   83.25 +  T heap_oop = oopDesc::load_heap_oop(referent_addr);
   83.26    debug_only(
   83.27      if(TraceReferenceGC && PrintGCDetails) {
   83.28        gclog_or_tty->print_cr("instanceRefKlass::oop_follow_contents " INTPTR_FORMAT, obj);
   83.29      }
   83.30    )
   83.31 -  if (referent != NULL) {
   83.32 +  if (!oopDesc::is_null(heap_oop)) {
   83.33 +    oop referent = oopDesc::decode_heap_oop_not_null(heap_oop);
   83.34      if (PSParallelCompact::mark_bitmap()->is_unmarked(referent) &&
   83.35          PSParallelCompact::ref_processor()->
   83.36            discover_reference(obj, ref->reference_type())) {
   83.37 @@ -182,9 +184,10 @@
   83.38    }                                                                             \
   83.39                                                                                  \
   83.40    T* referent_addr = (T*)java_lang_ref_Reference::referent_addr(obj);           \
   83.41 -  oop referent = oopDesc::load_decode_heap_oop(referent_addr);                  \
   83.42 -  if (referent != NULL && contains(referent_addr)) {                            \
   83.43 +  T heap_oop = oopDesc::load_heap_oop(referent_addr);                           \
   83.44 +  if (!oopDesc::is_null(heap_oop) && contains(referent_addr)) {                 \
   83.45      ReferenceProcessor* rp = closure->_ref_processor;                           \
   83.46 +    oop referent = oopDesc::decode_heap_oop_not_null(heap_oop);                 \
   83.47      if (!referent->is_gc_marked() && (rp != NULL) &&                            \
   83.48          rp->discover_reference(obj, reference_type())) {                        \
   83.49        return size;                                                              \

    84.1 --- a/src/share/vm/oops/methodKlass.cpp	Mon Jul 27 09:06:22 2009 -0700
    84.2 +++ b/src/share/vm/oops/methodKlass.cpp	Mon Jul 27 17:23:52 2009 -0400
    84.3 @@ -68,7 +68,7 @@
    84.4    m->set_constants(NULL);
    84.5    m->set_max_stack(0);
    84.6    m->set_max_locals(0);
    84.7 -  m->clear_intrinsic_id_cache();
    84.8 +  m->set_intrinsic_id(vmIntrinsics::_none);
    84.9    m->set_method_data(NULL);
   84.10    m->set_interpreter_throwout_count(0);
   84.11    m->set_vtable_index(methodOopDesc::garbage_vtable_index);

    85.1 --- a/src/share/vm/oops/methodOop.cpp	Mon Jul 27 09:06:22 2009 -0700
    85.2 +++ b/src/share/vm/oops/methodOop.cpp	Mon Jul 27 17:23:52 2009 -0400
    85.3 @@ -962,26 +962,39 @@
    85.4    return newm;
    85.5  }
    85.6  
    85.7 -vmIntrinsics::ID methodOopDesc::compute_intrinsic_id() const {
    85.8 -  assert(vmIntrinsics::_none == 0, "correct coding of default case");
    85.9 -  const uintptr_t max_cache_uint = right_n_bits((int)(sizeof(_intrinsic_id_cache) * BitsPerByte));
   85.10 -  assert((uintptr_t)vmIntrinsics::ID_LIMIT <= max_cache_uint, "else fix cache size");
   85.11 +vmSymbols::SID methodOopDesc::klass_id_for_intrinsics(klassOop holder) {
   85.12    // if loader is not the default loader (i.e., != NULL), we can't know the intrinsics
   85.13    // because we are not loading from core libraries
   85.14 -  if (instanceKlass::cast(method_holder())->class_loader() != NULL) return vmIntrinsics::_none;
   85.15 +  if (instanceKlass::cast(holder)->class_loader() != NULL)
   85.16 +    return vmSymbols::NO_SID;   // regardless of name, no intrinsics here
   85.17  
   85.18    // see if the klass name is well-known:
   85.19 -  symbolOop klass_name    = instanceKlass::cast(method_holder())->name();
   85.20 -  vmSymbols::SID klass_id = vmSymbols::find_sid(klass_name);
   85.21 -  if (klass_id == vmSymbols::NO_SID)  return vmIntrinsics::_none;
   85.22 +  symbolOop klass_name = instanceKlass::cast(holder)->name();
   85.23 +  return vmSymbols::find_sid(klass_name);
   85.24 +}
   85.25 +
   85.26 +void methodOopDesc::init_intrinsic_id() {
   85.27 +  assert(_intrinsic_id == vmIntrinsics::_none, "do this just once");
   85.28 +  const uintptr_t max_id_uint = right_n_bits((int)(sizeof(_intrinsic_id) * BitsPerByte));
   85.29 +  assert((uintptr_t)vmIntrinsics::ID_LIMIT <= max_id_uint, "else fix size");
   85.30 +
   85.31 +  // the klass name is well-known:
   85.32 +  vmSymbols::SID klass_id = klass_id_for_intrinsics(method_holder());
   85.33 +  assert(klass_id != vmSymbols::NO_SID, "caller responsibility");
   85.34  
   85.35    // ditto for method and signature:
   85.36    vmSymbols::SID  name_id = vmSymbols::find_sid(name());
   85.37 -  if (name_id  == vmSymbols::NO_SID)  return vmIntrinsics::_none;
   85.38 +  if (name_id  == vmSymbols::NO_SID)  return;
   85.39    vmSymbols::SID   sig_id = vmSymbols::find_sid(signature());
   85.40 -  if (sig_id   == vmSymbols::NO_SID)  return vmIntrinsics::_none;
   85.41 +  if (sig_id   == vmSymbols::NO_SID)  return;
   85.42    jshort flags = access_flags().as_short();
   85.43  
   85.44 +  vmIntrinsics::ID id = vmIntrinsics::find_id(klass_id, name_id, sig_id, flags);
   85.45 +  if (id != vmIntrinsics::_none) {
   85.46 +    set_intrinsic_id(id);
   85.47 +    return;
   85.48 +  }
   85.49 +
   85.50    // A few slightly irregular cases:
   85.51    switch (klass_id) {
   85.52    case vmSymbols::VM_SYMBOL_ENUM_NAME(java_lang_StrictMath):
   85.53 @@ -992,15 +1005,18 @@
   85.54      case vmSymbols::VM_SYMBOL_ENUM_NAME(sqrt_name):
   85.55        // pretend it is the corresponding method in the non-strict class:
   85.56        klass_id = vmSymbols::VM_SYMBOL_ENUM_NAME(java_lang_Math);
   85.57 +      id = vmIntrinsics::find_id(klass_id, name_id, sig_id, flags);
   85.58        break;
   85.59      }
   85.60    }
   85.61  
   85.62 -  // return intrinsic id if any
   85.63 -  return vmIntrinsics::find_id(klass_id, name_id, sig_id, flags);
   85.64 +  if (id != vmIntrinsics::_none) {
   85.65 +    // Set up its iid.  It is an alias method.
   85.66 +    set_intrinsic_id(id);
   85.67 +    return;
   85.68 +  }
   85.69  }
   85.70  
   85.71 -
   85.72  // These two methods are static since a GC may move the methodOopDesc
   85.73  bool methodOopDesc::load_signature_classes(methodHandle m, TRAPS) {
   85.74    bool sig_is_loaded = true;

    86.1 --- a/src/share/vm/oops/methodOop.hpp	Mon Jul 27 09:06:22 2009 -0700
    86.2 +++ b/src/share/vm/oops/methodOop.hpp	Mon Jul 27 17:23:52 2009 -0400
    86.3 @@ -104,7 +104,7 @@
    86.4    u2                _max_stack;                  // Maximum number of entries on the expression stack
    86.5    u2                _max_locals;                 // Number of local variables used by this method
    86.6    u2                _size_of_parameters;         // size of the parameter block (receiver + arguments) in words
    86.7 -  u1                _intrinsic_id_cache;         // Cache for intrinsic_id; 0 or 1+vmInt::ID
    86.8 +  u1                _intrinsic_id;               // vmSymbols::intrinsic_id (0 == _none)
    86.9    u1                _highest_tier_compile;       // Highest compile level this method has ever seen.
   86.10    u2                _interpreter_throwout_count; // Count of times method was exited via exception while interpreting
   86.11    u2                _number_of_breakpoints;      // fullspeed debugging support
   86.12 @@ -224,8 +224,6 @@
   86.13    int highest_tier_compile()                     { return _highest_tier_compile;}
   86.14    void set_highest_tier_compile(int level)      { _highest_tier_compile = level;}
   86.15  
   86.16 -  void clear_intrinsic_id_cache() { _intrinsic_id_cache = 0; }
   86.17 -
   86.18    // Count of times method was exited via exception while interpreting
   86.19    void interpreter_throwout_increment() {
   86.20      if (_interpreter_throwout_count < 65534) {
   86.21 @@ -571,18 +569,12 @@
   86.22    void set_cached_itable_index(int index)           { instanceKlass::cast(method_holder())->set_cached_itable_index(method_idnum(), index); }
   86.23  
   86.24    // Support for inlining of intrinsic methods
   86.25 -  vmIntrinsics::ID intrinsic_id() const { // returns zero if not an intrinsic
   86.26 -    const u1& cache = _intrinsic_id_cache;
   86.27 -    if (cache != 0) {
   86.28 -      return (vmIntrinsics::ID)(cache - 1);
   86.29 -    } else {
   86.30 -      vmIntrinsics::ID id = compute_intrinsic_id();
   86.31 -      *(u1*)&cache = ((u1) id) + 1;   // force the cache to be non-const
   86.32 -      vmIntrinsics::verify_method(id, (methodOop) this);
   86.33 -      assert((vmIntrinsics::ID)(cache - 1) == id, "proper conversion");
   86.34 -      return id;
   86.35 -    }
   86.36 -  }
   86.37 +  vmIntrinsics::ID intrinsic_id() const          { return (vmIntrinsics::ID) _intrinsic_id;           }
   86.38 +  void     set_intrinsic_id(vmIntrinsics::ID id) {                           _intrinsic_id = (u1) id; }
   86.39 +
   86.40 +  // Helper routines for intrinsic_id() and vmIntrinsics::method().
   86.41 +  void init_intrinsic_id();     // updates from _none if a match
   86.42 +  static vmSymbols::SID klass_id_for_intrinsics(klassOop holder);
   86.43  
   86.44    // On-stack replacement support
   86.45    bool has_osr_nmethod()                         { return instanceKlass::cast(method_holder())->lookup_osr_nmethod(this, InvocationEntryBci) != NULL; }
   86.46 @@ -635,9 +627,6 @@
   86.47    void set_size_of_parameters(int size)          { _size_of_parameters = size; }
   86.48   private:
   86.49  
   86.50 -  // Helper routine for intrinsic_id().
   86.51 -  vmIntrinsics::ID compute_intrinsic_id() const;
   86.52 -
   86.53    // Inlined elements
   86.54    address* native_function_addr() const          { assert(is_native(), "must be native"); return (address*) (this+1); }
   86.55    address* signature_handler_addr() const        { return native_function_addr() + 1; }

    87.1 --- a/src/share/vm/oops/objArrayKlass.cpp	Mon Jul 27 09:06:22 2009 -0700
    87.2 +++ b/src/share/vm/oops/objArrayKlass.cpp	Mon Jul 27 17:23:52 2009 -0400
    87.3 @@ -85,8 +85,6 @@
    87.4  template <class T> void objArrayKlass::do_copy(arrayOop s, T* src,
    87.5                                 arrayOop d, T* dst, int length, TRAPS) {
    87.6  
    87.7 -  const size_t word_len = objArrayOopDesc::array_size(length);
    87.8 -
    87.9    BarrierSet* bs = Universe::heap()->barrier_set();
   87.10    // For performance reasons, we assume we are that the write barrier we
   87.11    // are using has optimized modes for arrays of references.  At least one
   87.12 @@ -94,11 +92,10 @@
   87.13    assert(bs->has_write_ref_array_opt(), "Barrier set must have ref array opt");
   87.14    assert(bs->has_write_ref_array_pre_opt(), "For pre-barrier as well.");
   87.15  
   87.16 -  MemRegion dst_mr = MemRegion((HeapWord*)dst, word_len);
   87.17    if (s == d) {
   87.18      // since source and destination are equal we do not need conversion checks.
   87.19      assert(length > 0, "sanity check");
   87.20 -    bs->write_ref_array_pre(dst_mr);
   87.21 +    bs->write_ref_array_pre(dst, length);
   87.22      Copy::conjoint_oops_atomic(src, dst, length);
   87.23    } else {
   87.24      // We have to make sure all elements conform to the destination array
   87.25 @@ -106,7 +103,7 @@
   87.26      klassOop stype = objArrayKlass::cast(s->klass())->element_klass();
   87.27      if (stype == bound || Klass::cast(stype)->is_subtype_of(bound)) {
   87.28        // elements are guaranteed to be subtypes, so no check necessary
   87.29 -      bs->write_ref_array_pre(dst_mr);
   87.30 +      bs->write_ref_array_pre(dst, length);
   87.31        Copy::conjoint_oops_atomic(src, dst, length);
   87.32      } else {
   87.33        // slow case: need individual subtype checks
   87.34 @@ -138,6 +135,7 @@
   87.35        }
   87.36      }
   87.37    }
   87.38 +  const size_t word_len = objArrayOopDesc::array_size(length);
   87.39    bs->write_ref_array(MemRegion((HeapWord*)dst, word_len));
   87.40  }
   87.41

    88.1 --- a/src/share/vm/oops/oop.inline.hpp	Mon Jul 27 09:06:22 2009 -0700
    88.2 +++ b/src/share/vm/oops/oop.inline.hpp	Mon Jul 27 17:23:52 2009 -0400
    88.3 @@ -148,12 +148,14 @@
    88.4  
    88.5  inline narrowOop oopDesc::encode_heap_oop_not_null(oop v) {
    88.6    assert(!is_null(v), "oop value can never be zero");
    88.7 +  assert(Universe::heap()->is_in_reserved(v), "Address not in heap");
    88.8    address base = Universe::narrow_oop_base();
    88.9    int    shift = Universe::narrow_oop_shift();
   88.10    uint64_t  pd = (uint64_t)(pointer_delta((void*)v, (void*)base, 1));
   88.11    assert(OopEncodingHeapMax > pd, "change encoding max if new encoding");
   88.12    uint64_t result = pd >> shift;
   88.13    assert((result & CONST64(0xffffffff00000000)) == 0, "narrow oop overflow");
   88.14 +  assert(decode_heap_oop(result) == v, "reversibility");
   88.15    return (narrowOop)result;
   88.16  }
   88.17  
   88.18 @@ -449,7 +451,7 @@
   88.19    oopDesc::bs()->write_ref_field(p, v);
   88.20  }
   88.21  
   88.22 -inline void update_barrier_set_pre(void* p, oop v) {
   88.23 +template <class T> inline void update_barrier_set_pre(T* p, oop v) {
   88.24    oopDesc::bs()->write_ref_field_pre(p, v);
   88.25  }
   88.26  
   88.27 @@ -459,15 +461,15 @@
   88.28    } else {
   88.29      update_barrier_set_pre(p, v);
   88.30      oopDesc::encode_store_heap_oop(p, v);
   88.31 -    update_barrier_set(p, v);
   88.32 +    update_barrier_set((void*)p, v);  // cast away type
   88.33    }
   88.34  }
   88.35  
   88.36  template <class T> inline void oop_store(volatile T* p, oop v) {
   88.37 -  update_barrier_set_pre((void*)p, v);
   88.38 +  update_barrier_set_pre((T*)p, v);   // cast away volatile
   88.39    // Used by release_obj_field_put, so use release_store_ptr.
   88.40    oopDesc::release_encode_store_heap_oop(p, v);
   88.41 -  update_barrier_set((void*)p, v);
   88.42 +  update_barrier_set((void*)p, v);    // cast away type
   88.43  }
   88.44  
   88.45  template <class T> inline void oop_store_without_check(T* p, oop v) {

    89.1 --- a/src/share/vm/oops/oopsHierarchy.hpp	Mon Jul 27 09:06:22 2009 -0700
    89.2 +++ b/src/share/vm/oops/oopsHierarchy.hpp	Mon Jul 27 17:23:52 2009 -0400
    89.3 @@ -29,6 +29,7 @@
    89.4  typedef juint narrowOop; // Offset instead of address for an oop within a java object
    89.5  typedef class klassOopDesc* wideKlassOop; // to keep SA happy and unhandled oop
    89.6                                            // detector happy.
    89.7 +typedef void* OopOrNarrowOopStar;
    89.8  
    89.9  #ifndef CHECK_UNHANDLED_OOPS
   89.10

    90.1 --- a/src/share/vm/opto/block.cpp	Mon Jul 27 09:06:22 2009 -0700
    90.2 +++ b/src/share/vm/opto/block.cpp	Mon Jul 27 17:23:52 2009 -0400
    90.3 @@ -357,6 +357,9 @@
    90.4  #ifndef PRODUCT
    90.5    , _trace_opto_pipelining(TraceOptoPipelining || C->method_has_option("TraceOptoPipelining"))
    90.6  #endif
    90.7 +#ifdef ASSERT
    90.8 +  , _raw_oops(a)
    90.9 +#endif
   90.10  {
   90.11    ResourceMark rm;
   90.12    // I'll need a few machine-specific GotoNodes.  Make an Ideal GotoNode,

    91.1 --- a/src/share/vm/opto/block.hpp	Mon Jul 27 09:06:22 2009 -0700
    91.2 +++ b/src/share/vm/opto/block.hpp	Mon Jul 27 17:23:52 2009 -0400
    91.3 @@ -380,6 +380,10 @@
    91.4    bool _trace_opto_pipelining;  // tracing flag
    91.5  #endif
    91.6  
    91.7 +#ifdef ASSERT
    91.8 +  Unique_Node_List _raw_oops;
    91.9 +#endif
   91.10 +
   91.11    // Build dominators
   91.12    void Dominators();
   91.13

    92.1 --- a/src/share/vm/opto/buildOopMap.cpp	Mon Jul 27 09:06:22 2009 -0700
    92.2 +++ b/src/share/vm/opto/buildOopMap.cpp	Mon Jul 27 17:23:52 2009 -0400
    92.3 @@ -74,9 +74,11 @@
    92.4    // this block.
    92.5    Block *_b;                    // Block for this struct
    92.6    OopFlow *_next;               // Next free OopFlow
    92.7 +                                // or NULL if dead/conflict
    92.8 +  Compile* C;
    92.9  
   92.10 -  OopFlow( short *callees, Node **defs ) : _callees(callees), _defs(defs),
   92.11 -    _b(NULL), _next(NULL) { }
   92.12 +  OopFlow( short *callees, Node **defs, Compile* c ) : _callees(callees), _defs(defs),
   92.13 +    _b(NULL), _next(NULL), C(c) { }
   92.14  
   92.15    // Given reaching-defs for this block start, compute it for this block end
   92.16    void compute_reach( PhaseRegAlloc *regalloc, int max_reg, Dict *safehash );
   92.17 @@ -88,7 +90,7 @@
   92.18    void clone( OopFlow *flow, int max_size);
   92.19  
   92.20    // Make a new OopFlow from scratch
   92.21 -  static OopFlow *make( Arena *A, int max_size );
   92.22 +  static OopFlow *make( Arena *A, int max_size, Compile* C );
   92.23  
   92.24    // Build an oopmap from the current flow info
   92.25    OopMap *build_oop_map( Node *n, int max_reg, PhaseRegAlloc *regalloc, int* live );
   92.26 @@ -180,11 +182,11 @@
   92.27  }
   92.28  
   92.29  //------------------------------make-------------------------------------------
   92.30 -OopFlow *OopFlow::make( Arena *A, int max_size ) {
   92.31 +OopFlow *OopFlow::make( Arena *A, int max_size, Compile* C ) {
   92.32    short *callees = NEW_ARENA_ARRAY(A,short,max_size+1);
   92.33    Node **defs    = NEW_ARENA_ARRAY(A,Node*,max_size+1);
   92.34    debug_only( memset(defs,0,(max_size+1)*sizeof(Node*)) );
   92.35 -  OopFlow *flow = new (A) OopFlow(callees+1, defs+1);
   92.36 +  OopFlow *flow = new (A) OopFlow(callees+1, defs+1, C);
   92.37    assert( &flow->_callees[OptoReg::Bad] == callees, "Ok to index at OptoReg::Bad" );
   92.38    assert( &flow->_defs   [OptoReg::Bad] == defs   , "Ok to index at OptoReg::Bad" );
   92.39    return flow;
   92.40 @@ -288,7 +290,7 @@
   92.41                m = m->in(idx);
   92.42              }
   92.43            }
   92.44 -         guarantee( 0, "must find derived/base pair" );
   92.45 +          guarantee( 0, "must find derived/base pair" );
   92.46          }
   92.47        found: ;
   92.48          Node *base = n->in(i+1); // Base is other half of pair
   92.49 @@ -347,6 +349,13 @@
   92.50      } else {
   92.51        // Other - some reaching non-oop value
   92.52        omap->set_value( r);
   92.53 +#ifdef ASSERT
   92.54 +      if( t->isa_rawptr() && C->cfg()->_raw_oops.member(def) ) {
   92.55 +        def->dump();
   92.56 +        n->dump();
   92.57 +        assert(false, "there should be a oop in OopMap instead of a live raw oop at safepoint");
   92.58 +      }
   92.59 +#endif
   92.60      }
   92.61  
   92.62    }
   92.63 @@ -562,7 +571,7 @@
   92.64  
   92.65    // Do the first block 'by hand' to prime the worklist
   92.66    Block *entry = _cfg->_blocks[1];
   92.67 -  OopFlow *rootflow = OopFlow::make(A,max_reg);
   92.68 +  OopFlow *rootflow = OopFlow::make(A,max_reg,this);
   92.69    // Initialize to 'bottom' (not 'top')
   92.70    memset( rootflow->_callees, OptoReg::Bad, max_reg*sizeof(short) );
   92.71    memset( rootflow->_defs   ,            0, max_reg*sizeof(Node*) );
   92.72 @@ -628,7 +637,7 @@
   92.73        // Carry it forward.
   92.74      } else {                    // Draw a new OopFlow from the freelist
   92.75        if( !free_list )
   92.76 -        free_list = OopFlow::make(A,max_reg);
   92.77 +        free_list = OopFlow::make(A,max_reg,C);
   92.78        flow = free_list;
   92.79        assert( flow->_b == NULL, "oopFlow is not free" );
   92.80        free_list = flow->_next;

    93.1 --- a/src/share/vm/opto/c2compiler.cpp	Mon Jul 27 09:06:22 2009 -0700
    93.2 +++ b/src/share/vm/opto/c2compiler.cpp	Mon Jul 27 17:23:52 2009 -0400
    93.3 @@ -104,7 +104,9 @@
    93.4      initialize();
    93.5    }
    93.6    bool subsume_loads = true;
    93.7 -  bool do_escape_analysis = DoEscapeAnalysis;
    93.8 +  bool do_escape_analysis = DoEscapeAnalysis &&
    93.9 +                            !(env->jvmti_can_hotswap_or_post_breakpoint() ||
   93.10 +                              env->jvmti_can_examine_or_deopt_anywhere());
   93.11    while (!env->failing()) {
   93.12      // Attempt to compile while subsuming loads into machine instructions.
   93.13      Compile C(env, this, target, entry_bci, subsume_loads, do_escape_analysis);

    94.1 --- a/src/share/vm/opto/cfgnode.cpp	Mon Jul 27 09:06:22 2009 -0700
    94.2 +++ b/src/share/vm/opto/cfgnode.cpp	Mon Jul 27 17:23:52 2009 -0400
    94.3 @@ -1796,8 +1796,12 @@
    94.4      for (uint i=1; i<req(); ++i) {// For all paths in
    94.5        Node *ii = in(i);
    94.6        if (ii->is_DecodeN() && ii->bottom_type() == bottom_type()) {
    94.7 -        has_decodeN = true;
    94.8 -        in_decodeN = ii->in(1);
    94.9 +        // Note: in_decodeN is used only to define the type of new phi.
   94.10 +        // Find a non dead path otherwise phi type will be wrong.
   94.11 +        if (ii->in(1)->bottom_type() != Type::TOP) {
   94.12 +          has_decodeN = true;
   94.13 +          in_decodeN = ii->in(1);
   94.14 +        }
   94.15        } else if (!ii->is_Phi()) {
   94.16          may_push = false;
   94.17        }
   94.18 @@ -1805,7 +1809,6 @@
   94.19  
   94.20      if (has_decodeN && may_push) {
   94.21        PhaseIterGVN *igvn = phase->is_IterGVN();
   94.22 -      // Note: in_decodeN is used only to define the type of new phi here.
   94.23        PhiNode *new_phi = PhiNode::make_blank(in(0), in_decodeN);
   94.24        uint orig_cnt = req();
   94.25        for (uint i=1; i<req(); ++i) {// For all paths in

    95.1 --- a/src/share/vm/opto/compile.cpp	Mon Jul 27 09:06:22 2009 -0700
    95.2 +++ b/src/share/vm/opto/compile.cpp	Mon Jul 27 17:23:52 2009 -0400
    95.3 @@ -101,7 +101,8 @@
    95.4      }
    95.5    }
    95.6    // Lazily create intrinsics for intrinsic IDs well-known in the runtime.
    95.7 -  if (m->intrinsic_id() != vmIntrinsics::_none) {
    95.8 +  if (m->intrinsic_id() != vmIntrinsics::_none &&
    95.9 +      m->intrinsic_id() <= vmIntrinsics::LAST_COMPILER_INLINE) {
   95.10      CallGenerator* cg = make_vm_intrinsic(m, is_virtual);
   95.11      if (cg != NULL) {
   95.12        // Save it for next time:
   95.13 @@ -440,6 +441,8 @@
   95.14                    _orig_pc_slot_offset_in_bytes(0),
   95.15                    _node_bundling_limit(0),
   95.16                    _node_bundling_base(NULL),
   95.17 +                  _java_calls(0),
   95.18 +                  _inner_loops(0),
   95.19  #ifndef PRODUCT
   95.20                    _trace_opto_output(TraceOptoOutput || method()->has_option("TraceOptoOutput")),
   95.21                    _printer(IdealGraphPrinter::printer()),
   95.22 @@ -710,6 +713,8 @@
   95.23      _code_buffer("Compile::Fill_buffer"),
   95.24      _node_bundling_limit(0),
   95.25      _node_bundling_base(NULL),
   95.26 +    _java_calls(0),
   95.27 +    _inner_loops(0),
   95.28  #ifndef PRODUCT
   95.29      _trace_opto_output(TraceOptoOutput),
   95.30      _printer(NULL),
   95.31 @@ -1850,22 +1855,26 @@
   95.32    int  _float_count;            // count float ops requiring 24-bit precision
   95.33    int  _double_count;           // count double ops requiring more precision
   95.34    int  _java_call_count;        // count non-inlined 'java' calls
   95.35 +  int  _inner_loop_count;       // count loops which need alignment
   95.36    VectorSet _visited;           // Visitation flags
   95.37    Node_List _tests;             // Set of IfNodes & PCTableNodes
   95.38  
   95.39    Final_Reshape_Counts() :
   95.40 -    _call_count(0), _float_count(0), _double_count(0), _java_call_count(0),
   95.41 +    _call_count(0), _float_count(0), _double_count(0),
   95.42 +    _java_call_count(0), _inner_loop_count(0),
   95.43      _visited( Thread::current()->resource_area() ) { }
   95.44  
   95.45    void inc_call_count  () { _call_count  ++; }
   95.46    void inc_float_count () { _float_count ++; }
   95.47    void inc_double_count() { _double_count++; }
   95.48    void inc_java_call_count() { _java_call_count++; }
   95.49 +  void inc_inner_loop_count() { _inner_loop_count++; }
   95.50  
   95.51    int  get_call_count  () const { return _call_count  ; }
   95.52    int  get_float_count () const { return _float_count ; }
   95.53    int  get_double_count() const { return _double_count; }
   95.54    int  get_java_call_count() const { return _java_call_count; }
   95.55 +  int  get_inner_loop_count() const { return _inner_loop_count; }
   95.56  };
   95.57  
   95.58  static bool oop_offset_is_sane(const TypeInstPtr* tp) {
   95.59 @@ -1877,7 +1886,7 @@
   95.60  
   95.61  //------------------------------final_graph_reshaping_impl----------------------
   95.62  // Implement items 1-5 from final_graph_reshaping below.
   95.63 -static void final_graph_reshaping_impl( Node *n, Final_Reshape_Counts &fpu ) {
   95.64 +static void final_graph_reshaping_impl( Node *n, Final_Reshape_Counts &frc ) {
   95.65  
   95.66    if ( n->outcnt() == 0 ) return; // dead node
   95.67    uint nop = n->Opcode();
   95.68 @@ -1919,13 +1928,13 @@
   95.69    case Op_CmpF:
   95.70    case Op_CmpF3:
   95.71    // case Op_ConvL2F: // longs are split into 32-bit halves
   95.72 -    fpu.inc_float_count();
   95.73 +    frc.inc_float_count();
   95.74      break;
   95.75  
   95.76    case Op_ConvF2D:
   95.77    case Op_ConvD2F:
   95.78 -    fpu.inc_float_count();
   95.79 -    fpu.inc_double_count();
   95.80 +    frc.inc_float_count();
   95.81 +    frc.inc_double_count();
   95.82      break;
   95.83  
   95.84    // Count all double operations that may use FPU
   95.85 @@ -1942,7 +1951,7 @@
   95.86    case Op_ConD:
   95.87    case Op_CmpD:
   95.88    case Op_CmpD3:
   95.89 -    fpu.inc_double_count();
   95.90 +    frc.inc_double_count();
   95.91      break;
   95.92    case Op_Opaque1:              // Remove Opaque Nodes before matching
   95.93    case Op_Opaque2:              // Remove Opaque Nodes before matching
   95.94 @@ -1951,7 +1960,7 @@
   95.95    case Op_CallStaticJava:
   95.96    case Op_CallJava:
   95.97    case Op_CallDynamicJava:
   95.98 -    fpu.inc_java_call_count(); // Count java call site;
   95.99 +    frc.inc_java_call_count(); // Count java call site;
  95.100    case Op_CallRuntime:
  95.101    case Op_CallLeaf:
  95.102    case Op_CallLeafNoFP: {
  95.103 @@ -1962,7 +1971,7 @@
  95.104      // uncommon_trap, _complete_monitor_locking, _complete_monitor_unlocking,
  95.105      // _new_Java, _new_typeArray, _new_objArray, _rethrow_Java, ...
  95.106      if( !call->is_CallStaticJava() || !call->as_CallStaticJava()->_name ) {
  95.107 -      fpu.inc_call_count();   // Count the call site
  95.108 +      frc.inc_call_count();   // Count the call site
  95.109      } else {                  // See if uncommon argument is shared
  95.110        Node *n = call->in(TypeFunc::Parms);
  95.111        int nop = n->Opcode();
  95.112 @@ -1983,11 +1992,11 @@
  95.113    case Op_StoreD:
  95.114    case Op_LoadD:
  95.115    case Op_LoadD_unaligned:
  95.116 -    fpu.inc_double_count();
  95.117 +    frc.inc_double_count();
  95.118      goto handle_mem;
  95.119    case Op_StoreF:
  95.120    case Op_LoadF:
  95.121 -    fpu.inc_float_count();
  95.122 +    frc.inc_float_count();
  95.123      goto handle_mem;
  95.124  
  95.125    case Op_StoreB:
  95.126 @@ -2324,6 +2333,12 @@
  95.127        n->subsume_by(btp);
  95.128      }
  95.129      break;
  95.130 +  case Op_Loop:
  95.131 +  case Op_CountedLoop:
  95.132 +    if (n->as_Loop()->is_inner_loop()) {
  95.133 +      frc.inc_inner_loop_count();
  95.134 +    }
  95.135 +    break;
  95.136    default:
  95.137      assert( !n->is_Call(), "" );
  95.138      assert( !n->is_Mem(), "" );
  95.139 @@ -2332,17 +2347,17 @@
  95.140  
  95.141    // Collect CFG split points
  95.142    if (n->is_MultiBranch())
  95.143 -    fpu._tests.push(n);
  95.144 +    frc._tests.push(n);
  95.145  }
  95.146  
  95.147  //------------------------------final_graph_reshaping_walk---------------------
  95.148  // Replacing Opaque nodes with their input in final_graph_reshaping_impl(),
  95.149  // requires that the walk visits a node's inputs before visiting the node.
  95.150 -static void final_graph_reshaping_walk( Node_Stack &nstack, Node *root, Final_Reshape_Counts &fpu ) {
  95.151 +static void final_graph_reshaping_walk( Node_Stack &nstack, Node *root, Final_Reshape_Counts &frc ) {
  95.152    ResourceArea *area = Thread::current()->resource_area();
  95.153    Unique_Node_List sfpt(area);
  95.154  
  95.155 -  fpu._visited.set(root->_idx); // first, mark node as visited
  95.156 +  frc._visited.set(root->_idx); // first, mark node as visited
  95.157    uint cnt = root->req();
  95.158    Node *n = root;
  95.159    uint  i = 0;
  95.160 @@ -2351,7 +2366,7 @@
  95.161        // Place all non-visited non-null inputs onto stack
  95.162        Node* m = n->in(i);
  95.163        ++i;
  95.164 -      if (m != NULL && !fpu._visited.test_set(m->_idx)) {
  95.165 +      if (m != NULL && !frc._visited.test_set(m->_idx)) {
  95.166          if (m->is_SafePoint() && m->as_SafePoint()->jvms() != NULL)
  95.167            sfpt.push(m);
  95.168          cnt = m->req();
  95.169 @@ -2361,7 +2376,7 @@
  95.170        }
  95.171      } else {
  95.172        // Now do post-visit work
  95.173 -      final_graph_reshaping_impl( n, fpu );
  95.174 +      final_graph_reshaping_impl( n, frc );
  95.175        if (nstack.is_empty())
  95.176          break;             // finished
  95.177        n = nstack.node();   // Get node from stack
  95.178 @@ -2442,16 +2457,16 @@
  95.179      return true;
  95.180    }
  95.181  
  95.182 -  Final_Reshape_Counts fpu;
  95.183 +  Final_Reshape_Counts frc;
  95.184  
  95.185    // Visit everybody reachable!
  95.186    // Allocate stack of size C->unique()/2 to avoid frequent realloc
  95.187    Node_Stack nstack(unique() >> 1);
  95.188 -  final_graph_reshaping_walk(nstack, root(), fpu);
  95.189 +  final_graph_reshaping_walk(nstack, root(), frc);
  95.190  
  95.191    // Check for unreachable (from below) code (i.e., infinite loops).
  95.192 -  for( uint i = 0; i < fpu._tests.size(); i++ ) {
  95.193 -    MultiBranchNode *n = fpu._tests[i]->as_MultiBranch();
  95.194 +  for( uint i = 0; i < frc._tests.size(); i++ ) {
  95.195 +    MultiBranchNode *n = frc._tests[i]->as_MultiBranch();
  95.196      // Get number of CFG targets.
  95.197      // Note that PCTables include exception targets after calls.
  95.198      uint required_outcnt = n->required_outcnt();
  95.199 @@ -2497,7 +2512,7 @@
  95.200      // Check that I actually visited all kids.  Unreached kids
  95.201      // must be infinite loops.
  95.202      for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++)
  95.203 -      if (!fpu._visited.test(n->fast_out(j)->_idx)) {
  95.204 +      if (!frc._visited.test(n->fast_out(j)->_idx)) {
  95.205          record_method_not_compilable("infinite loop");
  95.206          return true;            // Found unvisited kid; must be unreach
  95.207        }
  95.208 @@ -2506,13 +2521,14 @@
  95.209    // If original bytecodes contained a mixture of floats and doubles
  95.210    // check if the optimizer has made it homogenous, item (3).
  95.211    if( Use24BitFPMode && Use24BitFP &&
  95.212 -      fpu.get_float_count() > 32 &&
  95.213 -      fpu.get_double_count() == 0 &&
  95.214 -      (10 * fpu.get_call_count() < fpu.get_float_count()) ) {
  95.215 +      frc.get_float_count() > 32 &&
  95.216 +      frc.get_double_count() == 0 &&
  95.217 +      (10 * frc.get_call_count() < frc.get_float_count()) ) {
  95.218      set_24_bit_selection_and_mode( false,  true );
  95.219    }
  95.220  
  95.221 -  set_has_java_calls(fpu.get_java_call_count() > 0);
  95.222 +  set_java_calls(frc.get_java_call_count());
  95.223 +  set_inner_loops(frc.get_inner_loop_count());
  95.224  
  95.225    // No infinite loops, no reason to bail out.
  95.226    return false;

    96.1 --- a/src/share/vm/opto/compile.hpp	Mon Jul 27 09:06:22 2009 -0700
    96.2 +++ b/src/share/vm/opto/compile.hpp	Mon Jul 27 17:23:52 2009 -0400
    96.3 @@ -223,7 +223,8 @@
    96.4    PhaseCFG*             _cfg;                   // Results of CFG finding
    96.5    bool                  _select_24_bit_instr;   // We selected an instruction with a 24-bit result
    96.6    bool                  _in_24_bit_fp_mode;     // We are emitting instructions with 24-bit results
    96.7 -  bool                  _has_java_calls;        // True if the method has java calls
    96.8 +  int                   _java_calls;            // Number of java calls in the method
    96.9 +  int                   _inner_loops;           // Number of inner loops in the method
   96.10    Matcher*              _matcher;               // Engine to map ideal to machine instructions
   96.11    PhaseRegAlloc*        _regalloc;              // Results of register allocation.
   96.12    int                   _frame_slots;           // Size of total frame in stack slots
   96.13 @@ -505,7 +506,9 @@
   96.14    PhaseCFG*         cfg()                       { return _cfg; }
   96.15    bool              select_24_bit_instr() const { return _select_24_bit_instr; }
   96.16    bool              in_24_bit_fp_mode() const   { return _in_24_bit_fp_mode; }
   96.17 -  bool              has_java_calls() const      { return _has_java_calls; }
   96.18 +  bool              has_java_calls() const      { return _java_calls > 0; }
   96.19 +  int               java_calls() const          { return _java_calls; }
   96.20 +  int               inner_loops() const         { return _inner_loops; }
   96.21    Matcher*          matcher()                   { return _matcher; }
   96.22    PhaseRegAlloc*    regalloc()                  { return _regalloc; }
   96.23    int               frame_slots() const         { return _frame_slots; }
   96.24 @@ -532,7 +535,8 @@
   96.25      _in_24_bit_fp_mode   = mode;
   96.26    }
   96.27  
   96.28 -  void set_has_java_calls(bool z) { _has_java_calls = z; }
   96.29 +  void  set_java_calls(int z) { _java_calls  = z; }
   96.30 +  void set_inner_loops(int z) { _inner_loops = z; }
   96.31  
   96.32    // Instruction bits passed off to the VM
   96.33    int               code_size()                 { return _method_size; }

    97.1 --- a/src/share/vm/opto/escape.cpp	Mon Jul 27 09:06:22 2009 -0700
    97.2 +++ b/src/share/vm/opto/escape.cpp	Mon Jul 27 17:23:52 2009 -0400
    97.3 @@ -578,11 +578,24 @@
    97.4    if (phi_alias_idx == alias_idx) {
    97.5      return orig_phi;
    97.6    }
    97.7 -  // have we already created a Phi for this alias index?
    97.8 +  // Have we recently created a Phi for this alias index?
    97.9    PhiNode *result = get_map_phi(orig_phi->_idx);
   97.10    if (result != NULL && C->get_alias_index(result->adr_type()) == alias_idx) {
   97.11      return result;
   97.12    }
   97.13 +  // Previous check may fail when the same wide memory Phi was split into Phis
   97.14 +  // for different memory slices. Search all Phis for this region.
   97.15 +  if (result != NULL) {
   97.16 +    Node* region = orig_phi->in(0);
   97.17 +    for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
   97.18 +      Node* phi = region->fast_out(i);
   97.19 +      if (phi->is_Phi() &&
   97.20 +          C->get_alias_index(phi->as_Phi()->adr_type()) == alias_idx) {
   97.21 +        assert(phi->_idx >= nodes_size(), "only new Phi per instance memory slice");
   97.22 +        return phi->as_Phi();
   97.23 +      }
   97.24 +    }
   97.25 +  }
   97.26    if ((int)C->unique() + 2*NodeLimitFudgeFactor > MaxNodeLimit) {
   97.27      if (C->do_escape_analysis() == true && !C->failing()) {
   97.28        // Retry compilation without escape analysis.
   97.29 @@ -595,6 +608,7 @@
   97.30    orig_phi_worklist.append_if_missing(orig_phi);
   97.31    const TypePtr *atype = C->get_adr_type(alias_idx);
   97.32    result = PhiNode::make(orig_phi->in(0), NULL, Type::MEMORY, atype);
   97.33 +  C->copy_node_notes_to(result, orig_phi);
   97.34    set_map_phi(orig_phi->_idx, result);
   97.35    igvn->set_type(result, result->bottom_type());
   97.36    record_for_optimizer(result);

    98.1 --- a/src/share/vm/opto/gcm.cpp	Mon Jul 27 09:06:22 2009 -0700
    98.2 +++ b/src/share/vm/opto/gcm.cpp	Mon Jul 27 17:23:52 2009 -0400
    98.3 @@ -1130,6 +1130,9 @@
    98.4          Node *def = self->in(1);
    98.5          if (def != NULL && def->bottom_type()->base() == Type::RawPtr) {
    98.6            early->add_inst(self);
    98.7 +#ifdef ASSERT
    98.8 +          _raw_oops.push(def);
    98.9 +#endif
   98.10            continue;
   98.11          }
   98.12          break;

    99.1 --- a/src/share/vm/opto/graphKit.cpp	Mon Jul 27 09:06:22 2009 -0700
    99.2 +++ b/src/share/vm/opto/graphKit.cpp	Mon Jul 27 17:23:52 2009 -0400
    99.3 @@ -1373,19 +1373,20 @@
    99.4    return st;
    99.5  }
    99.6  
    99.7 +
    99.8  void GraphKit::pre_barrier(Node* ctl,
    99.9                             Node* obj,
   99.10                             Node* adr,
   99.11 -                           uint adr_idx,
   99.12 -                           Node *val,
   99.13 -                           const Type* val_type,
   99.14 +                           uint  adr_idx,
   99.15 +                           Node* val,
   99.16 +                           const TypeOopPtr* val_type,
   99.17                             BasicType bt) {
   99.18    BarrierSet* bs = Universe::heap()->barrier_set();
   99.19    set_control(ctl);
   99.20    switch (bs->kind()) {
   99.21      case BarrierSet::G1SATBCT:
   99.22      case BarrierSet::G1SATBCTLogging:
   99.23 -        g1_write_barrier_pre(obj, adr, adr_idx, val, val_type, bt);
   99.24 +      g1_write_barrier_pre(obj, adr, adr_idx, val, val_type, bt);
   99.25        break;
   99.26  
   99.27      case BarrierSet::CardTableModRef:
   99.28 @@ -1404,8 +1405,8 @@
   99.29                              Node* store,
   99.30                              Node* obj,
   99.31                              Node* adr,
   99.32 -                            uint adr_idx,
   99.33 -                            Node *val,
   99.34 +                            uint  adr_idx,
   99.35 +                            Node* val,
   99.36                              BasicType bt,
   99.37                              bool use_precise) {
   99.38    BarrierSet* bs = Universe::heap()->barrier_set();
   99.39 @@ -1413,7 +1414,7 @@
   99.40    switch (bs->kind()) {
   99.41      case BarrierSet::G1SATBCT:
   99.42      case BarrierSet::G1SATBCTLogging:
   99.43 -        g1_write_barrier_post(store, obj, adr, adr_idx, val, bt, use_precise);
   99.44 +      g1_write_barrier_post(store, obj, adr, adr_idx, val, bt, use_precise);
   99.45        break;
   99.46  
   99.47      case BarrierSet::CardTableModRef:
   99.48 @@ -1431,49 +1432,55 @@
   99.49    }
   99.50  }
   99.51  
   99.52 -Node* GraphKit::store_oop_to_object(Node* ctl,
   99.53 -                                    Node* obj,
   99.54 -                                    Node* adr,
   99.55 -                                    const TypePtr* adr_type,
   99.56 -                                    Node *val,
   99.57 -                                    const Type* val_type,
   99.58 -                                    BasicType bt) {
   99.59 +Node* GraphKit::store_oop(Node* ctl,
   99.60 +                          Node* obj,
   99.61 +                          Node* adr,
   99.62 +                          const TypePtr* adr_type,
   99.63 +                          Node* val,
   99.64 +                          const TypeOopPtr* val_type,
   99.65 +                          BasicType bt,
   99.66 +                          bool use_precise) {
   99.67 +
   99.68 +  set_control(ctl);
   99.69 +  if (stopped()) return top(); // Dead path ?
   99.70 +
   99.71 +  assert(bt == T_OBJECT, "sanity");
   99.72 +  assert(val != NULL, "not dead path");
   99.73    uint adr_idx = C->get_alias_index(adr_type);
   99.74 -  Node* store;
   99.75 -  pre_barrier(ctl, obj, adr, adr_idx, val, val_type, bt);
   99.76 -  store = store_to_memory(control(), adr, val, bt, adr_idx);
   99.77 -  post_barrier(control(), store, obj, adr, adr_idx, val, bt, false);
   99.78 +  assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
   99.79 +
   99.80 +  pre_barrier(control(), obj, adr, adr_idx, val, val_type, bt);
   99.81 +  Node* store = store_to_memory(control(), adr, val, bt, adr_idx);
   99.82 +  post_barrier(control(), store, obj, adr, adr_idx, val, bt, use_precise);
   99.83    return store;
   99.84  }
   99.85  
   99.86 -Node* GraphKit::store_oop_to_array(Node* ctl,
   99.87 -                                   Node* obj,
   99.88 -                                   Node* adr,
   99.89 -                                   const TypePtr* adr_type,
   99.90 -                                   Node *val,
   99.91 -                                   const Type* val_type,
   99.92 -                                   BasicType bt) {
   99.93 -  uint adr_idx = C->get_alias_index(adr_type);
   99.94 -  Node* store;
   99.95 -  pre_barrier(ctl, obj, adr, adr_idx, val, val_type, bt);
   99.96 -  store = store_to_memory(control(), adr, val, bt, adr_idx);
   99.97 -  post_barrier(control(), store, obj, adr, adr_idx, val, bt, true);
   99.98 -  return store;
   99.99 -}
  99.100 -
  99.101 +// Could be an array or object we don't know at compile time (unsafe ref.)
  99.102  Node* GraphKit::store_oop_to_unknown(Node* ctl,
  99.103 -                                     Node* obj,
  99.104 -                                     Node* adr,
  99.105 -                                     const TypePtr* adr_type,
  99.106 -                                     Node *val,
  99.107 -                                     const Type* val_type,
  99.108 -                                     BasicType bt) {
  99.109 -  uint adr_idx = C->get_alias_index(adr_type);
  99.110 -  Node* store;
  99.111 -  pre_barrier(ctl, obj, adr, adr_idx, val, val_type, bt);
  99.112 -  store = store_to_memory(control(), adr, val, bt, adr_idx);
  99.113 -  post_barrier(control(), store, obj, adr, adr_idx, val, bt, true);
  99.114 -  return store;
  99.115 +                             Node* obj,   // containing obj
  99.116 +                             Node* adr,  // actual adress to store val at
  99.117 +                             const TypePtr* adr_type,
  99.118 +                             Node* val,
  99.119 +                             BasicType bt) {
  99.120 +  Compile::AliasType* at = C->alias_type(adr_type);
  99.121 +  const TypeOopPtr* val_type = NULL;
  99.122 +  if (adr_type->isa_instptr()) {
  99.123 +    if (at->field() != NULL) {
  99.124 +      // known field.  This code is a copy of the do_put_xxx logic.
  99.125 +      ciField* field = at->field();
  99.126 +      if (!field->type()->is_loaded()) {
  99.127 +        val_type = TypeInstPtr::BOTTOM;
  99.128 +      } else {
  99.129 +        val_type = TypeOopPtr::make_from_klass(field->type()->as_klass());
  99.130 +      }
  99.131 +    }
  99.132 +  } else if (adr_type->isa_aryptr()) {
  99.133 +    val_type = adr_type->is_aryptr()->elem()->make_oopptr();
  99.134 +  }
  99.135 +  if (val_type == NULL) {
  99.136 +    val_type = TypeInstPtr::BOTTOM;
  99.137 +  }
  99.138 +  return store_oop(ctl, obj, adr, adr_type, val, val_type, bt, true);
  99.139  }
  99.140  
  99.141  
  99.142 @@ -1787,93 +1794,6 @@
  99.143  }
  99.144  
  99.145  
  99.146 -//------------------------------store_barrier----------------------------------
  99.147 -// Insert a write-barrier store.  This is to let generational GC work; we have
  99.148 -// to flag all oop-stores before the next GC point.
  99.149 -void GraphKit::write_barrier_post(Node* oop_store, Node* obj, Node* adr,
  99.150 -                                  Node* val, bool use_precise) {
  99.151 -  // No store check needed if we're storing a NULL or an old object
  99.152 -  // (latter case is probably a string constant). The concurrent
  99.153 -  // mark sweep garbage collector, however, needs to have all nonNull
  99.154 -  // oop updates flagged via card-marks.
  99.155 -  if (val != NULL && val->is_Con()) {
  99.156 -    // must be either an oop or NULL
  99.157 -    const Type* t = val->bottom_type();
  99.158 -    if (t == TypePtr::NULL_PTR || t == Type::TOP)
  99.159 -      // stores of null never (?) need barriers
  99.160 -      return;
  99.161 -    ciObject* con = t->is_oopptr()->const_oop();
  99.162 -    if (con != NULL
  99.163 -        && con->is_perm()
  99.164 -        && Universe::heap()->can_elide_permanent_oop_store_barriers())
  99.165 -      // no store barrier needed, because no old-to-new ref created
  99.166 -      return;
  99.167 -  }
  99.168 -
  99.169 -  if (use_ReduceInitialCardMarks()
  99.170 -      && obj == just_allocated_object(control())) {
  99.171 -    // We can skip marks on a freshly-allocated object.
  99.172 -    // Keep this code in sync with do_eager_card_mark in runtime.cpp.
  99.173 -    // That routine eagerly marks the occasional object which is produced
  99.174 -    // by the slow path, so that we don't have to do it here.
  99.175 -    return;
  99.176 -  }
  99.177 -
  99.178 -  if (!use_precise) {
  99.179 -    // All card marks for a (non-array) instance are in one place:
  99.180 -    adr = obj;
  99.181 -  }
  99.182 -  // (Else it's an array (or unknown), and we want more precise card marks.)
  99.183 -  assert(adr != NULL, "");
  99.184 -
  99.185 -  // Get the alias_index for raw card-mark memory
  99.186 -  int adr_type = Compile::AliasIdxRaw;
  99.187 -  // Convert the pointer to an int prior to doing math on it
  99.188 -  Node* cast = _gvn.transform(new (C, 2) CastP2XNode(control(), adr));
  99.189 -  // Divide by card size
  99.190 -  assert(Universe::heap()->barrier_set()->kind() == BarrierSet::CardTableModRef,
  99.191 -         "Only one we handle so far.");
  99.192 -  CardTableModRefBS* ct =
  99.193 -    (CardTableModRefBS*)(Universe::heap()->barrier_set());
  99.194 -  Node *b = _gvn.transform(new (C, 3) URShiftXNode( cast, _gvn.intcon(CardTableModRefBS::card_shift) ));
  99.195 -  // We store into a byte array, so do not bother to left-shift by zero
  99.196 -  Node *c = byte_map_base_node();
  99.197 -  // Combine
  99.198 -  Node *sb_ctl = control();
  99.199 -  Node *sb_adr = _gvn.transform(new (C, 4) AddPNode( top()/*no base ptr*/, c, b ));
  99.200 -  Node *sb_val = _gvn.intcon(0);
  99.201 -  // Smash zero into card
  99.202 -  if( !UseConcMarkSweepGC ) {
  99.203 -    BasicType bt = T_BYTE;
  99.204 -    store_to_memory(sb_ctl, sb_adr, sb_val, bt, adr_type);
  99.205 -  } else {
  99.206 -    // Specialized path for CM store barrier
  99.207 -    cms_card_mark( sb_ctl, sb_adr, sb_val, oop_store);
  99.208 -  }
  99.209 -}
  99.210 -
  99.211 -// Specialized path for CMS store barrier
  99.212 -void GraphKit::cms_card_mark(Node* ctl, Node* adr, Node* val, Node *oop_store) {
  99.213 -  BasicType bt = T_BYTE;
  99.214 -  int adr_idx = Compile::AliasIdxRaw;
  99.215 -  Node* mem = memory(adr_idx);
  99.216 -
  99.217 -  // The type input is NULL in PRODUCT builds
  99.218 -  const TypePtr* type = NULL;
  99.219 -  debug_only(type = C->get_adr_type(adr_idx));
  99.220 -
  99.221 -  // Add required edge to oop_store, optimizer does not support precedence edges.
  99.222 -  // Convert required edge to precedence edge before allocation.
  99.223 -  Node *store = _gvn.transform( new (C, 5) StoreCMNode(ctl, mem, adr, type, val, oop_store) );
  99.224 -  set_memory(store, adr_idx);
  99.225 -
  99.226 -  // For CMS, back-to-back card-marks can only remove the first one
  99.227 -  // and this requires DU info.  Push on worklist for optimizer.
  99.228 -  if (mem->req() > MemNode::Address && adr == mem->in(MemNode::Address))
  99.229 -    record_for_igvn(store);
  99.230 -}
  99.231 -
  99.232 -
  99.233  void GraphKit::round_double_arguments(ciMethod* dest_method) {
  99.234    // (Note:  TypeFunc::make has a cache that makes this fast.)
  99.235    const TypeFunc* tf    = TypeFunc::make(dest_method);
  99.236 @@ -3198,17 +3118,88 @@
  99.237    return NULL;
  99.238  }
  99.239  
  99.240 +//----------------------------- store barriers ----------------------------
  99.241 +#define __ ideal.
  99.242 +
  99.243 +void GraphKit::sync_kit(IdealKit& ideal) {
  99.244 +  // Final sync IdealKit and graphKit.
  99.245 +  __ drain_delay_transform();
  99.246 +  set_all_memory(__ merged_memory());
  99.247 +  set_control(__ ctrl());
  99.248 +}
  99.249 +
  99.250 +// vanilla/CMS post barrier
  99.251 +// Insert a write-barrier store.  This is to let generational GC work; we have
  99.252 +// to flag all oop-stores before the next GC point.
  99.253 +void GraphKit::write_barrier_post(Node* oop_store,
  99.254 +                                  Node* obj,
  99.255 +                                  Node* adr,
  99.256 +                                  Node* val,
  99.257 +                                  bool use_precise) {
  99.258 +  // No store check needed if we're storing a NULL or an old object
  99.259 +  // (latter case is probably a string constant). The concurrent
  99.260 +  // mark sweep garbage collector, however, needs to have all nonNull
  99.261 +  // oop updates flagged via card-marks.
  99.262 +  if (val != NULL && val->is_Con()) {
  99.263 +    // must be either an oop or NULL
  99.264 +    const Type* t = val->bottom_type();
  99.265 +    if (t == TypePtr::NULL_PTR || t == Type::TOP)
  99.266 +      // stores of null never (?) need barriers
  99.267 +      return;
  99.268 +    ciObject* con = t->is_oopptr()->const_oop();
  99.269 +    if (con != NULL
  99.270 +        && con->is_perm()
  99.271 +        && Universe::heap()->can_elide_permanent_oop_store_barriers())
  99.272 +      // no store barrier needed, because no old-to-new ref created
  99.273 +      return;
  99.274 +  }
  99.275 +
  99.276 +  if (!use_precise) {
  99.277 +    // All card marks for a (non-array) instance are in one place:
  99.278 +    adr = obj;
  99.279 +  }
  99.280 +  // (Else it's an array (or unknown), and we want more precise card marks.)
  99.281 +  assert(adr != NULL, "");
  99.282 +
  99.283 +  IdealKit ideal(gvn(), control(), merged_memory(), true);
  99.284 +
  99.285 +  // Convert the pointer to an int prior to doing math on it
  99.286 +  Node* cast = __ CastPX(__ ctrl(), adr);
  99.287 +
  99.288 +  // Divide by card size
  99.289 +  assert(Universe::heap()->barrier_set()->kind() == BarrierSet::CardTableModRef,
  99.290 +         "Only one we handle so far.");
  99.291 +  Node* card_offset = __ URShiftX( cast, __ ConI(CardTableModRefBS::card_shift) );
  99.292 +
  99.293 +  // Combine card table base and card offset
  99.294 +  Node* card_adr = __ AddP(__ top(), byte_map_base_node(), card_offset );
  99.295 +
  99.296 +  // Get the alias_index for raw card-mark memory
  99.297 +  int adr_type = Compile::AliasIdxRaw;
  99.298 +  // Smash zero into card
  99.299 +  Node*   zero = __ ConI(0);
  99.300 +  BasicType bt = T_BYTE;
  99.301 +  if( !UseConcMarkSweepGC ) {
  99.302 +    __ store(__ ctrl(), card_adr, zero, bt, adr_type);
  99.303 +  } else {
  99.304 +    // Specialized path for CM store barrier
  99.305 +    __ storeCM(__ ctrl(), card_adr, zero, oop_store, bt, adr_type);
  99.306 +  }
  99.307 +
  99.308 +  // Final sync IdealKit and GraphKit.
  99.309 +  sync_kit(ideal);
  99.310 +}
  99.311 +
  99.312 +// G1 pre/post barriers
  99.313  void GraphKit::g1_write_barrier_pre(Node* obj,
  99.314                                      Node* adr,
  99.315                                      uint alias_idx,
  99.316                                      Node* val,
  99.317 -                                    const Type* val_type,
  99.318 +                                    const TypeOopPtr* val_type,
  99.319                                      BasicType bt) {
  99.320    IdealKit ideal(gvn(), control(), merged_memory(), true);
  99.321 -#define __ ideal.
  99.322 -  __ declares_done();
  99.323 -
  99.324 -  Node* thread = __ thread();
  99.325 +
  99.326 +  Node* tls = __ thread(); // ThreadLocalStorage
  99.327  
  99.328    Node* no_ctrl = NULL;
  99.329    Node* no_base = __ top();
  99.330 @@ -3231,9 +3222,9 @@
  99.331  
  99.332    // set_control( ctl);
  99.333  
  99.334 -  Node* marking_adr = __ AddP(no_base, thread, __ ConX(marking_offset));
  99.335 -  Node* buffer_adr  = __ AddP(no_base, thread, __ ConX(buffer_offset));
  99.336 -  Node* index_adr   = __ AddP(no_base, thread, __ ConX(index_offset));
  99.337 +  Node* marking_adr = __ AddP(no_base, tls, __ ConX(marking_offset));
  99.338 +  Node* buffer_adr  = __ AddP(no_base, tls, __ ConX(buffer_offset));
  99.339 +  Node* index_adr   = __ AddP(no_base, tls, __ ConX(index_offset));
  99.340  
  99.341    // Now some of the values
  99.342  
  99.343 @@ -3261,55 +3252,52 @@
  99.344          Node* next_index = __ SubI(index,  __ ConI(sizeof(intptr_t)));
  99.345          Node* next_indexX = next_index;
  99.346  #ifdef _LP64
  99.347 -          // We could refine the type for what it's worth
  99.348 -          // const TypeLong* lidxtype = TypeLong::make(CONST64(0), get_size_from_queue);
  99.349 -          next_indexX = _gvn.transform( new (C, 2) ConvI2LNode(next_index, TypeLong::make(0, max_jlong, Type::WidenMax)) );
  99.350 -#endif // _LP64
  99.351 +        // We could refine the type for what it's worth
  99.352 +        // const TypeLong* lidxtype = TypeLong::make(CONST64(0), get_size_from_queue);
  99.353 +        next_indexX = _gvn.transform( new (C, 2) ConvI2LNode(next_index, TypeLong::make(0, max_jlong, Type::WidenMax)) );
  99.354 +#endif
  99.355  
  99.356          // Now get the buffer location we will log the original value into and store it
  99.357 -
  99.358          Node *log_addr = __ AddP(no_base, buffer, next_indexX);
  99.359 -        // __ store(__ ctrl(), log_addr, orig, T_OBJECT, C->get_alias_index(TypeOopPtr::BOTTOM));
  99.360          __ store(__ ctrl(), log_addr, orig, T_OBJECT, Compile::AliasIdxRaw);
  99.361  
  99.362 -
  99.363          // update the index
  99.364 -        // __ store(__ ctrl(), index_adr, next_index, T_INT, Compile::AliasIdxRaw);
  99.365 -        // This is a hack to force this store to occur before the oop store that is coming up
  99.366 -        __ store(__ ctrl(), index_adr, next_index, T_INT, C->get_alias_index(TypeOopPtr::BOTTOM));
  99.367 +        __ store(__ ctrl(), index_adr, next_index, T_INT, Compile::AliasIdxRaw);
  99.368  
  99.369        } __ else_(); {
  99.370  
  99.371          // logging buffer is full, call the runtime
  99.372          const TypeFunc *tf = OptoRuntime::g1_wb_pre_Type();
  99.373 -        // __ make_leaf_call(tf, OptoRuntime::g1_wb_pre_Java(), "g1_wb_pre", orig, thread);
  99.374 -        __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), "g1_wb_pre", orig, thread);
  99.375 -      } __ end_if();
  99.376 -    } __ end_if();
  99.377 -  } __ end_if();
  99.378 -
  99.379 -  __ drain_delay_transform();
  99.380 -  set_control( __ ctrl());
  99.381 -  set_all_memory( __ merged_memory());
  99.382 -
  99.383 -#undef __
  99.384 +        __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), "g1_wb_pre", orig, tls);
  99.385 +      } __ end_if();  // (!index)
  99.386 +    } __ end_if();  // (orig != NULL)
  99.387 +  } __ end_if();  // (!marking)
  99.388 +
  99.389 +  // Final sync IdealKit and GraphKit.
  99.390 +  sync_kit(ideal);
  99.391  }
  99.392  
  99.393  //
  99.394  // Update the card table and add card address to the queue
  99.395  //
  99.396 -void GraphKit::g1_mark_card(IdealKit* ideal, Node* card_adr, Node* store,  Node* index, Node* index_adr, Node* buffer, const TypeFunc* tf) {
  99.397 -#define __ ideal->
  99.398 +void GraphKit::g1_mark_card(IdealKit& ideal,
  99.399 +                            Node* card_adr,
  99.400 +                            Node* oop_store,
  99.401 +                            Node* index,
  99.402 +                            Node* index_adr,
  99.403 +                            Node* buffer,
  99.404 +                            const TypeFunc* tf) {
  99.405 +
  99.406    Node* zero = __ ConI(0);
  99.407    Node* no_base = __ top();
  99.408    BasicType card_bt = T_BYTE;
  99.409    // Smash zero into card. MUST BE ORDERED WRT TO STORE
  99.410 -  __ storeCM(__ ctrl(), card_adr, zero, store, card_bt, Compile::AliasIdxRaw);
  99.411 +  __ storeCM(__ ctrl(), card_adr, zero, oop_store, card_bt, Compile::AliasIdxRaw);
  99.412  
  99.413    //  Now do the queue work
  99.414    __ if_then(index, BoolTest::ne, zero); {
  99.415  
  99.416 -    Node* next_index = __ SubI(index,  __ ConI(sizeof(intptr_t)));
  99.417 +    Node* next_index = __ SubI(index, __ ConI(sizeof(intptr_t)));
  99.418      Node* next_indexX = next_index;
  99.419  #ifdef _LP64
  99.420      // We could refine the type for what it's worth
  99.421 @@ -3324,10 +3312,10 @@
  99.422    } __ else_(); {
  99.423      __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), "g1_wb_post", card_adr, __ thread());
  99.424    } __ end_if();
  99.425 -#undef __
  99.426 +
  99.427  }
  99.428  
  99.429 -void GraphKit::g1_write_barrier_post(Node* store,
  99.430 +void GraphKit::g1_write_barrier_post(Node* oop_store,
  99.431                                       Node* obj,
  99.432                                       Node* adr,
  99.433                                       uint alias_idx,
  99.434 @@ -3352,10 +3340,8 @@
  99.435    assert(adr != NULL, "");
  99.436  
  99.437    IdealKit ideal(gvn(), control(), merged_memory(), true);
  99.438 -#define __ ideal.
  99.439 -  __ declares_done();
  99.440 -
  99.441 -  Node* thread = __ thread();
  99.442 +
  99.443 +  Node* tls = __ thread(); // ThreadLocalStorage
  99.444  
  99.445    Node* no_ctrl = NULL;
  99.446    Node* no_base = __ top();
  99.447 @@ -3377,8 +3363,8 @@
  99.448  
  99.449    // Pointers into the thread
  99.450  
  99.451 -  Node* buffer_adr = __ AddP(no_base, thread, __ ConX(buffer_offset));
  99.452 -  Node* index_adr =  __ AddP(no_base, thread, __ ConX(index_offset));
  99.453 +  Node* buffer_adr = __ AddP(no_base, tls, __ ConX(buffer_offset));
  99.454 +  Node* index_adr =  __ AddP(no_base, tls, __ ConX(index_offset));
  99.455  
  99.456    // Now some values
  99.457  
  99.458 @@ -3387,18 +3373,14 @@
  99.459  
  99.460  
  99.461    // Convert the store obj pointer to an int prior to doing math on it
  99.462 -  // Use addr not obj gets accurate card marks
  99.463 -
  99.464 -  // Node* cast = __ CastPX(no_ctrl, adr /* obj */);
  99.465 -
  99.466    // Must use ctrl to prevent "integerized oop" existing across safepoint
  99.467 -  Node* cast =  __ CastPX(__ ctrl(), ( use_precise ? adr : obj ));
  99.468 +  Node* cast =  __ CastPX(__ ctrl(), adr);
  99.469  
  99.470    // Divide pointer by card size
  99.471    Node* card_offset = __ URShiftX( cast, __ ConI(CardTableModRefBS::card_shift) );
  99.472  
  99.473    // Combine card table base and card offset
  99.474 -  Node *card_adr = __ AddP(no_base, byte_map_base_node(), card_offset );
  99.475 +  Node* card_adr = __ AddP(no_base, byte_map_base_node(), card_offset );
  99.476  
  99.477    // If we know the value being stored does it cross regions?
  99.478  
  99.479 @@ -3422,18 +3404,17 @@
  99.480          Node* card_val = __ load(__ ctrl(), card_adr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw);
  99.481  
  99.482          __ if_then(card_val, BoolTest::ne, zero); {
  99.483 -          g1_mark_card(&ideal, card_adr, store, index, index_adr, buffer, tf);
  99.484 +          g1_mark_card(ideal, card_adr, oop_store, index, index_adr, buffer, tf);
  99.485          } __ end_if();
  99.486        } __ end_if();
  99.487      } __ end_if();
  99.488    } else {
  99.489 -    g1_mark_card(&ideal, card_adr, store, index, index_adr, buffer, tf);
  99.490 +    // Object.clone() instrinsic uses this path.
  99.491 +    g1_mark_card(ideal, card_adr, oop_store, index, index_adr, buffer, tf);
  99.492    }
  99.493  
  99.494 -
  99.495 -  __ drain_delay_transform();
  99.496 -  set_control( __ ctrl());
  99.497 -  set_all_memory( __ merged_memory());
  99.498 +  // Final sync IdealKit and GraphKit.
  99.499 +  sync_kit(ideal);
  99.500 +}
  99.501  #undef __
  99.502  
  99.503 -}

   100.1 --- a/src/share/vm/opto/graphKit.hpp	Mon Jul 27 09:06:22 2009 -0700
   100.2 +++ b/src/share/vm/opto/graphKit.hpp	Mon Jul 27 17:23:52 2009 -0400
   100.3 @@ -449,21 +449,34 @@
   100.4    //
   100.5    // If val==NULL, it is taken to be a completely unknown value. QQQ
   100.6  
   100.7 +  Node* store_oop(Node* ctl,
   100.8 +                  Node* obj,   // containing obj
   100.9 +                  Node* adr,  // actual adress to store val at
  100.10 +                  const TypePtr* adr_type,
  100.11 +                  Node* val,
  100.12 +                  const TypeOopPtr* val_type,
  100.13 +                  BasicType bt,
  100.14 +                  bool use_precise);
  100.15 +
  100.16    Node* store_oop_to_object(Node* ctl,
  100.17                              Node* obj,   // containing obj
  100.18                              Node* adr,  // actual adress to store val at
  100.19                              const TypePtr* adr_type,
  100.20                              Node* val,
  100.21 -                            const Type* val_type,
  100.22 -                            BasicType bt);
  100.23 +                            const TypeOopPtr* val_type,
  100.24 +                            BasicType bt) {
  100.25 +    return store_oop(ctl, obj, adr, adr_type, val, val_type, bt, false);
  100.26 +  }
  100.27  
  100.28    Node* store_oop_to_array(Node* ctl,
  100.29                             Node* obj,   // containing obj
  100.30                             Node* adr,  // actual adress to store val at
  100.31                             const TypePtr* adr_type,
  100.32                             Node* val,
  100.33 -                           const Type* val_type,
  100.34 -                           BasicType bt);
  100.35 +                           const TypeOopPtr* val_type,
  100.36 +                           BasicType bt) {
  100.37 +    return store_oop(ctl, obj, adr, adr_type, val, val_type, bt, true);
  100.38 +  }
  100.39  
  100.40    // Could be an array or object we don't know at compile time (unsafe ref.)
  100.41    Node* store_oop_to_unknown(Node* ctl,
  100.42 @@ -471,12 +484,11 @@
  100.43                               Node* adr,  // actual adress to store val at
  100.44                               const TypePtr* adr_type,
  100.45                               Node* val,
  100.46 -                             const Type* val_type,
  100.47                               BasicType bt);
  100.48  
  100.49    // For the few case where the barriers need special help
  100.50    void pre_barrier(Node* ctl, Node* obj, Node* adr, uint adr_idx,
  100.51 -                   Node* val, const Type* val_type, BasicType bt);
  100.52 +                   Node* val, const TypeOopPtr* val_type, BasicType bt);
  100.53  
  100.54    void post_barrier(Node* ctl, Node* store, Node* obj, Node* adr, uint adr_idx,
  100.55                      Node* val, BasicType bt, bool use_precise);
  100.56 @@ -489,9 +501,6 @@
  100.57    // Return a load of array element at idx.
  100.58    Node* load_array_element(Node* ctl, Node* ary, Node* idx, const TypeAryPtr* arytype);
  100.59  
  100.60 -  // CMS card-marks have an input from the corresponding oop_store
  100.61 -  void  cms_card_mark(Node* ctl, Node* adr, Node* val, Node* oop_store);
  100.62 -
  100.63    //---------------- Dtrace support --------------------
  100.64    void make_dtrace_method_entry_exit(ciMethod* method, bool is_entry);
  100.65    void make_dtrace_method_entry(ciMethod* method) {
  100.66 @@ -583,9 +592,6 @@
  100.67      return C->too_many_recompiles(method(), bci(), reason);
  100.68    }
  100.69  
  100.70 -  // vanilla/CMS post barrier
  100.71 -  void write_barrier_post(Node *store, Node* obj, Node* adr, Node* val, bool use_precise);
  100.72 -
  100.73    // Returns the object (if any) which was created the moment before.
  100.74    Node* just_allocated_object(Node* current_control);
  100.75  
  100.76 @@ -594,12 +600,17 @@
  100.77              && Universe::heap()->can_elide_tlab_store_barriers());
  100.78    }
  100.79  
  100.80 +  void sync_kit(IdealKit& ideal);
  100.81 +
  100.82 +  // vanilla/CMS post barrier
  100.83 +  void write_barrier_post(Node *store, Node* obj, Node* adr, Node* val, bool use_precise);
  100.84 +
  100.85    // G1 pre/post barriers
  100.86    void g1_write_barrier_pre(Node* obj,
  100.87                              Node* adr,
  100.88                              uint alias_idx,
  100.89                              Node* val,
  100.90 -                            const Type* val_type,
  100.91 +                            const TypeOopPtr* val_type,
  100.92                              BasicType bt);
  100.93  
  100.94    void g1_write_barrier_post(Node* store,
  100.95 @@ -611,7 +622,7 @@
  100.96                               bool use_precise);
  100.97    // Helper function for g1
  100.98    private:
  100.99 -  void g1_mark_card(IdealKit* ideal, Node* card_adr, Node* store,  Node* index, Node* index_adr,
 100.100 +  void g1_mark_card(IdealKit& ideal, Node* card_adr, Node* store,  Node* index, Node* index_adr,
 100.101                      Node* buffer, const TypeFunc* tf);
 100.102  
 100.103    public:

   101.1 --- a/src/share/vm/opto/idealKit.cpp	Mon Jul 27 09:06:22 2009 -0700
   101.2 +++ b/src/share/vm/opto/idealKit.cpp	Mon Jul 27 17:23:52 2009 -0400
   101.3 @@ -34,7 +34,7 @@
   101.4  const uint IdealKit::first_var = TypeFunc::Parms + 1;
   101.5  
   101.6  //----------------------------IdealKit-----------------------------------------
   101.7 -IdealKit::IdealKit(PhaseGVN &gvn, Node* control, Node* mem, bool delay_all_transforms) :
   101.8 +IdealKit::IdealKit(PhaseGVN &gvn, Node* control, Node* mem, bool delay_all_transforms, bool has_declarations) :
   101.9    _gvn(gvn), C(gvn.C) {
  101.10    _initial_ctrl = control;
  101.11    _initial_memory = mem;
  101.12 @@ -47,6 +47,9 @@
  101.13    _pending_cvstates = new (C->node_arena()) GrowableArray<Node*>(C->node_arena(), init_size, 0, 0);
  101.14    _delay_transform  = new (C->node_arena()) GrowableArray<Node*>(C->node_arena(), init_size, 0, 0);
  101.15    DEBUG_ONLY(_state = new (C->node_arena()) GrowableArray<int>(C->node_arena(), init_size, 0, 0));
  101.16 +  if (!has_declarations) {
  101.17 +     declarations_done();
  101.18 +  }
  101.19  }
  101.20  
  101.21  //-------------------------------if_then-------------------------------------
  101.22 @@ -97,7 +100,7 @@
  101.23  //-------------------------------end_if-------------------------------------
  101.24  // Merge the "then" and "else" cvstates.
  101.25  //
  101.26 -// The if_then() pushed the current state for later use
  101.27 +// The if_then() pushed a copy of the current state for later use
  101.28  // as the initial state for a future "else" clause.  The
  101.29  // current state then became the initial state for the
  101.30  // then clause.  If an "else" clause was encountered, it will
  101.31 @@ -258,8 +261,8 @@
  101.32    return delay_transform(PhiNode::make(reg, n, ct));
  101.33  }
  101.34  
  101.35 -//-----------------------------declares_done-----------------------------------
  101.36 -void IdealKit::declares_done() {
  101.37 +//-----------------------------declarations_done-------------------------------
  101.38 +void IdealKit::declarations_done() {
  101.39    _cvstate = new_cvstate();   // initialize current cvstate
  101.40    set_ctrl(_initial_ctrl);    // initialize control in current cvstate
  101.41    set_all_memory(_initial_memory);// initialize memory in current cvstate
  101.42 @@ -277,7 +280,9 @@
  101.43  
  101.44  //-----------------------------delay_transform-----------------------------------
  101.45  Node* IdealKit::delay_transform(Node* n) {
  101.46 -  gvn().set_type(n, n->bottom_type());
  101.47 +  if (!gvn().is_IterGVN() || !gvn().is_IterGVN()->delay_transform()) {
  101.48 +    gvn().set_type(n, n->bottom_type());
  101.49 +  }
  101.50    _delay_transform->push(n);
  101.51    return n;
  101.52  }
  101.53 @@ -321,7 +326,9 @@
  101.54  Node* IdealKit::memory(uint alias_idx) {
  101.55    MergeMemNode* mem = merged_memory();
  101.56    Node* p = mem->memory_at(alias_idx);
  101.57 -  _gvn.set_type(p, Type::MEMORY);  // must be mapped
  101.58 +  if (!gvn().is_IterGVN() || !gvn().is_IterGVN()->delay_transform()) {
  101.59 +    _gvn.set_type(p, Type::MEMORY);  // must be mapped
  101.60 +  }
  101.61    return p;
  101.62  }
  101.63  
  101.64 @@ -462,9 +469,6 @@
  101.65    const TypePtr* adr_type = TypeRawPtr::BOTTOM;
  101.66    uint adr_idx = C->get_alias_index(adr_type);
  101.67  
  101.68 -  // Clone initial memory
  101.69 -  MergeMemNode* cloned_mem =  MergeMemNode::make(C, merged_memory());
  101.70 -
  101.71    // Slow-path leaf call
  101.72    int size = slow_call_type->domain()->cnt();
  101.73    CallNode *call =  (CallNode*)new (C, size) CallLeafNode( slow_call_type, slow_call, leaf_name, adr_type);
  101.74 @@ -489,9 +493,6 @@
  101.75  
  101.76    set_ctrl(transform( new (C, 1) ProjNode(call,TypeFunc::Control) ));
  101.77  
  101.78 -  // Set the incoming clone of memory as current memory
  101.79 -  set_all_memory(cloned_mem);
  101.80 -
  101.81    // Make memory for the call
  101.82    Node* mem = _gvn.transform( new (C, 1) ProjNode(call, TypeFunc::Memory) );
  101.83

   102.1 --- a/src/share/vm/opto/idealKit.hpp	Mon Jul 27 09:06:22 2009 -0700
   102.2 +++ b/src/share/vm/opto/idealKit.hpp	Mon Jul 27 17:23:52 2009 -0400
   102.3 @@ -49,7 +49,7 @@
   102.4  // Example:
   102.5  //    Node* limit = ??
   102.6  //    IdealVariable i(kit), j(kit);
   102.7 -//    declares_done();
   102.8 +//    declarations_done();
   102.9  //    Node* exit = make_label(1); // 1 goto
  102.10  //    set(j, ConI(0));
  102.11  //    loop(i, ConI(0), BoolTest::lt, limit); {
  102.12 @@ -101,10 +101,7 @@
  102.13    Node* new_cvstate();                     // Create a new cvstate
  102.14    Node* cvstate() { return _cvstate; }     // current cvstate
  102.15    Node* copy_cvstate();                    // copy current cvstate
  102.16 -  void set_ctrl(Node* ctrl) { _cvstate->set_req(TypeFunc::Control, ctrl); }
  102.17  
  102.18 -  // Should this assert this is a MergeMem???
  102.19 -  void set_all_memory(Node* mem){ _cvstate->set_req(TypeFunc::Memory, mem); }
  102.20    void set_memory(Node* mem, uint alias_idx );
  102.21    void do_memory_merge(Node* merging, Node* join);
  102.22    void clear(Node* m);                     // clear a cvstate
  102.23 @@ -132,15 +129,17 @@
  102.24    Node* memory(uint alias_idx);
  102.25  
  102.26   public:
  102.27 -  IdealKit(PhaseGVN &gvn, Node* control, Node* memory, bool delay_all_transforms = false);
  102.28 +  IdealKit(PhaseGVN &gvn, Node* control, Node* memory, bool delay_all_transforms = false, bool has_declarations = false);
  102.29    ~IdealKit() {
  102.30      stop();
  102.31      drain_delay_transform();
  102.32    }
  102.33    // Control
  102.34    Node* ctrl()                          { return _cvstate->in(TypeFunc::Control); }
  102.35 +  void set_ctrl(Node* ctrl)             { _cvstate->set_req(TypeFunc::Control, ctrl); }
  102.36    Node* top()                           { return C->top(); }
  102.37    MergeMemNode* merged_memory()         { return _cvstate->in(TypeFunc::Memory)->as_MergeMem(); }
  102.38 +  void set_all_memory(Node* mem)        { _cvstate->set_req(TypeFunc::Memory, mem); }
  102.39    void set(IdealVariable& v, Node* rhs) { _cvstate->set_req(first_var + v.id(), rhs); }
  102.40    Node* value(IdealVariable& v)         { return _cvstate->in(first_var + v.id()); }
  102.41    void dead(IdealVariable& v)           { set(v, (Node*)NULL); }
  102.42 @@ -155,7 +154,7 @@
  102.43    Node* make_label(int goto_ct);
  102.44    void bind(Node* lab);
  102.45    void goto_(Node* lab, bool bind = false);
  102.46 -  void declares_done();
  102.47 +  void declarations_done();
  102.48    void drain_delay_transform();
  102.49  
  102.50    Node* IfTrue(IfNode* iff)  { return transform(new (C,1) IfTrueNode(iff)); }

   103.1 --- a/src/share/vm/opto/ifnode.cpp	Mon Jul 27 09:06:22 2009 -0700
   103.2 +++ b/src/share/vm/opto/ifnode.cpp	Mon Jul 27 17:23:52 2009 -0400
   103.3 @@ -378,7 +378,18 @@
   103.4  
   103.5    // Force the original merge dead
   103.6    igvn->hash_delete(r);
   103.7 -  r->set_req_X(0,NULL,igvn);
   103.8 +  // First, remove region's dead users.
   103.9 +  for (DUIterator_Last lmin, l = r->last_outs(lmin); l >= lmin;) {
  103.10 +    Node* u = r->last_out(l);
  103.11 +    if( u == r ) {
  103.12 +      r->set_req(0, NULL);
  103.13 +    } else {
  103.14 +      assert(u->outcnt() == 0, "only dead users");
  103.15 +      igvn->remove_dead_node(u);
  103.16 +    }
  103.17 +    l -= 1;
  103.18 +  }
  103.19 +  igvn->remove_dead_node(r);
  103.20  
  103.21    // Now remove the bogus extra edges used to keep things alive
  103.22    igvn->remove_dead_node( hook );

   104.1 --- a/src/share/vm/opto/library_call.cpp	Mon Jul 27 09:06:22 2009 -0700
   104.2 +++ b/src/share/vm/opto/library_call.cpp	Mon Jul 27 17:23:52 2009 -0400
   104.3 @@ -165,6 +165,7 @@
   104.4    bool inline_native_getLength();
   104.5    bool inline_array_copyOf(bool is_copyOfRange);
   104.6    bool inline_array_equals();
   104.7 +  void copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark);
   104.8    bool inline_native_clone(bool is_virtual);
   104.9    bool inline_native_Reflection_getCallerClass();
  104.10    bool inline_native_AtomicLong_get();
  104.11 @@ -181,7 +182,6 @@
  104.12                            Node* src,  Node* src_offset,
  104.13                            Node* dest, Node* dest_offset,
  104.14                            Node* copy_length,
  104.15 -                          int nargs,  // arguments on stack for debug info
  104.16                            bool disjoint_bases = false,
  104.17                            bool length_never_negative = false,
  104.18                            RegionNode* slow_region = NULL);
  104.19 @@ -202,17 +202,16 @@
  104.20    void generate_slow_arraycopy(const TypePtr* adr_type,
  104.21                                 Node* src,  Node* src_offset,
  104.22                                 Node* dest, Node* dest_offset,
  104.23 -                               Node* copy_length,
  104.24 -                               int nargs);
  104.25 +                               Node* copy_length);
  104.26    Node* generate_checkcast_arraycopy(const TypePtr* adr_type,
  104.27                                       Node* dest_elem_klass,
  104.28                                       Node* src,  Node* src_offset,
  104.29                                       Node* dest, Node* dest_offset,
  104.30 -                                     Node* copy_length, int nargs);
  104.31 +                                     Node* copy_length);
  104.32    Node* generate_generic_arraycopy(const TypePtr* adr_type,
  104.33                                     Node* src,  Node* src_offset,
  104.34                                     Node* dest, Node* dest_offset,
  104.35 -                                   Node* copy_length, int nargs);
  104.36 +                                   Node* copy_length);
  104.37    void generate_unchecked_arraycopy(const TypePtr* adr_type,
  104.38                                      BasicType basic_elem_type,
  104.39                                      bool disjoint_bases,
  104.40 @@ -311,11 +310,6 @@
  104.41      if (!InlineAtomicLong)  return NULL;
  104.42      break;
  104.43  
  104.44 -  case vmIntrinsics::_Object_init:
  104.45 -  case vmIntrinsics::_invoke:
  104.46 -    // We do not intrinsify these; they are marked for other purposes.
  104.47 -    return NULL;
  104.48 -
  104.49    case vmIntrinsics::_getCallerClass:
  104.50      if (!UseNewReflection)  return NULL;
  104.51      if (!InlineReflectionGetCallerClass)  return NULL;
  104.52 @@ -328,6 +322,8 @@
  104.53      break;
  104.54  
  104.55   default:
  104.56 +    assert(id <= vmIntrinsics::LAST_COMPILER_INLINE, "caller responsibility");
  104.57 +    assert(id != vmIntrinsics::_Object_init && id != vmIntrinsics::_invoke, "enum out of order?");
  104.58      break;
  104.59    }
  104.60  
  104.61 @@ -395,18 +391,11 @@
  104.62    }
  104.63  
  104.64    if (PrintIntrinsics) {
  104.65 -    switch (intrinsic_id()) {
  104.66 -    case vmIntrinsics::_invoke:
  104.67 -    case vmIntrinsics::_Object_init:
  104.68 -      // We do not expect to inline these, so do not produce any noise about them.
  104.69 -      break;
  104.70 -    default:
  104.71 -      tty->print("Did not inline intrinsic %s%s at bci:%d in",
  104.72 -                 vmIntrinsics::name_at(intrinsic_id()),
  104.73 -                 (is_virtual() ? " (virtual)" : ""), kit.bci());
  104.74 -      kit.caller()->print_short_name(tty);
  104.75 -      tty->print_cr(" (%d bytes)", kit.caller()->code_size());
  104.76 -    }
  104.77 +    tty->print("Did not inline intrinsic %s%s at bci:%d in",
  104.78 +               vmIntrinsics::name_at(intrinsic_id()),
  104.79 +               (is_virtual() ? " (virtual)" : ""), kit.bci());
  104.80 +    kit.caller()->print_short_name(tty);
  104.81 +    tty->print_cr(" (%d bytes)", kit.caller()->code_size());
  104.82    }
  104.83    C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed);
  104.84    return NULL;
  104.85 @@ -1031,7 +1020,7 @@
  104.86    const TypeAry* target_array_type = TypeAry::make(TypeInt::CHAR, TypeInt::make(0, target_length, Type::WidenMin));
  104.87    const TypeAryPtr* target_type = TypeAryPtr::make(TypePtr::BotPTR, target_array_type, target_array->klass(), true, Type::OffsetBot);
  104.88  
  104.89 -  IdealKit kit(gvn(), control(), merged_memory());
  104.90 +  IdealKit kit(gvn(), control(), merged_memory(), false, true);
  104.91  #define __ kit.
  104.92    Node* zero             = __ ConI(0);
  104.93    Node* one              = __ ConI(1);
  104.94 @@ -1043,7 +1032,7 @@
  104.95    Node* targetOffset     = __ ConI(targetOffset_i);
  104.96    Node* sourceEnd        = __ SubI(__ AddI(sourceOffset, sourceCount), targetCountLess1);
  104.97  
  104.98 -  IdealVariable rtn(kit), i(kit), j(kit); __ declares_done();
  104.99 +  IdealVariable rtn(kit), i(kit), j(kit); __ declarations_done();
 104.100    Node* outer_loop = __ make_label(2 /* goto */);
 104.101    Node* return_    = __ make_label(1);
 104.102  
 104.103 @@ -1080,9 +1069,9 @@
 104.104         __ bind(outer_loop);
 104.105    }__ end_loop(); __ dead(i);
 104.106    __ bind(return_);
 104.107 -  __ drain_delay_transform();
 104.108 -
 104.109 -  set_control(__ ctrl());
 104.110 +
 104.111 +  // Final sync IdealKit and GraphKit.
 104.112 +  sync_kit(kit);
 104.113    Node* result = __ value(rtn);
 104.114  #undef __
 104.115    C->set_has_loops(true);
 104.116 @@ -2178,21 +2167,29 @@
 104.117        // Possibly an oop being stored to Java heap or native memory
 104.118        if (!TypePtr::NULL_PTR->higher_equal(_gvn.type(heap_base_oop))) {
 104.119          // oop to Java heap.
 104.120 -        (void) store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, val->bottom_type(), type);
 104.121 +        (void) store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type);
 104.122        } else {
 104.123 -
 104.124          // We can't tell at compile time if we are storing in the Java heap or outside
 104.125          // of it. So we need to emit code to conditionally do the proper type of
 104.126          // store.
 104.127  
 104.128 -        IdealKit kit(gvn(), control(),  merged_memory());
 104.129 -        kit.declares_done();
 104.130 +        IdealKit ideal(gvn(), control(),  merged_memory());
 104.131 +#define __ ideal.
 104.132          // QQQ who knows what probability is here??
 104.133 -        kit.if_then(heap_base_oop, BoolTest::ne, null(), PROB_UNLIKELY(0.999)); {
 104.134 -          (void) store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, val->bottom_type(), type);
 104.135 -        } kit.else_(); {
 104.136 -          (void) store_to_memory(control(), adr, val, type, adr_type, is_volatile);
 104.137 -        } kit.end_if();
 104.138 +        __ if_then(heap_base_oop, BoolTest::ne, null(), PROB_UNLIKELY(0.999)); {
 104.139 +          // Sync IdealKit and graphKit.
 104.140 +          set_all_memory( __ merged_memory());
 104.141 +          set_control(__ ctrl());
 104.142 +          Node* st = store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type);
 104.143 +          // Update IdealKit memory.
 104.144 +          __ set_all_memory(merged_memory());
 104.145 +          __ set_ctrl(control());
 104.146 +        } __ else_(); {
 104.147 +          __ store(__ ctrl(), adr, val, type, alias_type->index(), is_volatile);
 104.148 +        } __ end_if();
 104.149 +        // Final sync IdealKit and GraphKit.
 104.150 +        sync_kit(ideal);
 104.151 +#undef __
 104.152        }
 104.153      }
 104.154    }
 104.155 @@ -2394,7 +2391,7 @@
 104.156    case T_OBJECT:
 104.157       // reference stores need a store barrier.
 104.158      // (They don't if CAS fails, but it isn't worth checking.)
 104.159 -    pre_barrier(control(), base, adr, alias_idx, newval, value_type, T_OBJECT);
 104.160 +    pre_barrier(control(), base, adr, alias_idx, newval, value_type->make_oopptr(), T_OBJECT);
 104.161  #ifdef _LP64
 104.162      if (adr->bottom_type()->is_ptr_to_narrowoop()) {
 104.163        Node *newval_enc = _gvn.transform(new (C, 2) EncodePNode(newval, newval->bottom_type()->make_narrowoop()));
 104.164 @@ -2489,7 +2486,7 @@
 104.165    bool require_atomic_access = true;
 104.166    Node* store;
 104.167    if (type == T_OBJECT) // reference stores need a store barrier.
 104.168 -    store = store_oop_to_unknown(control(), base, adr, adr_type, val, value_type, type);
 104.169 +    store = store_oop_to_unknown(control(), base, adr, adr_type, val, type);
 104.170    else {
 104.171      store = store_to_memory(control(), adr, val, type, adr_type, require_atomic_access);
 104.172    }
 104.173 @@ -3230,7 +3227,8 @@
 104.174      Node* orig_tail = _gvn.transform( new(C, 3) SubINode(orig_length, start) );
 104.175      Node* moved = generate_min_max(vmIntrinsics::_min, orig_tail, length);
 104.176  
 104.177 -    Node* newcopy = new_array(klass_node, length, nargs);
 104.178 +    const bool raw_mem_only = true;
 104.179 +    Node* newcopy = new_array(klass_node, length, nargs, raw_mem_only);
 104.180  
 104.181      // Generate a direct call to the right arraycopy function(s).
 104.182      // We know the copy is disjoint but we might not know if the
 104.183 @@ -3241,7 +3239,7 @@
 104.184      bool length_never_negative = true;
 104.185      generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT,
 104.186                         original, start, newcopy, intcon(0), moved,
 104.187 -                       nargs, disjoint_bases, length_never_negative);
 104.188 +                       disjoint_bases, length_never_negative);
 104.189  
 104.190      push(newcopy);
 104.191    }
 104.192 @@ -3883,6 +3881,98 @@
 104.193    return true;
 104.194  }
 104.195  
 104.196 +//------------------------clone_coping-----------------------------------
 104.197 +// Helper function for inline_native_clone.
 104.198 +void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark) {
 104.199 +  assert(obj_size != NULL, "");
 104.200 +  Node* raw_obj = alloc_obj->in(1);
 104.201 +  assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), "");
 104.202 +
 104.203 +  if (ReduceBulkZeroing) {
 104.204 +    // We will be completely responsible for initializing this object -
 104.205 +    // mark Initialize node as complete.
 104.206 +    AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn);
 104.207 +    // The object was just allocated - there should be no any stores!
 104.208 +    guarantee(alloc != NULL && alloc->maybe_set_complete(&_gvn), "");
 104.209 +  }
 104.210 +
 104.211 +  // Cast to Object for arraycopy.
 104.212 +  // We can't use the original CheckCastPP since it should be moved
 104.213 +  // after the arraycopy to prevent stores flowing above it.
 104.214 +  Node* new_obj = new(C, 2) CheckCastPPNode(alloc_obj->in(0), raw_obj,
 104.215 +                                            TypeInstPtr::NOTNULL);
 104.216 +  new_obj = _gvn.transform(new_obj);
 104.217 +  // Substitute in the locally valid dest_oop.
 104.218 +  replace_in_map(alloc_obj, new_obj);
 104.219 +
 104.220 +  // Copy the fastest available way.
 104.221 +  // TODO: generate fields copies for small objects instead.
 104.222 +  Node* src  = obj;
 104.223 +  Node* dest = new_obj;
 104.224 +  Node* size = _gvn.transform(obj_size);
 104.225 +
 104.226 +  // Exclude the header but include array length to copy by 8 bytes words.
 104.227 +  // Can't use base_offset_in_bytes(bt) since basic type is unknown.
 104.228 +  int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() :
 104.229 +                            instanceOopDesc::base_offset_in_bytes();
 104.230 +  // base_off:
 104.231 +  // 8  - 32-bit VM
 104.232 +  // 12 - 64-bit VM, compressed oops
 104.233 +  // 16 - 64-bit VM, normal oops
 104.234 +  if (base_off % BytesPerLong != 0) {
 104.235 +    assert(UseCompressedOops, "");
 104.236 +    if (is_array) {
 104.237 +      // Exclude length to copy by 8 bytes words.
 104.238 +      base_off += sizeof(int);
 104.239 +    } else {
 104.240 +      // Include klass to copy by 8 bytes words.
 104.241 +      base_off = instanceOopDesc::klass_offset_in_bytes();
 104.242 +    }
 104.243 +    assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
 104.244 +  }
 104.245 +  src  = basic_plus_adr(src,  base_off);
 104.246 +  dest = basic_plus_adr(dest, base_off);
 104.247 +
 104.248 +  // Compute the length also, if needed:
 104.249 +  Node* countx = size;
 104.250 +  countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(base_off)) );
 104.251 +  countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong) ));
 104.252 +
 104.253 +  const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
 104.254 +  bool disjoint_bases = true;
 104.255 +  generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases,
 104.256 +                               src, NULL, dest, NULL, countx);
 104.257 +
 104.258 +  // If necessary, emit some card marks afterwards.  (Non-arrays only.)
 104.259 +  if (card_mark) {
 104.260 +    assert(!is_array, "");
 104.261 +    // Put in store barrier for any and all oops we are sticking
 104.262 +    // into this object.  (We could avoid this if we could prove
 104.263 +    // that the object type contains no oop fields at all.)
 104.264 +    Node* no_particular_value = NULL;
 104.265 +    Node* no_particular_field = NULL;
 104.266 +    int raw_adr_idx = Compile::AliasIdxRaw;
 104.267 +    post_barrier(control(),
 104.268 +                 memory(raw_adr_type),
 104.269 +                 new_obj,
 104.270 +                 no_particular_field,
 104.271 +                 raw_adr_idx,
 104.272 +                 no_particular_value,
 104.273 +                 T_OBJECT,
 104.274 +                 false);
 104.275 +  }
 104.276 +
 104.277 +  // Move the original CheckCastPP after arraycopy.
 104.278 +  _gvn.hash_delete(alloc_obj);
 104.279 +  alloc_obj->set_req(0, control());
 104.280 +  // Replace raw memory edge with new CheckCastPP to have a live oop
 104.281 +  // at safepoints instead of raw value.
 104.282 +  assert(new_obj->is_CheckCastPP() && new_obj->in(1) == alloc_obj->in(1), "sanity");
 104.283 +  alloc_obj->set_req(1, new_obj);    // cast to the original type
 104.284 +  _gvn.hash_find_insert(alloc_obj);  // put back into GVN table
 104.285 +  // Restore in the locally valid dest_oop.
 104.286 +  replace_in_map(new_obj, alloc_obj);
 104.287 +}
 104.288  
 104.289  //------------------------inline_native_clone----------------------------
 104.290  // Here are the simple edge cases:
 104.291 @@ -3917,8 +4007,9 @@
 104.292    // paths into result_reg:
 104.293    enum {
 104.294      _slow_path = 1,     // out-of-line call to clone method (virtual or not)
 104.295 -    _objArray_path,     // plain allocation, plus arrayof_oop_arraycopy
 104.296 -    _fast_path,         // plain allocation, plus a CopyArray operation
 104.297 +    _objArray_path,     // plain array allocation, plus arrayof_oop_arraycopy
 104.298 +    _array_path,        // plain array allocation, plus arrayof_long_arraycopy
 104.299 +    _instance_path,     // plain instance allocation, plus arrayof_long_arraycopy
 104.300      PATH_LIMIT
 104.301    };
 104.302    RegionNode* result_reg = new(C, PATH_LIMIT) RegionNode(PATH_LIMIT);
 104.303 @@ -3933,18 +4024,6 @@
 104.304    int raw_adr_idx = Compile::AliasIdxRaw;
 104.305    const bool raw_mem_only = true;
 104.306  
 104.307 -  // paths into alloc_reg (on the fast path, just before the CopyArray):
 104.308 -  enum { _typeArray_alloc = 1, _instance_alloc, ALLOC_LIMIT };
 104.309 -  RegionNode* alloc_reg = new(C, ALLOC_LIMIT) RegionNode(ALLOC_LIMIT);
 104.310 -  PhiNode*    alloc_val = new(C, ALLOC_LIMIT) PhiNode(alloc_reg, raw_adr_type);
 104.311 -  PhiNode*    alloc_siz = new(C, ALLOC_LIMIT) PhiNode(alloc_reg, TypeX_X);
 104.312 -  PhiNode*    alloc_i_o = new(C, ALLOC_LIMIT) PhiNode(alloc_reg, Type::ABIO);
 104.313 -  PhiNode*    alloc_mem = new(C, ALLOC_LIMIT) PhiNode(alloc_reg, Type::MEMORY,
 104.314 -                                                      raw_adr_type);
 104.315 -  record_for_igvn(alloc_reg);
 104.316 -
 104.317 -  bool card_mark = false;  // (see below)
 104.318 -
 104.319    Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL);
 104.320    if (array_ctl != NULL) {
 104.321      // It's an array.
 104.322 @@ -3954,16 +4033,6 @@
 104.323      Node* obj_size = NULL;
 104.324      Node* alloc_obj = new_array(obj_klass, obj_length, nargs,
 104.325                                  raw_mem_only, &obj_size);
 104.326 -    assert(obj_size != NULL, "");
 104.327 -    Node* raw_obj = alloc_obj->in(1);
 104.328 -    assert(raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), "");
 104.329 -    if (ReduceBulkZeroing) {
 104.330 -      AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn);
 104.331 -      if (alloc != NULL) {
 104.332 -        // We will be completely responsible for initializing this object.
 104.333 -        alloc->maybe_set_complete(&_gvn);
 104.334 -      }
 104.335 -    }
 104.336  
 104.337      if (!use_ReduceInitialCardMarks()) {
 104.338        // If it is an oop array, it requires very special treatment,
 104.339 @@ -3977,7 +4046,7 @@
 104.340          bool length_never_negative = true;
 104.341          generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT,
 104.342                             obj, intcon(0), alloc_obj, intcon(0),
 104.343 -                           obj_length, nargs,
 104.344 +                           obj_length,
 104.345                             disjoint_bases, length_never_negative);
 104.346          result_reg->init_req(_objArray_path, control());
 104.347          result_val->init_req(_objArray_path, alloc_obj);
 104.348 @@ -3992,19 +4061,24 @@
 104.349      // the object.
 104.350  
 104.351      // Otherwise, there are no card marks to worry about.
 104.352 -    alloc_val->init_req(_typeArray_alloc, raw_obj);
 104.353 -    alloc_siz->init_req(_typeArray_alloc, obj_size);
 104.354 -    alloc_reg->init_req(_typeArray_alloc, control());
 104.355 -    alloc_i_o->init_req(_typeArray_alloc, i_o());
 104.356 -    alloc_mem->init_req(_typeArray_alloc, memory(raw_adr_type));
 104.357 +
 104.358 +    if (!stopped()) {
 104.359 +      copy_to_clone(obj, alloc_obj, obj_size, true, false);
 104.360 +
 104.361 +      // Present the results of the copy.
 104.362 +      result_reg->init_req(_array_path, control());
 104.363 +      result_val->init_req(_array_path, alloc_obj);
 104.364 +      result_i_o ->set_req(_array_path, i_o());
 104.365 +      result_mem ->set_req(_array_path, reset_memory());
 104.366 +    }
 104.367    }
 104.368  
 104.369 -  // We only go to the fast case code if we pass a number of guards.
 104.370 +  // We only go to the instance fast case code if we pass a number of guards.
 104.371    // The paths which do not pass are accumulated in the slow_region.
 104.372    RegionNode* slow_region = new (C, 1) RegionNode(1);
 104.373    record_for_igvn(slow_region);
 104.374    if (!stopped()) {
 104.375 -    // It's an instance.  Make the slow-path tests.
 104.376 +    // It's an instance (we did array above).  Make the slow-path tests.
 104.377      // If this is a virtual call, we generate a funny guard.  We grab
 104.378      // the vtable entry corresponding to clone() from the target object.
 104.379      // If the target method which we are calling happens to be the
 104.380 @@ -4031,25 +4105,14 @@
 104.381      PreserveJVMState pjvms(this);
 104.382      Node* obj_size = NULL;
 104.383      Node* alloc_obj = new_instance(obj_klass, NULL, raw_mem_only, &obj_size);
 104.384 -    assert(obj_size != NULL, "");
 104.385 -    Node* raw_obj = alloc_obj->in(1);
 104.386 -    assert(raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), "");
 104.387 -    if (ReduceBulkZeroing) {
 104.388 -      AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn);
 104.389 -      if (alloc != NULL && !alloc->maybe_set_complete(&_gvn))
 104.390 -        alloc = NULL;
 104.391 -    }
 104.392 -    if (!use_ReduceInitialCardMarks()) {
 104.393 -      // Put in store barrier for any and all oops we are sticking
 104.394 -      // into this object.  (We could avoid this if we could prove
 104.395 -      // that the object type contains no oop fields at all.)
 104.396 -      card_mark = true;
 104.397 -    }
 104.398 -    alloc_val->init_req(_instance_alloc, raw_obj);
 104.399 -    alloc_siz->init_req(_instance_alloc, obj_size);
 104.400 -    alloc_reg->init_req(_instance_alloc, control());
 104.401 -    alloc_i_o->init_req(_instance_alloc, i_o());
 104.402 -    alloc_mem->init_req(_instance_alloc, memory(raw_adr_type));
 104.403 +
 104.404 +    copy_to_clone(obj, alloc_obj, obj_size, false, !use_ReduceInitialCardMarks());
 104.405 +
 104.406 +    // Present the results of the slow call.
 104.407 +    result_reg->init_req(_instance_path, control());
 104.408 +    result_val->init_req(_instance_path, alloc_obj);
 104.409 +    result_i_o ->set_req(_instance_path, i_o());
 104.410 +    result_mem ->set_req(_instance_path, reset_memory());
 104.411    }
 104.412  
 104.413    // Generate code for the slow case.  We make a call to clone().
 104.414 @@ -4065,82 +4128,12 @@
 104.415      result_mem ->set_req(_slow_path, reset_memory());
 104.416    }
 104.417  
 104.418 -  // The object is allocated, as an array and/or an instance.  Now copy it.
 104.419 -  set_control( _gvn.transform(alloc_reg) );
 104.420 -  set_i_o(     _gvn.transform(alloc_i_o) );
 104.421 -  set_memory(  _gvn.transform(alloc_mem), raw_adr_type );
 104.422 -  Node* raw_obj  = _gvn.transform(alloc_val);
 104.423 -
 104.424 -  if (!stopped()) {
 104.425 -    // Copy the fastest available way.
 104.426 -    // (No need for PreserveJVMState, since we're using it all up now.)
 104.427 -    // TODO: generate fields/elements copies for small objects instead.
 104.428 -    Node* src  = obj;
 104.429 -    Node* dest = raw_obj;
 104.430 -    Node* size = _gvn.transform(alloc_siz);
 104.431 -
 104.432 -    // Exclude the header.
 104.433 -    int base_off = instanceOopDesc::base_offset_in_bytes();
 104.434 -    if (UseCompressedOops) {
 104.435 -      assert(base_off % BytesPerLong != 0, "base with compressed oops");
 104.436 -      // With compressed oops base_offset_in_bytes is 12 which creates
 104.437 -      // the gap since countx is rounded by 8 bytes below.
 104.438 -      // Copy klass and the gap.
 104.439 -      base_off = instanceOopDesc::klass_offset_in_bytes();
 104.440 -    }
 104.441 -    src  = basic_plus_adr(src,  base_off);
 104.442 -    dest = basic_plus_adr(dest, base_off);
 104.443 -
 104.444 -    // Compute the length also, if needed:
 104.445 -    Node* countx = size;
 104.446 -    countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(base_off)) );
 104.447 -    countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong) ));
 104.448 -
 104.449 -    // Select an appropriate instruction to initialize the range.
 104.450 -    // The CopyArray instruction (if supported) can be optimized
 104.451 -    // into a discrete set of scalar loads and stores.
 104.452 -    bool disjoint_bases = true;
 104.453 -    generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases,
 104.454 -                                 src, NULL, dest, NULL, countx);
 104.455 -
 104.456 -    // Now that the object is properly initialized, type it as an oop.
 104.457 -    // Use a secondary InitializeNode memory barrier.
 104.458 -    InitializeNode* init = insert_mem_bar_volatile(Op_Initialize, raw_adr_idx,
 104.459 -                                                   raw_obj)->as_Initialize();
 104.460 -    init->set_complete(&_gvn);  // (there is no corresponding AllocateNode)
 104.461 -    Node* new_obj = new(C, 2) CheckCastPPNode(control(), raw_obj,
 104.462 -                                              TypeInstPtr::NOTNULL);
 104.463 -    new_obj = _gvn.transform(new_obj);
 104.464 -
 104.465 -    // If necessary, emit some card marks afterwards.  (Non-arrays only.)
 104.466 -    if (card_mark) {
 104.467 -      Node* no_particular_value = NULL;
 104.468 -      Node* no_particular_field = NULL;
 104.469 -      post_barrier(control(),
 104.470 -                   memory(raw_adr_type),
 104.471 -                   new_obj,
 104.472 -                   no_particular_field,
 104.473 -                   raw_adr_idx,
 104.474 -                   no_particular_value,
 104.475 -                   T_OBJECT,
 104.476 -                   false);
 104.477 -    }
 104.478 -    // Present the results of the slow call.
 104.479 -    result_reg->init_req(_fast_path, control());
 104.480 -    result_val->init_req(_fast_path, new_obj);
 104.481 -    result_i_o ->set_req(_fast_path, i_o());
 104.482 -    result_mem ->set_req(_fast_path, reset_memory());
 104.483 -  }
 104.484 -
 104.485    // Return the combined state.
 104.486    set_control(    _gvn.transform(result_reg) );
 104.487    set_i_o(        _gvn.transform(result_i_o) );
 104.488    set_all_memory( _gvn.transform(result_mem) );
 104.489  
 104.490 -  // Cast the result to a sharper type, since we know what clone does.
 104.491 -  Node* new_obj = _gvn.transform(result_val);
 104.492 -  Node* cast    = new (C, 2) CheckCastPPNode(control(), new_obj, toop);
 104.493 -  push(_gvn.transform(cast));
 104.494 +  push(_gvn.transform(result_val));
 104.495  
 104.496    return true;
 104.497  }
 104.498 @@ -4279,8 +4272,7 @@
 104.499  
 104.500      // Call StubRoutines::generic_arraycopy stub.
 104.501      generate_arraycopy(TypeRawPtr::BOTTOM, T_CONFLICT,
 104.502 -                       src, src_offset, dest, dest_offset, length,
 104.503 -                       nargs);
 104.504 +                       src, src_offset, dest, dest_offset, length);
 104.505  
 104.506      // Do not let reads from the destination float above the arraycopy.
 104.507      // Since we cannot type the arrays, we don't know which slices
 104.508 @@ -4303,8 +4295,7 @@
 104.509      // The component types are not the same or are not recognized.  Punt.
 104.510      // (But, avoid the native method wrapper to JVM_ArrayCopy.)
 104.511      generate_slow_arraycopy(TypePtr::BOTTOM,
 104.512 -                            src, src_offset, dest, dest_offset, length,
 104.513 -                            nargs);
 104.514 +                            src, src_offset, dest, dest_offset, length);
 104.515      return true;
 104.516    }
 104.517  
 104.518 @@ -4361,7 +4352,7 @@
 104.519    const TypePtr* adr_type = TypeAryPtr::get_array_body_type(dest_elem);
 104.520    generate_arraycopy(adr_type, dest_elem,
 104.521                       src, src_offset, dest, dest_offset, length,
 104.522 -                     nargs, false, false, slow_region);
 104.523 +                     false, false, slow_region);
 104.524  
 104.525    return true;
 104.526  }
 104.527 @@ -4406,7 +4397,6 @@
 104.528                                     Node* src,  Node* src_offset,
 104.529                                     Node* dest, Node* dest_offset,
 104.530                                     Node* copy_length,
 104.531 -                                   int nargs,
 104.532                                     bool disjoint_bases,
 104.533                                     bool length_never_negative,
 104.534                                     RegionNode* slow_region) {
 104.535 @@ -4418,7 +4408,6 @@
 104.536  
 104.537    Node* original_dest      = dest;
 104.538    AllocateArrayNode* alloc = NULL;  // used for zeroing, if needed
 104.539 -  Node* raw_dest           = NULL;  // used before zeroing, if needed
 104.540    bool  must_clear_dest    = false;
 104.541  
 104.542    // See if this is the initialization of a newly-allocated array.
 104.543 @@ -4437,15 +4426,18 @@
 104.544      // "You break it, you buy it."
 104.545      InitializeNode* init = alloc->initialization();
 104.546      assert(init->is_complete(), "we just did this");
 104.547 -    assert(dest->Opcode() == Op_CheckCastPP, "sanity");
 104.548 +    assert(dest->is_CheckCastPP(), "sanity");
 104.549      assert(dest->in(0)->in(0) == init, "dest pinned");
 104.550 -    raw_dest = dest->in(1);  // grab the raw pointer!
 104.551 -    original_dest = dest;
 104.552 -    dest = raw_dest;
 104.553 +
 104.554 +    // Cast to Object for arraycopy.
 104.555 +    // We can't use the original CheckCastPP since it should be moved
 104.556 +    // after the arraycopy to prevent stores flowing above it.
 104.557 +    Node* new_obj = new(C, 2) CheckCastPPNode(dest->in(0), dest->in(1),
 104.558 +                                              TypeInstPtr::NOTNULL);
 104.559 +    dest = _gvn.transform(new_obj);
 104.560 +    // Substitute in the locally valid dest_oop.
 104.561 +    replace_in_map(original_dest, dest);
 104.562      adr_type = TypeRawPtr::BOTTOM;  // all initializations are into raw memory
 104.563 -    // Decouple the original InitializeNode, turning it into a simple membar.
 104.564 -    // We will build a new one at the end of this routine.
 104.565 -    init->set_req(InitializeNode::RawAddress, top());
 104.566      // From this point on, every exit path is responsible for
 104.567      // initializing any non-copied parts of the object to zero.
 104.568      must_clear_dest = true;
 104.569 @@ -4488,7 +4480,7 @@
 104.570      assert(!must_clear_dest, "");
 104.571      Node* cv = generate_generic_arraycopy(adr_type,
 104.572                                            src, src_offset, dest, dest_offset,
 104.573 -                                          copy_length, nargs);
 104.574 +                                          copy_length);
 104.575      if (cv == NULL)  cv = intcon(-1);  // failure (no stub available)
 104.576      checked_control = control();
 104.577      checked_i_o     = i_o();
 104.578 @@ -4507,16 +4499,24 @@
 104.579        generate_negative_guard(copy_length, slow_region);
 104.580      }
 104.581  
 104.582 +    // copy_length is 0.
 104.583      if (!stopped() && must_clear_dest) {
 104.584        Node* dest_length = alloc->in(AllocateNode::ALength);
 104.585        if (_gvn.eqv_uncast(copy_length, dest_length)
 104.586            || _gvn.find_int_con(dest_length, 1) <= 0) {
 104.587 -        // There is no zeroing to do.
 104.588 +        // There is no zeroing to do. No need for a secondary raw memory barrier.
 104.589        } else {
 104.590          // Clear the whole thing since there are no source elements to copy.
 104.591          generate_clear_array(adr_type, dest, basic_elem_type,
 104.592                               intcon(0), NULL,
 104.593                               alloc->in(AllocateNode::AllocSize));
 104.594 +        // Use a secondary InitializeNode as raw memory barrier.
 104.595 +        // Currently it is needed only on this path since other
 104.596 +        // paths have stub or runtime calls as raw memory barriers.
 104.597 +        InitializeNode* init = insert_mem_bar_volatile(Op_Initialize,
 104.598 +                                                       Compile::AliasIdxRaw,
 104.599 +                                                       top())->as_Initialize();
 104.600 +        init->set_complete(&_gvn);  // (there is no corresponding AllocateNode)
 104.601        }
 104.602      }
 104.603  
 104.604 @@ -4638,8 +4638,7 @@
 104.605        Node* cv = generate_checkcast_arraycopy(adr_type,
 104.606                                                dest_elem_klass,
 104.607                                                src, src_offset, dest, dest_offset,
 104.608 -                                              copy_length,
 104.609 -                                              nargs);
 104.610 +                                              copy_length);
 104.611        if (cv == NULL)  cv = intcon(-1);  // failure (no stub available)
 104.612        checked_control = control();
 104.613        checked_i_o     = i_o();
 104.614 @@ -4701,8 +4700,8 @@
 104.615      slow_i_o2  ->init_req(1, slow_i_o);
 104.616      slow_mem2  ->init_req(1, slow_mem);
 104.617      slow_reg2  ->init_req(2, control());
 104.618 -    slow_i_o2  ->init_req(2, i_o());
 104.619 -    slow_mem2  ->init_req(2, memory(adr_type));
 104.620 +    slow_i_o2  ->init_req(2, checked_i_o);
 104.621 +    slow_mem2  ->init_req(2, checked_mem);
 104.622  
 104.623      slow_control = _gvn.transform(slow_reg2);
 104.624      slow_i_o     = _gvn.transform(slow_i_o2);
 104.625 @@ -4747,21 +4746,9 @@
 104.626                             alloc->in(AllocateNode::AllocSize));
 104.627      }
 104.628  
 104.629 -    if (dest != original_dest) {
 104.630 -      // Promote from rawptr to oop, so it looks right in the call's GC map.
 104.631 -      dest = _gvn.transform( new(C,2) CheckCastPPNode(control(), dest,
 104.632 -                                                      TypeInstPtr::NOTNULL) );
 104.633 -
 104.634 -      // Edit the call's debug-info to avoid referring to original_dest.
 104.635 -      // (The problem with original_dest is that it isn't ready until
 104.636 -      // after the InitializeNode completes, but this stuff is before.)
 104.637 -      // Substitute in the locally valid dest_oop.
 104.638 -      replace_in_map(original_dest, dest);
 104.639 -    }
 104.640 -
 104.641      generate_slow_arraycopy(adr_type,
 104.642                              src, src_offset, dest, dest_offset,
 104.643 -                            copy_length, nargs);
 104.644 +                            copy_length);
 104.645  
 104.646      result_region->init_req(slow_call_path, control());
 104.647      result_i_o   ->init_req(slow_call_path, i_o());
 104.648 @@ -4781,16 +4768,16 @@
 104.649  
 104.650    if (dest != original_dest) {
 104.651      // Pin the "finished" array node after the arraycopy/zeroing operations.
 104.652 -    // Use a secondary InitializeNode memory barrier.
 104.653 -    InitializeNode* init = insert_mem_bar_volatile(Op_Initialize,
 104.654 -                                                   Compile::AliasIdxRaw,
 104.655 -                                                   raw_dest)->as_Initialize();
 104.656 -    init->set_complete(&_gvn);  // (there is no corresponding AllocateNode)
 104.657      _gvn.hash_delete(original_dest);
 104.658      original_dest->set_req(0, control());
 104.659 +    // Replace raw memory edge with new CheckCastPP to have a live oop
 104.660 +    // at safepoints instead of raw value.
 104.661 +    assert(dest->is_CheckCastPP() && dest->in(1) == original_dest->in(1), "sanity");
 104.662 +    original_dest->set_req(1, dest);       // cast to the original type
 104.663      _gvn.hash_find_insert(original_dest);  // put back into GVN table
 104.664 +    // Restore in the locally valid dest_oop.
 104.665 +    replace_in_map(dest, original_dest);
 104.666    }
 104.667 -
 104.668    // The memory edges above are precise in order to model effects around
 104.669    // array copies accurately to allow value numbering of field loads around
 104.670    // arraycopy.  Such field loads, both before and after, are common in Java
 104.671 @@ -5074,16 +5061,13 @@
 104.672  LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type,
 104.673                                          Node* src,  Node* src_offset,
 104.674                                          Node* dest, Node* dest_offset,
 104.675 -                                        Node* copy_length,
 104.676 -                                        int nargs) {
 104.677 -  _sp += nargs; // any deopt will start just before call to enclosing method
 104.678 +                                        Node* copy_length) {
 104.679    Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON,
 104.680                                   OptoRuntime::slow_arraycopy_Type(),
 104.681                                   OptoRuntime::slow_arraycopy_Java(),
 104.682                                   "slow_arraycopy", adr_type,
 104.683                                   src, src_offset, dest, dest_offset,
 104.684                                   copy_length);
 104.685 -  _sp -= nargs;
 104.686  
 104.687    // Handle exceptions thrown by this fellow:
 104.688    make_slow_call_ex(call, env()->Throwable_klass(), false);
 104.689 @@ -5095,8 +5079,7 @@
 104.690                                               Node* dest_elem_klass,
 104.691                                               Node* src,  Node* src_offset,
 104.692                                               Node* dest, Node* dest_offset,
 104.693 -                                             Node* copy_length,
 104.694 -                                             int nargs) {
 104.695 +                                             Node* copy_length) {
 104.696    if (stopped())  return NULL;
 104.697  
 104.698    address copyfunc_addr = StubRoutines::checkcast_arraycopy();
 104.699 @@ -5137,8 +5120,7 @@
 104.700  LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type,
 104.701                                             Node* src,  Node* src_offset,
 104.702                                             Node* dest, Node* dest_offset,
 104.703 -                                           Node* copy_length,
 104.704 -                                           int nargs) {
 104.705 +                                           Node* copy_length) {
 104.706    if (stopped())  return NULL;
 104.707  
 104.708    address copyfunc_addr = StubRoutines::generic_arraycopy();

   105.1 --- a/src/share/vm/opto/loopopts.cpp	Mon Jul 27 09:06:22 2009 -0700
   105.2 +++ b/src/share/vm/opto/loopopts.cpp	Mon Jul 27 17:23:52 2009 -0400
   105.3 @@ -346,7 +346,10 @@
   105.4  
   105.5      // Yes!  Reshape address expression!
   105.6      Node *inv_scale = new (C, 3) LShiftINode( add_invar, scale );
   105.7 -    register_new_node( inv_scale, add_invar_ctrl );
   105.8 +    Node *inv_scale_ctrl =
   105.9 +      dom_depth(add_invar_ctrl) > dom_depth(scale_ctrl) ?
  105.10 +      add_invar_ctrl : scale_ctrl;
  105.11 +    register_new_node( inv_scale, inv_scale_ctrl );
  105.12      Node *var_scale = new (C, 3) LShiftINode( add_var, scale );
  105.13      register_new_node( var_scale, n_ctrl );
  105.14      Node *var_add = new (C, 3) AddINode( var_scale, inv_scale );
  105.15 @@ -667,7 +670,6 @@
  105.16    }
  105.17  }
  105.18  
  105.19 -#ifdef _LP64
  105.20  static bool merge_point_safe(Node* region) {
  105.21    // 4799512: Stop split_if_with_blocks from splitting a block with a ConvI2LNode
  105.22    // having a PhiNode input. This sidesteps the dangerous case where the split
  105.23 @@ -676,20 +678,25 @@
  105.24    // uses.
  105.25    // A better fix for this problem can be found in the BugTraq entry, but
  105.26    // expediency for Mantis demands this hack.
  105.27 +  // 6855164: If the merge point has a FastLockNode with a PhiNode input, we stop
  105.28 +  // split_if_with_blocks from splitting a block because we could not move around
  105.29 +  // the FastLockNode.
  105.30    for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
  105.31      Node* n = region->fast_out(i);
  105.32      if (n->is_Phi()) {
  105.33        for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
  105.34          Node* m = n->fast_out(j);
  105.35 -        if (m->Opcode() == Op_ConvI2L) {
  105.36 +        if (m->is_FastLock())
  105.37            return false;
  105.38 -        }
  105.39 +#ifdef _LP64
  105.40 +        if (m->Opcode() == Op_ConvI2L)
  105.41 +          return false;
  105.42 +#endif
  105.43        }
  105.44      }
  105.45    }
  105.46    return true;
  105.47  }
  105.48 -#endif
  105.49  
  105.50  
  105.51  //------------------------------place_near_use---------------------------------
  105.52 @@ -771,12 +778,10 @@
  105.53        if( get_loop(n_ctrl->in(j)) != n_loop )
  105.54          return;
  105.55  
  105.56 -#ifdef _LP64
  105.57      // Check for safety of the merge point.
  105.58      if( !merge_point_safe(n_ctrl) ) {
  105.59        return;
  105.60      }
  105.61 -#endif
  105.62  
  105.63      // Split compare 'n' through the merge point if it is profitable
  105.64      Node *phi = split_thru_phi( n, n_ctrl, policy );

   106.1 --- a/src/share/vm/opto/machnode.cpp	Mon Jul 27 09:06:22 2009 -0700
   106.2 +++ b/src/share/vm/opto/machnode.cpp	Mon Jul 27 17:23:52 2009 -0400
   106.3 @@ -300,6 +300,12 @@
   106.4            }
   106.5          }
   106.6          adr_type = t_disp->add_offset(offset);
   106.7 +      } else if( base == NULL && offset != 0 && offset != Type::OffsetBot ) {
   106.8 +        // Use ideal type if it is oop ptr.
   106.9 +        const TypePtr *tp = oper->type()->isa_ptr();
  106.10 +        if( tp != NULL) {
  106.11 +          adr_type = tp;
  106.12 +        }
  106.13        }
  106.14      }
  106.15

   107.1 --- a/src/share/vm/opto/macro.cpp	Mon Jul 27 09:06:22 2009 -0700
   107.2 +++ b/src/share/vm/opto/macro.cpp	Mon Jul 27 17:23:52 2009 -0400
   107.3 @@ -198,14 +198,79 @@
   107.4  }
   107.5  
   107.6  // Eliminate a card mark sequence.  p2x is a ConvP2XNode
   107.7 -void PhaseMacroExpand::eliminate_card_mark(Node *p2x) {
   107.8 +void PhaseMacroExpand::eliminate_card_mark(Node* p2x) {
   107.9    assert(p2x->Opcode() == Op_CastP2X, "ConvP2XNode required");
  107.10 -  Node *shift = p2x->unique_out();
  107.11 -  Node *addp = shift->unique_out();
  107.12 -  for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
  107.13 -    Node *st = addp->last_out(j);
  107.14 -    assert(st->is_Store(), "store required");
  107.15 -    _igvn.replace_node(st, st->in(MemNode::Memory));
  107.16 +  if (!UseG1GC) {
  107.17 +    // vanilla/CMS post barrier
  107.18 +    Node *shift = p2x->unique_out();
  107.19 +    Node *addp = shift->unique_out();
  107.20 +    for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
  107.21 +      Node *st = addp->last_out(j);
  107.22 +      assert(st->is_Store(), "store required");
  107.23 +      _igvn.replace_node(st, st->in(MemNode::Memory));
  107.24 +    }
  107.25 +  } else {
  107.26 +    // G1 pre/post barriers
  107.27 +    assert(p2x->outcnt() == 2, "expects 2 users: Xor and URShift nodes");
  107.28 +    // It could be only one user, URShift node, in Object.clone() instrinsic
  107.29 +    // but the new allocation is passed to arraycopy stub and it could not
  107.30 +    // be scalar replaced. So we don't check the case.
  107.31 +
  107.32 +    // Remove G1 post barrier.
  107.33 +
  107.34 +    // Search for CastP2X->Xor->URShift->Cmp path which
  107.35 +    // checks if the store done to a different from the value's region.
  107.36 +    // And replace Cmp with #0 (false) to collapse G1 post barrier.
  107.37 +    Node* xorx = NULL;
  107.38 +    for (DUIterator_Fast imax, i = p2x->fast_outs(imax); i < imax; i++) {
  107.39 +      Node* u = p2x->fast_out(i);
  107.40 +      if (u->Opcode() == Op_XorX) {
  107.41 +        xorx = u;
  107.42 +        break;
  107.43 +      }
  107.44 +    }
  107.45 +    assert(xorx != NULL, "missing G1 post barrier");
  107.46 +    Node* shift = xorx->unique_out();
  107.47 +    Node* cmpx = shift->unique_out();
  107.48 +    assert(cmpx->is_Cmp() && cmpx->unique_out()->is_Bool() &&
  107.49 +    cmpx->unique_out()->as_Bool()->_test._test == BoolTest::ne,
  107.50 +    "missing region check in G1 post barrier");
  107.51 +    _igvn.replace_node(cmpx, makecon(TypeInt::CC_EQ));
  107.52 +
  107.53 +    // Remove G1 pre barrier.
  107.54 +
  107.55 +    // Search "if (marking != 0)" check and set it to "false".
  107.56 +    Node* this_region = p2x->in(0);
  107.57 +    assert(this_region != NULL, "");
  107.58 +    // There is no G1 pre barrier if previous stored value is NULL
  107.59 +    // (for example, after initialization).
  107.60 +    if (this_region->is_Region() && this_region->req() == 3) {
  107.61 +      int ind = 1;
  107.62 +      if (!this_region->in(ind)->is_IfFalse()) {
  107.63 +        ind = 2;
  107.64 +      }
  107.65 +      if (this_region->in(ind)->is_IfFalse()) {
  107.66 +        Node* bol = this_region->in(ind)->in(0)->in(1);
  107.67 +        assert(bol->is_Bool(), "");
  107.68 +        cmpx = bol->in(1);
  107.69 +        if (bol->as_Bool()->_test._test == BoolTest::ne &&
  107.70 +            cmpx->is_Cmp() && cmpx->in(2) == intcon(0) &&
  107.71 +            cmpx->in(1)->is_Load()) {
  107.72 +          Node* adr = cmpx->in(1)->as_Load()->in(MemNode::Address);
  107.73 +          const int marking_offset = in_bytes(JavaThread::satb_mark_queue_offset() +
  107.74 +                                              PtrQueue::byte_offset_of_active());
  107.75 +          if (adr->is_AddP() && adr->in(AddPNode::Base) == top() &&
  107.76 +              adr->in(AddPNode::Address)->Opcode() == Op_ThreadLocal &&
  107.77 +              adr->in(AddPNode::Offset) == MakeConX(marking_offset)) {
  107.78 +            _igvn.replace_node(cmpx, makecon(TypeInt::CC_EQ));
  107.79 +          }
  107.80 +        }
  107.81 +      }
  107.82 +    }
  107.83 +    // Now CastP2X can be removed since it is used only on dead path
  107.84 +    // which currently still alive until igvn optimize it.
  107.85 +    assert(p2x->unique_out()->Opcode() == Op_URShiftX, "");
  107.86 +    _igvn.replace_node(p2x, top());
  107.87    }
  107.88  }
  107.89  
  107.90 @@ -760,14 +825,11 @@
  107.91            if (n->is_Store()) {
  107.92              _igvn.replace_node(n, n->in(MemNode::Memory));
  107.93            } else {
  107.94 -            assert( n->Opcode() == Op_CastP2X, "CastP2X required");
  107.95              eliminate_card_mark(n);
  107.96            }
  107.97            k -= (oc2 - use->outcnt());
  107.98          }
  107.99        } else {
 107.100 -        assert( !use->is_SafePoint(), "safepoint uses must have been already elimiated");
 107.101 -        assert( use->Opcode() == Op_CastP2X, "CastP2X required");
 107.102          eliminate_card_mark(use);
 107.103        }
 107.104        j -= (oc1 - res->outcnt());

   108.1 --- a/src/share/vm/opto/matcher.cpp	Mon Jul 27 09:06:22 2009 -0700
   108.2 +++ b/src/share/vm/opto/matcher.cpp	Mon Jul 27 17:23:52 2009 -0400
   108.3 @@ -141,6 +141,10 @@
   108.4  
   108.5  //---------------------------match---------------------------------------------
   108.6  void Matcher::match( ) {
   108.7 +  if( MaxLabelRootDepth < 100 ) { // Too small?
   108.8 +    assert(false, "invalid MaxLabelRootDepth, increase it to 100 minimum");
   108.9 +    MaxLabelRootDepth = 100;
  108.10 +  }
  108.11    // One-time initialization of some register masks.
  108.12    init_spill_mask( C->root()->in(1) );
  108.13    _return_addr_mask = return_addr();
  108.14 @@ -1485,8 +1489,7 @@
  108.15  #ifdef ASSERT
  108.16      // Verify adr type after matching memory operation
  108.17      const MachOper* oper = mach->memory_operand();
  108.18 -    if (oper != NULL && oper != (MachOper*)-1 &&
  108.19 -        mach->adr_type() != TypeRawPtr::BOTTOM) { // non-direct addressing mode
  108.20 +    if (oper != NULL && oper != (MachOper*)-1) {
  108.21        // It has a unique memory operand.  Find corresponding ideal mem node.
  108.22        Node* m = NULL;
  108.23        if (leaf->is_Mem()) {

   109.1 --- a/src/share/vm/opto/mulnode.cpp	Mon Jul 27 09:06:22 2009 -0700
   109.2 +++ b/src/share/vm/opto/mulnode.cpp	Mon Jul 27 17:23:52 2009 -0400
   109.3 @@ -430,31 +430,28 @@
   109.4    // x & x => x
   109.5    if (phase->eqv(in(1), in(2))) return in(1);
   109.6  
   109.7 -  Node *load = in(1);
   109.8 -  const TypeInt *t2 = phase->type( in(2) )->isa_int();
   109.9 -  if( t2 && t2->is_con() ) {
  109.10 +  Node* in1 = in(1);
  109.11 +  uint op = in1->Opcode();
  109.12 +  const TypeInt* t2 = phase->type(in(2))->isa_int();
  109.13 +  if (t2 && t2->is_con()) {
  109.14      int con = t2->get_con();
  109.15      // Masking off high bits which are always zero is useless.
  109.16      const TypeInt* t1 = phase->type( in(1) )->isa_int();
  109.17      if (t1 != NULL && t1->_lo >= 0) {
  109.18 -      jint t1_support = ((jint)1 << (1 + log2_intptr(t1->_hi))) - 1;
  109.19 +      jint t1_support = right_n_bits(1 + log2_intptr(t1->_hi));
  109.20        if ((t1_support & con) == t1_support)
  109.21 -        return load;
  109.22 +        return in1;
  109.23      }
  109.24 -    uint lop = load->Opcode();
  109.25 -    if( lop == Op_LoadUS &&
  109.26 -        con == 0x0000FFFF )     // Already zero-extended
  109.27 -      return load;
  109.28      // Masking off the high bits of a unsigned-shift-right is not
  109.29      // needed either.
  109.30 -    if( lop == Op_URShiftI ) {
  109.31 -      const TypeInt *t12 = phase->type( load->in(2) )->isa_int();
  109.32 -      if( t12 && t12->is_con() ) {  // Shift is by a constant
  109.33 +    if (op == Op_URShiftI) {
  109.34 +      const TypeInt* t12 = phase->type(in1->in(2))->isa_int();
  109.35 +      if (t12 && t12->is_con()) {  // Shift is by a constant
  109.36          int shift = t12->get_con();
  109.37          shift &= BitsPerJavaInteger - 1;  // semantics of Java shifts
  109.38          int mask = max_juint >> shift;
  109.39 -        if( (mask&con) == mask )  // If AND is useless, skip it
  109.40 -          return load;
  109.41 +        if ((mask & con) == mask)  // If AND is useless, skip it
  109.42 +          return in1;
  109.43        }
  109.44      }
  109.45    }
  109.46 @@ -476,26 +473,17 @@
  109.47      return new (phase->C, 3) AndINode(load,phase->intcon(mask&0xFFFF));
  109.48  
  109.49    // Masking bits off of a Short?  Loading a Character does some masking
  109.50 -  if( lop == Op_LoadS &&
  109.51 -      (mask & 0xFFFF0000) == 0 ) {
  109.52 +  if (lop == Op_LoadS && (mask & 0xFFFF0000) == 0 ) {
  109.53      Node *ldus = new (phase->C, 3) LoadUSNode(load->in(MemNode::Control),
  109.54 -                                  load->in(MemNode::Memory),
  109.55 -                                  load->in(MemNode::Address),
  109.56 -                                  load->adr_type());
  109.57 +                                              load->in(MemNode::Memory),
  109.58 +                                              load->in(MemNode::Address),
  109.59 +                                              load->adr_type());
  109.60      ldus = phase->transform(ldus);
  109.61 -    return new (phase->C, 3) AndINode(ldus, phase->intcon(mask&0xFFFF));
  109.62 +    return new (phase->C, 3) AndINode(ldus, phase->intcon(mask & 0xFFFF));
  109.63    }
  109.64  
  109.65 -  // Masking sign bits off of a Byte?  Do an unsigned byte load.
  109.66 -  if (lop == Op_LoadB && mask == 0x000000FF) {
  109.67 -    return new (phase->C, 3) LoadUBNode(load->in(MemNode::Control),
  109.68 -                                        load->in(MemNode::Memory),
  109.69 -                                        load->in(MemNode::Address),
  109.70 -                                        load->adr_type());
  109.71 -  }
  109.72 -
  109.73 -  // Masking sign bits off of a Byte plus additional lower bits?  Do
  109.74 -  // an unsigned byte load plus an and.
  109.75 +  // Masking sign bits off of a Byte?  Do an unsigned byte load plus
  109.76 +  // an and.
  109.77    if (lop == Op_LoadB && (mask & 0xFFFFFF00) == 0) {
  109.78      Node* ldub = new (phase->C, 3) LoadUBNode(load->in(MemNode::Control),
  109.79                                                load->in(MemNode::Memory),
  109.80 @@ -605,8 +593,13 @@
  109.81    Node* in1 = in(1);
  109.82    uint op = in1->Opcode();
  109.83  
  109.84 -  // Masking sign bits off of an integer?  Do an unsigned integer to long load.
  109.85 -  if (op == Op_ConvI2L && in1->in(1)->Opcode() == Op_LoadI && mask == 0x00000000FFFFFFFFL) {
  109.86 +  // Masking sign bits off of an integer?  Do an unsigned integer to
  109.87 +  // long load.
  109.88 +  // NOTE: This check must be *before* we try to convert the AndLNode
  109.89 +  // to an AndINode and commute it with ConvI2LNode because
  109.90 +  // 0xFFFFFFFFL masks the whole integer and we get a sign extension,
  109.91 +  // which is wrong.
  109.92 +  if (op == Op_ConvI2L && in1->in(1)->Opcode() == Op_LoadI && mask == CONST64(0x00000000FFFFFFFF)) {
  109.93      Node* load = in1->in(1);
  109.94      return new (phase->C, 3) LoadUI2LNode(load->in(MemNode::Control),
  109.95                                            load->in(MemNode::Memory),
  109.96 @@ -614,9 +607,22 @@
  109.97                                            load->adr_type());
  109.98    }
  109.99  
 109.100 +  // Are we masking a long that was converted from an int with a mask
 109.101 +  // that fits in 32-bits?  Commute them and use an AndINode.
 109.102 +  if (op == Op_ConvI2L && (mask & CONST64(0xFFFFFFFF00000000)) == 0) {
 109.103 +    // If we are doing an UI2L conversion (i.e. the mask is
 109.104 +    // 0x00000000FFFFFFFF) we cannot convert the AndL to an AndI
 109.105 +    // because the AndI would be optimized away later in Identity.
 109.106 +    if (mask != CONST64(0x00000000FFFFFFFF)) {
 109.107 +      Node* andi = new (phase->C, 3) AndINode(in1->in(1), phase->intcon(mask));
 109.108 +      andi = phase->transform(andi);
 109.109 +      return new (phase->C, 2) ConvI2LNode(andi);
 109.110 +    }
 109.111 +  }
 109.112 +
 109.113    // Masking off sign bits?  Dont make them!
 109.114    if (op == Op_RShiftL) {
 109.115 -    const TypeInt *t12 = phase->type(in1->in(2))->isa_int();
 109.116 +    const TypeInt* t12 = phase->type(in1->in(2))->isa_int();
 109.117      if( t12 && t12->is_con() ) { // Shift is by a constant
 109.118        int shift = t12->get_con();
 109.119        shift &= BitsPerJavaLong - 1;  // semantics of Java shifts
 109.120 @@ -626,7 +632,7 @@
 109.121        if( (sign_bits_mask & mask) == 0 ) {
 109.122          // Use zero-fill shift instead
 109.123          Node *zshift = phase->transform(new (phase->C, 3) URShiftLNode(in1->in(1), in1->in(2)));
 109.124 -        return new (phase->C, 3) AndLNode( zshift, in(2) );
 109.125 +        return new (phase->C, 3) AndLNode(zshift, in(2));
 109.126        }
 109.127      }
 109.128    }

   110.1 --- a/src/share/vm/opto/output.cpp	Mon Jul 27 09:06:22 2009 -0700
   110.2 +++ b/src/share/vm/opto/output.cpp	Mon Jul 27 17:23:52 2009 -0400
   110.3 @@ -50,6 +50,13 @@
   110.4    init_scratch_buffer_blob();
   110.5    if (failing())  return; // Out of memory
   110.6  
   110.7 +  // The number of new nodes (mostly MachNop) is proportional to
   110.8 +  // the number of java calls and inner loops which are aligned.
   110.9 +  if ( C->check_node_count((NodeLimitFudgeFactor + C->java_calls()*3 +
  110.10 +                            C->inner_loops()*(OptoLoopAlignment-1)),
  110.11 +                           "out of nodes before code generation" ) ) {
  110.12 +    return;
  110.13 +  }
  110.14    // Make sure I can find the Start Node
  110.15    Block_Array& bbs = _cfg->_bbs;
  110.16    Block *entry = _cfg->_blocks[1];
  110.17 @@ -1105,7 +1112,7 @@
  110.18    uint *call_returns = NEW_RESOURCE_ARRAY(uint, _cfg->_num_blocks+1);
  110.19  
  110.20    uint  return_offset = 0;
  110.21 -  MachNode *nop = new (this) MachNopNode();
  110.22 +  int nop_size = (new (this) MachNopNode())->size(_regalloc);
  110.23  
  110.24    int previous_offset = 0;
  110.25    int current_offset  = 0;
  110.26 @@ -1188,7 +1195,6 @@
  110.27          }
  110.28  
  110.29          // align the instruction if necessary
  110.30 -        int nop_size = nop->size(_regalloc);
  110.31          int padding = mach->compute_padding(current_offset);
  110.32          // Make sure safepoint node for polling is distinct from a call's
  110.33          // return by adding a nop if needed.
  110.34 @@ -1372,7 +1378,6 @@
  110.35  
  110.36      // If the next block is the top of a loop, pad this block out to align
  110.37      // the loop top a little. Helps prevent pipe stalls at loop back branches.
  110.38 -    int nop_size = (new (this) MachNopNode())->size(_regalloc);
  110.39      if( i<_cfg->_num_blocks-1 ) {
  110.40        Block *nb = _cfg->_blocks[i+1];
  110.41        uint padding = nb->alignment_padding(current_offset);

   111.1 --- a/src/share/vm/opto/parse2.cpp	Mon Jul 27 09:06:22 2009 -0700
   111.2 +++ b/src/share/vm/opto/parse2.cpp	Mon Jul 27 17:23:52 2009 -0400
   111.3 @@ -1565,7 +1565,7 @@
   111.4      c = pop();                  // Oop to store
   111.5      b = pop();                  // index (already used)
   111.6      a = pop();                  // the array itself
   111.7 -    const Type* elemtype  = _gvn.type(a)->is_aryptr()->elem();
   111.8 +    const TypeOopPtr* elemtype  = _gvn.type(a)->is_aryptr()->elem()->make_oopptr();
   111.9      const TypeAryPtr* adr_type = TypeAryPtr::OOPS;
  111.10      Node* store = store_oop_to_array(control(), a, d, adr_type, c, elemtype, T_OBJECT);
  111.11      break;

   112.1 --- a/src/share/vm/opto/parse3.cpp	Mon Jul 27 09:06:22 2009 -0700
   112.2 +++ b/src/share/vm/opto/parse3.cpp	Mon Jul 27 17:23:52 2009 -0400
   112.3 @@ -222,7 +222,7 @@
   112.4    // Store the value.
   112.5    Node* store;
   112.6    if (bt == T_OBJECT) {
   112.7 -    const TypePtr* field_type;
   112.8 +    const TypeOopPtr* field_type;
   112.9      if (!field->type()->is_loaded()) {
  112.10        field_type = TypeInstPtr::BOTTOM;
  112.11      } else {
  112.12 @@ -361,7 +361,7 @@
  112.13      guarantee(length_con >= 0, "non-constant multianewarray");
  112.14      ciArrayKlass* array_klass_1 = array_klass->as_obj_array_klass()->element_klass()->as_array_klass();
  112.15      const TypePtr* adr_type = TypeAryPtr::OOPS;
  112.16 -    const Type*    elemtype = _gvn.type(array)->is_aryptr()->elem();
  112.17 +    const TypeOopPtr*    elemtype = _gvn.type(array)->is_aryptr()->elem()->make_oopptr();
  112.18      const intptr_t header   = arrayOopDesc::base_offset_in_bytes(T_OBJECT);
  112.19      for (jint i = 0; i < length_con; i++) {
  112.20        Node*    elem   = expand_multianewarray(array_klass_1, &lengths[1], ndimensions-1, nargs);

   113.1 --- a/src/share/vm/opto/phaseX.hpp	Mon Jul 27 09:06:22 2009 -0700
   113.2 +++ b/src/share/vm/opto/phaseX.hpp	Mon Jul 27 17:23:52 2009 -0400
   113.3 @@ -450,6 +450,8 @@
   113.4      subsume_node(old, nn);
   113.5    }
   113.6  
   113.7 +  bool delay_transform() const { return _delay_transform; }
   113.8 +
   113.9    void set_delay_transform(bool delay) {
  113.10      _delay_transform = delay;
  113.11    }

   114.1 --- a/src/share/vm/opto/type.cpp	Mon Jul 27 09:06:22 2009 -0700
   114.2 +++ b/src/share/vm/opto/type.cpp	Mon Jul 27 17:23:52 2009 -0400
   114.3 @@ -487,6 +487,23 @@
   114.4    return false;
   114.5  }
   114.6  
   114.7 +//----------------------interface_vs_oop---------------------------------------
   114.8 +#ifdef ASSERT
   114.9 +bool Type::interface_vs_oop(const Type *t) const {
  114.10 +  bool result = false;
  114.11 +
  114.12 +  const TypeInstPtr* this_inst = this->isa_instptr();
  114.13 +  const TypeInstPtr*    t_inst =    t->isa_instptr();
  114.14 +  if( this_inst && this_inst->is_loaded() && t_inst && t_inst->is_loaded() ) {
  114.15 +    bool this_interface = this_inst->klass()->is_interface();
  114.16 +    bool    t_interface =    t_inst->klass()->is_interface();
  114.17 +    result = this_interface ^ t_interface;
  114.18 +  }
  114.19 +
  114.20 +  return result;
  114.21 +}
  114.22 +#endif
  114.23 +
  114.24  //------------------------------meet-------------------------------------------
  114.25  // Compute the MEET of two types.  NOT virtual.  It enforces that meet is
  114.26  // commutative and the lattice is symmetric.
  114.27 @@ -507,16 +524,8 @@
  114.28    // Interface meet Oop is Not Symmetric:
  114.29    // Interface:AnyNull meet Oop:AnyNull == Interface:AnyNull
  114.30    // Interface:NotNull meet Oop:NotNull == java/lang/Object:NotNull
  114.31 -  const TypeInstPtr* this_inst = this->isa_instptr();
  114.32 -  const TypeInstPtr*    t_inst =    t->isa_instptr();
  114.33 -  bool interface_vs_oop = false;
  114.34 -  if( this_inst && this_inst->is_loaded() && t_inst && t_inst->is_loaded() ) {
  114.35 -    bool this_interface = this_inst->klass()->is_interface();
  114.36 -    bool    t_interface =    t_inst->klass()->is_interface();
  114.37 -    interface_vs_oop = this_interface ^ t_interface;
  114.38 -  }
  114.39 -
  114.40 -  if( !interface_vs_oop && (t2t != t->_dual || t2this != _dual) ) {
  114.41 +
  114.42 +  if( !interface_vs_oop(t) && (t2t != t->_dual || t2this != _dual) ) {
  114.43      tty->print_cr("=== Meet Not Symmetric ===");
  114.44      tty->print("t   =                   ");         t->dump(); tty->cr();
  114.45      tty->print("this=                   ");            dump(); tty->cr();
  114.46 @@ -1800,6 +1809,17 @@
  114.47    return (intptr_t)_elem + (intptr_t)_size;
  114.48  }
  114.49  
  114.50 +//----------------------interface_vs_oop---------------------------------------
  114.51 +#ifdef ASSERT
  114.52 +bool TypeAry::interface_vs_oop(const Type *t) const {
  114.53 +  const TypeAry* t_ary = t->is_ary();
  114.54 +  if (t_ary) {
  114.55 +    return _elem->interface_vs_oop(t_ary->_elem);
  114.56 +  }
  114.57 +  return false;
  114.58 +}
  114.59 +#endif
  114.60 +
  114.61  //------------------------------dump2------------------------------------------
  114.62  #ifndef PRODUCT
  114.63  void TypeAry::dump2( Dict &d, uint depth, outputStream *st ) const {
  114.64 @@ -3389,6 +3409,17 @@
  114.65    return new TypeAryPtr( dual_ptr(), _const_oop, _ary->dual()->is_ary(),_klass, _klass_is_exact, dual_offset(), dual_instance_id() );
  114.66  }
  114.67  
  114.68 +//----------------------interface_vs_oop---------------------------------------
  114.69 +#ifdef ASSERT
  114.70 +bool TypeAryPtr::interface_vs_oop(const Type *t) const {
  114.71 +  const TypeAryPtr* t_aryptr = t->isa_aryptr();
  114.72 +  if (t_aryptr) {
  114.73 +    return _ary->interface_vs_oop(t_aryptr->_ary);
  114.74 +  }
  114.75 +  return false;
  114.76 +}
  114.77 +#endif
  114.78 +
  114.79  //------------------------------dump2------------------------------------------
  114.80  #ifndef PRODUCT
  114.81  void TypeAryPtr::dump2( Dict &d, uint depth, outputStream *st ) const {
  114.82 @@ -3453,27 +3484,27 @@
  114.83  //------------------------------hash-------------------------------------------
  114.84  // Type-specific hashing function.
  114.85  int TypeNarrowOop::hash(void) const {
  114.86 -  return _ooptype->hash() + 7;
  114.87 +  return _ptrtype->hash() + 7;
  114.88  }
  114.89  
  114.90  
  114.91  bool TypeNarrowOop::eq( const Type *t ) const {
  114.92    const TypeNarrowOop* tc = t->isa_narrowoop();
  114.93    if (tc != NULL) {
  114.94 -    if (_ooptype->base() != tc->_ooptype->base()) {
  114.95 +    if (_ptrtype->base() != tc->_ptrtype->base()) {
  114.96        return false;
  114.97      }
  114.98 -    return tc->_ooptype->eq(_ooptype);
  114.99 +    return tc->_ptrtype->eq(_ptrtype);
 114.100    }
 114.101    return false;
 114.102  }
 114.103  
 114.104  bool TypeNarrowOop::singleton(void) const {    // TRUE if type is a singleton
 114.105 -  return _ooptype->singleton();
 114.106 +  return _ptrtype->singleton();
 114.107  }
 114.108  
 114.109  bool TypeNarrowOop::empty(void) const {
 114.110 -  return _ooptype->empty();
 114.111 +  return _ptrtype->empty();
 114.112  }
 114.113  
 114.114  //------------------------------xmeet------------------------------------------
 114.115 @@ -3507,7 +3538,7 @@
 114.116      return this;
 114.117  
 114.118    case NarrowOop: {
 114.119 -    const Type* result = _ooptype->xmeet(t->make_ptr());
 114.120 +    const Type* result = _ptrtype->xmeet(t->make_ptr());
 114.121      if (result->isa_ptr()) {
 114.122        return TypeNarrowOop::make(result->is_ptr());
 114.123      }
 114.124 @@ -3523,13 +3554,13 @@
 114.125  }
 114.126  
 114.127  const Type *TypeNarrowOop::xdual() const {    // Compute dual right now.
 114.128 -  const TypePtr* odual = _ooptype->dual()->is_ptr();
 114.129 +  const TypePtr* odual = _ptrtype->dual()->is_ptr();
 114.130    return new TypeNarrowOop(odual);
 114.131  }
 114.132  
 114.133  const Type *TypeNarrowOop::filter( const Type *kills ) const {
 114.134    if (kills->isa_narrowoop()) {
 114.135 -    const Type* ft =_ooptype->filter(kills->is_narrowoop()->_ooptype);
 114.136 +    const Type* ft =_ptrtype->filter(kills->is_narrowoop()->_ptrtype);
 114.137      if (ft->empty())
 114.138        return Type::TOP;           // Canonical empty value
 114.139      if (ft->isa_ptr()) {
 114.140 @@ -3537,7 +3568,7 @@
 114.141      }
 114.142      return ft;
 114.143    } else if (kills->isa_ptr()) {
 114.144 -    const Type* ft = _ooptype->join(kills);
 114.145 +    const Type* ft = _ptrtype->join(kills);
 114.146      if (ft->empty())
 114.147        return Type::TOP;           // Canonical empty value
 114.148      return ft;
 114.149 @@ -3548,13 +3579,13 @@
 114.150  
 114.151  
 114.152  intptr_t TypeNarrowOop::get_con() const {
 114.153 -  return _ooptype->get_con();
 114.154 +  return _ptrtype->get_con();
 114.155  }
 114.156  
 114.157  #ifndef PRODUCT
 114.158  void TypeNarrowOop::dump2( Dict & d, uint depth, outputStream *st ) const {
 114.159    st->print("narrowoop: ");
 114.160 -  _ooptype->dump2(d, depth, st);
 114.161 +  _ptrtype->dump2(d, depth, st);
 114.162  }
 114.163  #endif
 114.164

   115.1 --- a/src/share/vm/opto/type.hpp	Mon Jul 27 09:06:22 2009 -0700
   115.2 +++ b/src/share/vm/opto/type.hpp	Mon Jul 27 17:23:52 2009 -0400
   115.3 @@ -190,6 +190,11 @@
   115.4    // Currently, it also works around limitations involving interface types.
   115.5    virtual const Type *filter( const Type *kills ) const;
   115.6  
   115.7 +#ifdef ASSERT
   115.8 +  // One type is interface, the other is oop
   115.9 +  virtual bool interface_vs_oop(const Type *t) const;
  115.10 +#endif
  115.11 +
  115.12    // Returns true if this pointer points at memory which contains a
  115.13    // compressed oop references.
  115.14    bool is_ptr_to_narrowoop() const;
  115.15 @@ -227,6 +232,11 @@
  115.16  
  115.17    // Returns this ptr type or the equivalent ptr type for this compressed pointer.
  115.18    const TypePtr* make_ptr() const;
  115.19 +
  115.20 +  // Returns this oopptr type or the equivalent oopptr type for this compressed pointer.
  115.21 +  // Asserts if the underlying type is not an oopptr or narrowoop.
  115.22 +  const TypeOopPtr* make_oopptr() const;
  115.23 +
  115.24    // Returns this compressed pointer or the equivalent compressed version
  115.25    // of this pointer type.
  115.26    const TypeNarrowOop* make_narrowoop() const;
  115.27 @@ -546,6 +556,10 @@
  115.28    virtual const Type *xmeet( const Type *t ) const;
  115.29    virtual const Type *xdual() const;    // Compute dual right now.
  115.30    bool ary_must_be_exact() const;  // true if arrays of such are never generic
  115.31 +#ifdef ASSERT
  115.32 +  // One type is interface, the other is oop
  115.33 +  virtual bool interface_vs_oop(const Type *t) const;
  115.34 +#endif
  115.35  #ifndef PRODUCT
  115.36    virtual void dump2( Dict &d, uint, outputStream *st  ) const; // Specialized per-Type dumping
  115.37  #endif
  115.38 @@ -867,6 +881,10 @@
  115.39    }
  115.40    static const TypeAryPtr *_array_body_type[T_CONFLICT+1];
  115.41    // sharpen the type of an int which is used as an array size
  115.42 +#ifdef ASSERT
  115.43 +  // One type is interface, the other is oop
  115.44 +  virtual bool interface_vs_oop(const Type *t) const;
  115.45 +#endif
  115.46  #ifndef PRODUCT
  115.47    virtual void dump2( Dict &d, uint depth, outputStream *st ) const; // Specialized per-Type dumping
  115.48  #endif
  115.49 @@ -919,13 +937,13 @@
  115.50  // between the normal and the compressed form.
  115.51  class TypeNarrowOop : public Type {
  115.52  protected:
  115.53 -  const TypePtr* _ooptype; // Could be TypePtr::NULL_PTR
  115.54 +  const TypePtr* _ptrtype; // Could be TypePtr::NULL_PTR
  115.55  
  115.56 -  TypeNarrowOop( const TypePtr* ooptype): Type(NarrowOop),
  115.57 -    _ooptype(ooptype) {
  115.58 -    assert(ooptype->offset() == 0 ||
  115.59 -           ooptype->offset() == OffsetBot ||
  115.60 -           ooptype->offset() == OffsetTop, "no real offsets");
  115.61 +  TypeNarrowOop( const TypePtr* ptrtype): Type(NarrowOop),
  115.62 +    _ptrtype(ptrtype) {
  115.63 +    assert(ptrtype->offset() == 0 ||
  115.64 +           ptrtype->offset() == OffsetBot ||
  115.65 +           ptrtype->offset() == OffsetTop, "no real offsets");
  115.66    }
  115.67  public:
  115.68    virtual bool eq( const Type *t ) const;
  115.69 @@ -949,8 +967,8 @@
  115.70    }
  115.71  
  115.72    // returns the equivalent ptr type for this compressed pointer
  115.73 -  const TypePtr *make_oopptr() const {
  115.74 -    return _ooptype;
  115.75 +  const TypePtr *get_ptrtype() const {
  115.76 +    return _ptrtype;
  115.77    }
  115.78  
  115.79    static const TypeNarrowOop *BOTTOM;
  115.80 @@ -1137,10 +1155,14 @@
  115.81  }
  115.82  
  115.83  inline const TypePtr* Type::make_ptr() const {
  115.84 -  return (_base == NarrowOop) ? is_narrowoop()->make_oopptr() :
  115.85 +  return (_base == NarrowOop) ? is_narrowoop()->get_ptrtype() :
  115.86                                  (isa_ptr() ? is_ptr() : NULL);
  115.87  }
  115.88  
  115.89 +inline const TypeOopPtr* Type::make_oopptr() const {
  115.90 +  return (_base == NarrowOop) ? is_narrowoop()->get_ptrtype()->is_oopptr() : is_oopptr();
  115.91 +}
  115.92 +
  115.93  inline const TypeNarrowOop* Type::make_narrowoop() const {
  115.94    return (_base == NarrowOop) ? is_narrowoop() :
  115.95                                  (isa_ptr() ? TypeNarrowOop::make(is_ptr()) : NULL);
  115.96 @@ -1194,6 +1216,8 @@
  115.97  #define Op_AndX      Op_AndL
  115.98  #define Op_AddX      Op_AddL
  115.99  #define Op_SubX      Op_SubL
 115.100 +#define Op_XorX      Op_XorL
 115.101 +#define Op_URShiftX  Op_URShiftL
 115.102  // conversions
 115.103  #define ConvI2X(x)   ConvI2L(x)
 115.104  #define ConvL2X(x)   (x)
 115.105 @@ -1236,6 +1260,8 @@
 115.106  #define Op_AndX      Op_AndI
 115.107  #define Op_AddX      Op_AddI
 115.108  #define Op_SubX      Op_SubI
 115.109 +#define Op_XorX      Op_XorI
 115.110 +#define Op_URShiftX  Op_URShiftI
 115.111  // conversions
 115.112  #define ConvI2X(x)   (x)
 115.113  #define ConvL2X(x)   ConvL2I(x)

   116.1 --- a/src/share/vm/prims/jvmtiEnvBase.cpp	Mon Jul 27 09:06:22 2009 -0700
   116.2 +++ b/src/share/vm/prims/jvmtiEnvBase.cpp	Mon Jul 27 17:23:52 2009 -0400
   116.3 @@ -606,6 +606,7 @@
   116.4      if (!mons->is_empty()) {
   116.5        for (int i = 0; i < mons->length(); i++) {
   116.6          MonitorInfo *mi = mons->at(i);
   116.7 +        if (mi->owner_is_scalar_replaced()) continue;
   116.8  
   116.9          // see if owner of the monitor is our object
  116.10          if (mi->owner() != NULL && mi->owner() == hobj()) {
  116.11 @@ -726,6 +727,8 @@
  116.12    for (int i = 0; i < mons->length(); i++) {
  116.13      MonitorInfo *mi = mons->at(i);
  116.14  
  116.15 +    if (mi->owner_is_scalar_replaced()) continue;
  116.16 +
  116.17      oop obj = mi->owner();
  116.18      if (obj == NULL) {
  116.19        // this monitor doesn't have an owning object so skip it

   117.1 --- a/src/share/vm/prims/unsafe.cpp	Mon Jul 27 09:06:22 2009 -0700
   117.2 +++ b/src/share/vm/prims/unsafe.cpp	Mon Jul 27 17:23:52 2009 -0400
   117.3 @@ -1048,7 +1048,11 @@
   117.4    oop e = JNIHandles::resolve(e_h);
   117.5    oop p = JNIHandles::resolve(obj);
   117.6    HeapWord* addr = (HeapWord *)index_oop_from_field_offset_long(p, offset);
   117.7 -  update_barrier_set_pre((void*)addr, e);
   117.8 +  if (UseCompressedOops) {
   117.9 +    update_barrier_set_pre((narrowOop*)addr, e);
  117.10 +  } else {
  117.11 +    update_barrier_set_pre((oop*)addr, e);
  117.12 +  }
  117.13    oop res = oopDesc::atomic_compare_exchange_oop(x, addr, e);
  117.14    jboolean success  = (res == e);
  117.15    if (success)

   118.1 --- a/src/share/vm/runtime/arguments.cpp	Mon Jul 27 09:06:22 2009 -0700
   118.2 +++ b/src/share/vm/runtime/arguments.cpp	Mon Jul 27 17:23:52 2009 -0400
   118.3 @@ -1202,18 +1202,13 @@
   118.4    }
   118.5  
   118.6  #ifdef _LP64
   118.7 -  // Compressed Headers do not work with CMS, which uses a bit in the klass
   118.8 -  // field offset to determine free list chunk markers.
   118.9    // Check that UseCompressedOops can be set with the max heap size allocated
  118.10    // by ergonomics.
  118.11    if (MaxHeapSize <= max_heap_for_compressed_oops()) {
  118.12 -    if (FLAG_IS_DEFAULT(UseCompressedOops) && !UseG1GC) {
  118.13 +    if (FLAG_IS_DEFAULT(UseCompressedOops)) {
  118.14        // Turn off until bug is fixed.
  118.15        // the following line to return it to default status.
  118.16        // FLAG_SET_ERGO(bool, UseCompressedOops, true);
  118.17 -    } else if (UseCompressedOops && UseG1GC) {
  118.18 -      warning(" UseCompressedOops does not currently work with UseG1GC; switching off UseCompressedOops. ");
  118.19 -      FLAG_SET_DEFAULT(UseCompressedOops, false);
  118.20      }
  118.21  #ifdef _WIN64
  118.22      if (UseLargePages && UseCompressedOops) {
  118.23 @@ -1454,6 +1449,7 @@
  118.24    if (UseSerialGC)                       i++;
  118.25    if (UseConcMarkSweepGC || UseParNewGC) i++;
  118.26    if (UseParallelGC || UseParallelOldGC) i++;
  118.27 +  if (UseG1GC)                           i++;
  118.28    if (i > 1) {
  118.29      jio_fprintf(defaultStream::error_stream(),
  118.30                  "Conflicting collector combinations in option list; "
  118.31 @@ -2603,22 +2599,6 @@
  118.32      return result;
  118.33    }
  118.34  
  118.35 -  // These are hacks until G1 is fully supported and tested
  118.36 -  // but lets you force -XX:+UseG1GC in PRT and get it where it (mostly) works
  118.37 -  if (UseG1GC) {
  118.38 -    if (UseConcMarkSweepGC || UseParNewGC || UseParallelGC || UseParallelOldGC || UseSerialGC) {
  118.39 -#ifndef PRODUCT
  118.40 -      tty->print_cr("-XX:+UseG1GC is incompatible with other collectors, using UseG1GC");
  118.41 -#endif // PRODUCT
  118.42 -      UseConcMarkSweepGC = false;
  118.43 -      UseParNewGC        = false;
  118.44 -      UseParallelGC      = false;
  118.45 -      UseParallelOldGC   = false;
  118.46 -      UseSerialGC        = false;
  118.47 -    }
  118.48 -    no_shared_spaces();
  118.49 -  }
  118.50 -
  118.51  #ifndef PRODUCT
  118.52    if (TraceBytecodesAt != 0) {
  118.53      TraceBytecodes = true;
  118.54 @@ -2676,10 +2656,7 @@
  118.55    } else if (UseParNewGC) {
  118.56      // Set some flags for ParNew
  118.57      set_parnew_gc_flags();
  118.58 -  }
  118.59 -  // Temporary; make the "if" an "else-if" before
  118.60 -  // we integrate G1. XXX
  118.61 -  if (UseG1GC) {
  118.62 +  } else if (UseG1GC) {
  118.63      // Set some flags for garbage-first, if needed.
  118.64      set_g1_gc_flags();
  118.65    }

   119.1 --- a/src/share/vm/runtime/biasedLocking.cpp	Mon Jul 27 09:06:22 2009 -0700
   119.2 +++ b/src/share/vm/runtime/biasedLocking.cpp	Mon Jul 27 17:23:52 2009 -0400
   119.3 @@ -121,6 +121,7 @@
   119.4          // Walk monitors youngest to oldest
   119.5          for (int i = len - 1; i >= 0; i--) {
   119.6            MonitorInfo* mon_info = monitors->at(i);
   119.7 +          if (mon_info->owner_is_scalar_replaced()) continue;
   119.8            oop owner = mon_info->owner();
   119.9            if (owner != NULL) {
  119.10              info->append(mon_info);
  119.11 @@ -694,6 +695,7 @@
  119.12            // Walk monitors youngest to oldest
  119.13            for (int i = len - 1; i >= 0; i--) {
  119.14              MonitorInfo* mon_info = monitors->at(i);
  119.15 +            if (mon_info->owner_is_scalar_replaced()) continue;
  119.16              oop owner = mon_info->owner();
  119.17              if (owner != NULL) {
  119.18                markOop mark = owner->mark();

   120.1 --- a/src/share/vm/runtime/deoptimization.cpp	Mon Jul 27 09:06:22 2009 -0700
   120.2 +++ b/src/share/vm/runtime/deoptimization.cpp	Mon Jul 27 17:23:52 2009 -0400
   120.3 @@ -933,7 +933,7 @@
   120.4    GrowableArray<MonitorInfo*>* monitors = cvf->monitors();
   120.5    for (int i = 0; i < monitors->length(); i++) {
   120.6      MonitorInfo* mon_info = monitors->at(i);
   120.7 -    if (mon_info->owner() != NULL && !mon_info->eliminated()) {
   120.8 +    if (!mon_info->eliminated() && mon_info->owner() != NULL) {
   120.9        objects_to_revoke->append(Handle(mon_info->owner()));
  120.10      }
  120.11    }

   121.1 --- a/src/share/vm/runtime/globals.hpp	Mon Jul 27 09:06:22 2009 -0700
   121.2 +++ b/src/share/vm/runtime/globals.hpp	Mon Jul 27 17:23:52 2009 -0400
   121.3 @@ -1994,6 +1994,10 @@
   121.4    product_rw(bool, PrintHeapAtGC, false,                                    \
   121.5            "Print heap layout before and after each GC")                     \
   121.6                                                                              \
   121.7 +  product_rw(bool, PrintHeapAtGCExtended, false,                            \
   121.8 +          "Prints extended information about the layout of the heap "       \
   121.9 +          "when -XX:+PrintHeapAtGC is set")                                 \
  121.10 +                                                                            \
  121.11    product(bool, PrintHeapAtSIGBREAK, true,                                  \
  121.12            "Print heap layout in response to SIGBREAK")                      \
  121.13                                                                              \

   122.1 --- a/src/share/vm/runtime/safepoint.cpp	Mon Jul 27 09:06:22 2009 -0700
   122.2 +++ b/src/share/vm/runtime/safepoint.cpp	Mon Jul 27 17:23:52 2009 -0400
   122.3 @@ -49,7 +49,7 @@
   122.4      // In the future we should investigate whether CMS can use the
   122.5      // more-general mechanism below.  DLD (01/05).
   122.6      ConcurrentMarkSweepThread::synchronize(false);
   122.7 -  } else {
   122.8 +  } else if (UseG1GC) {
   122.9      ConcurrentGCThread::safepoint_synchronize();
  122.10    }
  122.11  #endif // SERIALGC
  122.12 @@ -400,7 +400,7 @@
  122.13    // If there are any concurrent GC threads resume them.
  122.14    if (UseConcMarkSweepGC) {
  122.15      ConcurrentMarkSweepThread::desynchronize(false);
  122.16 -  } else {
  122.17 +  } else if (UseG1GC) {
  122.18      ConcurrentGCThread::safepoint_desynchronize();
  122.19    }
  122.20  #endif // SERIALGC

   123.1 --- a/src/share/vm/runtime/sharedRuntime.cpp	Mon Jul 27 09:06:22 2009 -0700
   123.2 +++ b/src/share/vm/runtime/sharedRuntime.cpp	Mon Jul 27 17:23:52 2009 -0400
   123.3 @@ -119,6 +119,7 @@
   123.4      assert(false, "should be optimized out");
   123.5      return;
   123.6    }
   123.7 +  assert(orig->is_oop(true /* ignore mark word */), "Error");
   123.8    // store the original value that was in the field reference
   123.9    thread->satb_mark_queue().enqueue(orig);
  123.10  JRT_END

   124.1 --- a/src/share/vm/runtime/stackValue.cpp	Mon Jul 27 09:06:22 2009 -0700
   124.2 +++ b/src/share/vm/runtime/stackValue.cpp	Mon Jul 27 17:23:52 2009 -0400
   124.3 @@ -104,7 +104,17 @@
   124.4      }
   124.5  #endif
   124.6      case Location::oop: {
   124.7 -      Handle h(*(oop *)value_addr); // Wrap a handle around the oop
   124.8 +      oop val = *(oop *)value_addr;
   124.9 +#ifdef _LP64
  124.10 +      if (Universe::is_narrow_oop_base(val)) {
  124.11 +         // Compiled code may produce decoded oop = narrow_oop_base
  124.12 +         // when a narrow oop implicit null check is used.
  124.13 +         // The narrow_oop_base could be NULL or be the address
  124.14 +         // of the page below heap. Use NULL value for both cases.
  124.15 +         val = (oop)NULL;
  124.16 +      }
  124.17 +#endif
  124.18 +      Handle h(val); // Wrap a handle around the oop
  124.19        return new StackValue(h);
  124.20      }
  124.21      case Location::addr: {
  124.22 @@ -146,8 +156,9 @@
  124.23      value.jl = ((ConstantLongValue *)sv)->value();
  124.24      return new StackValue(value.p);
  124.25  #endif
  124.26 -  } else if (sv->is_object()) {
  124.27 -    return new StackValue(((ObjectValue *)sv)->value());
  124.28 +  } else if (sv->is_object()) { // Scalar replaced object in compiled frame
  124.29 +    Handle ov = ((ObjectValue *)sv)->value();
  124.30 +    return new StackValue(ov, (ov.is_null()) ? 1 : 0);
  124.31    }
  124.32  
  124.33    // Unknown ScopeValue type

   125.1 --- a/src/share/vm/runtime/stackValue.hpp	Mon Jul 27 09:06:22 2009 -0700
   125.2 +++ b/src/share/vm/runtime/stackValue.hpp	Mon Jul 27 17:23:52 2009 -0400
   125.3 @@ -34,9 +34,11 @@
   125.4      _i     = value;
   125.5    }
   125.6  
   125.7 -  StackValue(Handle value) {
   125.8 +  StackValue(Handle value, intptr_t scalar_replaced = 0) {
   125.9      _type    = T_OBJECT;
  125.10 +    _i       = scalar_replaced;
  125.11      _o       = value;
  125.12 +    assert(_i == 0 || _o.is_null(), "not null object should not be marked as scalar replaced");
  125.13    }
  125.14  
  125.15    StackValue() {
  125.16 @@ -56,6 +58,11 @@
  125.17      return _o;
  125.18    }
  125.19  
  125.20 +  bool obj_is_scalar_replaced() const {
  125.21 +    assert(type() == T_OBJECT, "type check");
  125.22 +    return _i != 0;
  125.23 +  }
  125.24 +
  125.25    void set_obj(Handle value) {
  125.26      assert(type() == T_OBJECT, "type check");
  125.27      _o = value;

   126.1 --- a/src/share/vm/runtime/vframe.cpp	Mon Jul 27 09:06:22 2009 -0700
   126.2 +++ b/src/share/vm/runtime/vframe.cpp	Mon Jul 27 17:23:52 2009 -0400
   126.3 @@ -106,6 +106,7 @@
   126.4  
   126.5    for (int index = (mons->length()-1); index >= 0; index--) {
   126.6      MonitorInfo* monitor = mons->at(index);
   126.7 +    if (monitor->eliminated() && is_compiled_frame()) continue; // skip eliminated monitor
   126.8      oop obj = monitor->owner();
   126.9      if (obj == NULL) continue; // skip unowned monitor
  126.10      //
  126.11 @@ -162,6 +163,18 @@
  126.12      bool found_first_monitor = false;
  126.13      for (int index = (mons->length()-1); index >= 0; index--) {
  126.14        MonitorInfo* monitor = mons->at(index);
  126.15 +      if (monitor->eliminated() && is_compiled_frame()) { // Eliminated in compiled code
  126.16 +        if (monitor->owner_is_scalar_replaced()) {
  126.17 +          Klass* k = Klass::cast(monitor->owner_klass());
  126.18 +          st->print("\t- eliminated <owner is scalar replaced> (a %s)", k->external_name());
  126.19 +        } else {
  126.20 +          oop obj = monitor->owner();
  126.21 +          if (obj != NULL) {
  126.22 +            print_locked_object_class_name(st, obj, "eliminated");
  126.23 +          }
  126.24 +        }
  126.25 +        continue;
  126.26 +      }
  126.27        if (monitor->owner() != NULL) {
  126.28  
  126.29          // First, assume we have the monitor locked. If we haven't found an
  126.30 @@ -171,11 +184,11 @@
  126.31  
  126.32          const char *lock_state = "locked"; // assume we have the monitor locked
  126.33          if (!found_first_monitor && frame_count == 0) {
  126.34 -         markOop mark = monitor->owner()->mark();
  126.35 -         if (mark->has_monitor() &&
  126.36 -             mark->monitor() == thread()->current_pending_monitor()) {
  126.37 +          markOop mark = monitor->owner()->mark();
  126.38 +          if (mark->has_monitor() &&
  126.39 +              mark->monitor() == thread()->current_pending_monitor()) {
  126.40              lock_state = "waiting to lock";
  126.41 -         }
  126.42 +          }
  126.43          }
  126.44  
  126.45          found_first_monitor = true;
  126.46 @@ -206,7 +219,7 @@
  126.47    for (BasicObjectLock* current = (fr().previous_monitor_in_interpreter_frame(fr().interpreter_frame_monitor_begin()));
  126.48         current >= fr().interpreter_frame_monitor_end();
  126.49         current = fr().previous_monitor_in_interpreter_frame(current)) {
  126.50 -    result->push(new MonitorInfo(current->obj(), current->lock(), false));
  126.51 +    result->push(new MonitorInfo(current->obj(), current->lock(), false, false));
  126.52    }
  126.53    return result;
  126.54  }
  126.55 @@ -531,8 +544,18 @@
  126.56    tty->print_cr("\tmonitor list:");
  126.57    for (int index = (list->length()-1); index >= 0; index--) {
  126.58      MonitorInfo* monitor = list->at(index);
  126.59 -    tty->print("\t  obj\t"); monitor->owner()->print_value();
  126.60 -    tty->print("(" INTPTR_FORMAT ")", (address)monitor->owner());
  126.61 +    tty->print("\t  obj\t");
  126.62 +    if (monitor->owner_is_scalar_replaced()) {
  126.63 +      Klass* k = Klass::cast(monitor->owner_klass());
  126.64 +      tty->print("( is scalar replaced %s)", k->external_name());
  126.65 +    } else if (monitor->owner() == NULL) {
  126.66 +      tty->print("( null )");
  126.67 +    } else {
  126.68 +      monitor->owner()->print_value();
  126.69 +      tty->print("(" INTPTR_FORMAT ")", (address)monitor->owner());
  126.70 +    }
  126.71 +    if (monitor->eliminated() && is_compiled_frame())
  126.72 +      tty->print(" ( lock is eliminated )");
  126.73      tty->cr();
  126.74      tty->print("\t  ");
  126.75      monitor->lock()->print_on(tty);

   127.1 --- a/src/share/vm/runtime/vframe.hpp	Mon Jul 27 09:06:22 2009 -0700
   127.2 +++ b/src/share/vm/runtime/vframe.hpp	Mon Jul 27 17:23:52 2009 -0400
   127.3 @@ -230,18 +230,36 @@
   127.4   private:
   127.5    oop        _owner; // the object owning the monitor
   127.6    BasicLock* _lock;
   127.7 +  oop        _owner_klass; // klass if owner was scalar replaced
   127.8    bool       _eliminated;
   127.9 +  bool       _owner_is_scalar_replaced;
  127.10   public:
  127.11    // Constructor
  127.12 -  MonitorInfo(oop owner, BasicLock* lock, bool eliminated) {
  127.13 -    _owner = owner;
  127.14 +  MonitorInfo(oop owner, BasicLock* lock, bool eliminated, bool owner_is_scalar_replaced) {
  127.15 +    if (!owner_is_scalar_replaced) {
  127.16 +      _owner = owner;
  127.17 +      _owner_klass = NULL;
  127.18 +    } else {
  127.19 +      assert(eliminated, "monitor should be eliminated for scalar replaced object");
  127.20 +      _owner = NULL;
  127.21 +      _owner_klass = owner;
  127.22 +    }
  127.23      _lock  = lock;
  127.24      _eliminated = eliminated;
  127.25 +    _owner_is_scalar_replaced = owner_is_scalar_replaced;
  127.26    }
  127.27    // Accessors
  127.28 -  oop        owner() const { return _owner; }
  127.29 +  oop        owner() const {
  127.30 +    assert(!_owner_is_scalar_replaced, "should not be called for scalar replaced object");
  127.31 +    return _owner;
  127.32 +  }
  127.33 +  klassOop   owner_klass() const {
  127.34 +    assert(_owner_is_scalar_replaced, "should not be called for not scalar replaced object");
  127.35 +    return (klassOop)_owner_klass;
  127.36 +  }
  127.37    BasicLock* lock()  const { return _lock;  }
  127.38    bool eliminated()  const { return _eliminated; }
  127.39 +  bool owner_is_scalar_replaced()  const { return _owner_is_scalar_replaced; }
  127.40  };
  127.41  
  127.42  class vframeStreamCommon : StackObj {

   128.1 --- a/src/share/vm/runtime/vframeArray.cpp	Mon Jul 27 09:06:22 2009 -0700
   128.2 +++ b/src/share/vm/runtime/vframeArray.cpp	Mon Jul 27 17:23:52 2009 -0400
   128.3 @@ -61,6 +61,7 @@
   128.4      // Migrate the BasicLocks from the stack to the monitor chunk
   128.5      for (index = 0; index < list->length(); index++) {
   128.6        MonitorInfo* monitor = list->at(index);
   128.7 +      assert(!monitor->owner_is_scalar_replaced(), "object should be reallocated already");
   128.8        assert(monitor->owner() == NULL || (!monitor->owner()->is_unlocked() && !monitor->owner()->has_bias_pattern()), "object must be null or locked, and unbiased");
   128.9        BasicObjectLock* dest = _monitors->at(index);
  128.10        dest->set_obj(monitor->owner());
  128.11 @@ -89,6 +90,7 @@
  128.12      StackValue* value = locs->at(index);
  128.13      switch(value->type()) {
  128.14        case T_OBJECT:
  128.15 +        assert(!value->obj_is_scalar_replaced(), "object should be reallocated already");
  128.16          // preserve object type
  128.17          _locals->add( new StackValue((intptr_t) (value->get_obj()()), T_OBJECT ));
  128.18          break;
  128.19 @@ -113,6 +115,7 @@
  128.20      StackValue* value = exprs->at(index);
  128.21      switch(value->type()) {
  128.22        case T_OBJECT:
  128.23 +        assert(!value->obj_is_scalar_replaced(), "object should be reallocated already");
  128.24          // preserve object type
  128.25          _expressions->add( new StackValue((intptr_t) (value->get_obj()()), T_OBJECT ));
  128.26          break;

   129.1 --- a/src/share/vm/runtime/vframe_hp.cpp	Mon Jul 27 09:06:22 2009 -0700
   129.2 +++ b/src/share/vm/runtime/vframe_hp.cpp	Mon Jul 27 17:23:52 2009 -0400
   129.3 @@ -190,7 +190,7 @@
   129.4      // Casting away const
   129.5      frame& fr = (frame&) _fr;
   129.6      MonitorInfo* info = new MonitorInfo(fr.compiled_synchronized_native_monitor_owner(nm),
   129.7 -                                        fr.compiled_synchronized_native_monitor(nm), false);
   129.8 +                                        fr.compiled_synchronized_native_monitor(nm), false, false);
   129.9      monitors->push(info);
  129.10      return monitors;
  129.11    }
  129.12 @@ -201,8 +201,20 @@
  129.13    GrowableArray<MonitorInfo*>* result = new GrowableArray<MonitorInfo*>(monitors->length());
  129.14    for (int index = 0; index < monitors->length(); index++) {
  129.15      MonitorValue* mv = monitors->at(index);
  129.16 -    StackValue *owner_sv = create_stack_value(mv->owner()); // it is an oop
  129.17 -    result->push(new MonitorInfo(owner_sv->get_obj()(), resolve_monitor_lock(mv->basic_lock()), mv->eliminated()));
  129.18 +    ScopeValue*   ov = mv->owner();
  129.19 +    StackValue *owner_sv = create_stack_value(ov); // it is an oop
  129.20 +    if (ov->is_object() && owner_sv->obj_is_scalar_replaced()) { // The owner object was scalar replaced
  129.21 +      assert(mv->eliminated(), "monitor should be eliminated for scalar replaced object");
  129.22 +      // Put klass for scalar replaced object.
  129.23 +      ScopeValue* kv = ((ObjectValue *)ov)->klass();
  129.24 +      assert(kv->is_constant_oop(), "klass should be oop constant for scalar replaced object");
  129.25 +      KlassHandle k(((ConstantOopReadValue*)kv)->value()());
  129.26 +      result->push(new MonitorInfo(k->as_klassOop(), resolve_monitor_lock(mv->basic_lock()),
  129.27 +                                   mv->eliminated(), true));
  129.28 +    } else {
  129.29 +      result->push(new MonitorInfo(owner_sv->get_obj()(), resolve_monitor_lock(mv->basic_lock()),
  129.30 +                                   mv->eliminated(), false));
  129.31 +    }
  129.32    }
  129.33    return result;
  129.34  }

   130.1 --- a/src/share/vm/utilities/taskqueue.cpp	Mon Jul 27 09:06:22 2009 -0700
   130.2 +++ b/src/share/vm/utilities/taskqueue.cpp	Mon Jul 27 17:23:52 2009 -0400
   130.3 @@ -64,15 +64,18 @@
   130.4  }
   130.5  
   130.6  void ParallelTaskTerminator::yield() {
   130.7 +  assert(_offered_termination <= _n_threads, "Invariant");
   130.8    os::yield();
   130.9  }
  130.10  
  130.11  void ParallelTaskTerminator::sleep(uint millis) {
  130.12 +  assert(_offered_termination <= _n_threads, "Invariant");
  130.13    os::sleep(Thread::current(), millis, false);
  130.14  }
  130.15  
  130.16  bool
  130.17  ParallelTaskTerminator::offer_termination(TerminatorTerminator* terminator) {
  130.18 +  assert(_offered_termination < _n_threads, "Invariant");
  130.19    Atomic::inc(&_offered_termination);
  130.20  
  130.21    uint yield_count = 0;
  130.22 @@ -96,6 +99,7 @@
  130.23    // Loop waiting for all threads to offer termination or
  130.24    // more work.
  130.25    while (true) {
  130.26 +    assert(_offered_termination <= _n_threads, "Invariant");
  130.27      // Are all threads offering termination?
  130.28      if (_offered_termination == _n_threads) {
  130.29        return true;
  130.30 @@ -151,6 +155,7 @@
  130.31        if (peek_in_queue_set() ||
  130.32            (terminator != NULL && terminator->should_exit_termination())) {
  130.33          Atomic::dec(&_offered_termination);
  130.34 +        assert(_offered_termination < _n_threads, "Invariant");
  130.35          return false;
  130.36        }
  130.37      }

   131.1 --- a/src/share/vm/utilities/taskqueue.hpp	Mon Jul 27 09:06:22 2009 -0700
   131.2 +++ b/src/share/vm/utilities/taskqueue.hpp	Mon Jul 27 17:23:52 2009 -0400
   131.3 @@ -560,8 +560,14 @@
   131.4  class StarTask {
   131.5    void*  _holder;        // either union oop* or narrowOop*
   131.6   public:
   131.7 -  StarTask(narrowOop *p) { _holder = (void *)((uintptr_t)p | COMPRESSED_OOP_MASK); }
   131.8 -  StarTask(oop *p)       { _holder = (void*)p; }
   131.9 +  StarTask(narrowOop* p) {
  131.10 +    assert(((uintptr_t)p & COMPRESSED_OOP_MASK) == 0, "Information loss!");
  131.11 +    _holder = (void *)((uintptr_t)p | COMPRESSED_OOP_MASK);
  131.12 +  }
  131.13 +  StarTask(oop* p)       {
  131.14 +    assert(((uintptr_t)p & COMPRESSED_OOP_MASK) == 0, "Information loss!");
  131.15 +    _holder = (void*)p;
  131.16 +  }
  131.17    StarTask()             { _holder = NULL; }
  131.18    operator oop*()        { return (oop*)_holder; }
  131.19    operator narrowOop*()  {

   132.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   132.2 +++ b/test/compiler/5057225/Test5057225.java	Mon Jul 27 17:23:52 2009 -0400
   132.3 @@ -0,0 +1,140 @@
   132.4 +/*
   132.5 + * Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
   132.6 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   132.7 + *
   132.8 + * This code is free software; you can redistribute it and/or modify it
   132.9 + * under the terms of the GNU General Public License version 2 only, as
  132.10 + * published by the Free Software Foundation.
  132.11 + *
  132.12 + * This code is distributed in the hope that it will be useful, but WITHOUT
  132.13 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  132.14 + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  132.15 + * version 2 for more details (a copy is included in the LICENSE file that
  132.16 + * accompanied this code).
  132.17 + *
  132.18 + * You should have received a copy of the GNU General Public License version
  132.19 + * 2 along with this work; if not, write to the Free Software Foundation,
  132.20 + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  132.21 + *
  132.22 + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  132.23 + * CA 95054 USA or visit www.sun.com if you need additional information or
  132.24 + * have any questions.
  132.25 + */
  132.26 +
  132.27 +/**
  132.28 + * @test
  132.29 + * @bug 5057225
  132.30 + * @summary Remove useless I2L conversions
  132.31 + *
  132.32 + * @run main/othervm -Xcomp -XX:CompileOnly=Test5057225.doload Test5057225
  132.33 + */
  132.34 +
  132.35 +import java.net.URLClassLoader;
  132.36 +
  132.37 +public class Test5057225 {
  132.38 +    static byte[]  ba = new byte[]  { -1 };
  132.39 +    static short[] sa = new short[] { -1 };
  132.40 +    static int[]   ia = new int[]   { -1 };
  132.41 +
  132.42 +    static final long[] BYTE_MASKS = {
  132.43 +         0x0FL,
  132.44 +         0x7FL,  // 7-bit
  132.45 +         0xFFL,
  132.46 +    };
  132.47 +
  132.48 +    static final long[] SHORT_MASKS = {
  132.49 +        0x000FL,
  132.50 +        0x007FL,  // 7-bit
  132.51 +        0x00FFL,
  132.52 +        0x0FFFL,
  132.53 +        0x3FFFL,  // 14-bit
  132.54 +        0x7FFFL,  // 15-bit
  132.55 +        0xFFFFL,
  132.56 +    };
  132.57 +
  132.58 +    static final long[] INT_MASKS = {
  132.59 +        0x0000000FL,
  132.60 +        0x0000007FL,  // 7-bit
  132.61 +        0x000000FFL,
  132.62 +        0x00000FFFL,
  132.63 +        0x00003FFFL,  // 14-bit
  132.64 +        0x00007FFFL,  // 15-bit
  132.65 +        0x0000FFFFL,
  132.66 +        0x00FFFFFFL,
  132.67 +        0x7FFFFFFFL,  // 31-bit
  132.68 +        0xFFFFFFFFL,
  132.69 +    };
  132.70 +
  132.71 +    public static void main(String[] args) throws Exception {
  132.72 +        for (int i = 0; i < BYTE_MASKS.length; i++) {
  132.73 +            System.setProperty("value", "" + BYTE_MASKS[i]);
  132.74 +            loadAndRunClass("Test5057225$loadUB2L");
  132.75 +        }
  132.76 +
  132.77 +        for (int i = 0; i < SHORT_MASKS.length; i++) {
  132.78 +            System.setProperty("value", "" + SHORT_MASKS[i]);
  132.79 +            loadAndRunClass("Test5057225$loadUS2L");
  132.80 +        }
  132.81 +
  132.82 +        for (int i = 0; i < INT_MASKS.length; i++) {
  132.83 +            System.setProperty("value", "" + INT_MASKS[i]);
  132.84 +            loadAndRunClass("Test5057225$loadUI2L");
  132.85 +        }
  132.86 +    }
  132.87 +
  132.88 +    static void check(long result, long expected) {
  132.89 +        if (result != expected)
  132.90 +            throw new InternalError(result + " != " + expected);
  132.91 +    }
  132.92 +
  132.93 +    static void loadAndRunClass(String classname) throws Exception {
  132.94 +        Class cl = Class.forName(classname);
  132.95 +        URLClassLoader apploader = (URLClassLoader) cl.getClassLoader();
  132.96 +        ClassLoader loader = new URLClassLoader(apploader.getURLs(), apploader.getParent());
  132.97 +        Class c = loader.loadClass(classname);
  132.98 +        Runnable r = (Runnable) c.newInstance();
  132.99 +        r.run();
 132.100 +    }
 132.101 +
 132.102 +    public static class loadUB2L implements Runnable {
 132.103 +        static final long MASK;
 132.104 +        static {
 132.105 +            long value = 0;
 132.106 +            try {
 132.107 +                value = Long.decode(System.getProperty("value"));
 132.108 +            } catch (Throwable e) {}
 132.109 +            MASK = value;
 132.110 +        }
 132.111 +
 132.112 +        public void run() { check(doload(ba), MASK); }
 132.113 +        static long doload(byte[] ba) { return ba[0] & MASK; }
 132.114 +    }
 132.115 +
 132.116 +    public static class loadUS2L implements Runnable {
 132.117 +        static final long MASK;
 132.118 +        static {
 132.119 +            long value = 0;
 132.120 +            try {
 132.121 +                value = Long.decode(System.getProperty("value"));
 132.122 +            } catch (Throwable e) {}
 132.123 +            MASK = value;
 132.124 +        }
 132.125 +
 132.126 +        public void run() { check(doload(sa), MASK); }
 132.127 +        static long doload(short[] sa) { return sa[0] & MASK; }
 132.128 +    }
 132.129 +
 132.130 +    public static class loadUI2L implements Runnable {
 132.131 +        static final long MASK;
 132.132 +        static {
 132.133 +            long value = 0;
 132.134 +            try {
 132.135 +                value = Long.decode(System.getProperty("value"));
 132.136 +            } catch (Throwable e) {}
 132.137 +            MASK = value;
 132.138 +        }
 132.139 +
 132.140 +        public void run() { check(doload(ia), MASK); }
 132.141 +        static long doload(int[] ia) { return ia[0] & MASK; }
 132.142 +    }
 132.143 +}

   133.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   133.2 +++ b/test/compiler/6826736/Test.java	Mon Jul 27 17:23:52 2009 -0400
   133.3 @@ -0,0 +1,75 @@
   133.4 +/*
   133.5 + * Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
   133.6 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   133.7 + *
   133.8 + * This code is free software; you can redistribute it and/or modify it
   133.9 + * under the terms of the GNU General Public License version 2 only, as
  133.10 + * published by the Free Software Foundation.
  133.11 + *
  133.12 + * This code is distributed in the hope that it will be useful, but WITHOUT
  133.13 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  133.14 + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  133.15 + * version 2 for more details (a copy is included in the LICENSE file that
  133.16 + * accompanied this code).
  133.17 + *
  133.18 + * You should have received a copy of the GNU General Public License version
  133.19 + * 2 along with this work; if not, write to the Free Software Foundation,
  133.20 + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  133.21 + *
  133.22 + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  133.23 + * CA 95054 USA or visit www.sun.com if you need additional information or
  133.24 + * have any questions.
  133.25 + *
  133.26 + */
  133.27 +
  133.28 +/**
  133.29 + * @test
  133.30 + * @bug 6826736
  133.31 + * @summary CMS: core dump with -XX:+UseCompressedOops
  133.32 + *
  133.33 + * @run main/othervm -XX:+IgnoreUnrecognizedVMOptions -Xbatch -XX:+ScavengeALot -XX:+UseCompressedOops -XX:HeapBaseMinAddress=32g -XX:CompileThreshold=100 -XX:CompileOnly=Test.test -XX:-BlockLayoutRotateLoops -XX:LoopUnrollLimit=0 Test
  133.34 + */
  133.35 +
  133.36 +public class Test {
  133.37 +    int[] arr;
  133.38 +    int[] arr2;
  133.39 +    int test(int r) {
  133.40 +        for (int i = 0; i < 100; i++) {
  133.41 +            for (int j = i; j < 100; j++) {
  133.42 +               int a = 0;
  133.43 +               for (long k = 0; k < 100; k++) {
  133.44 +                  a += k;
  133.45 +               }
  133.46 +               if (arr != null)
  133.47 +                   a = arr[j];
  133.48 +               r += a;
  133.49 +            }
  133.50 +        }
  133.51 +        return r;
  133.52 +    }
  133.53 +
  133.54 +    public static void main(String[] args) {
  133.55 +        int r = 0;
  133.56 +        Test t = new Test();
  133.57 +        for (int i = 0; i < 100; i++) {
  133.58 +            t.arr = new int[100];
  133.59 +            r = t.test(r);
  133.60 +        }
  133.61 +        System.out.println("Warmup 1 is done.");
  133.62 +        for (int i = 0; i < 100; i++) {
  133.63 +            t.arr = null;
  133.64 +            r = t.test(r);
  133.65 +        }
  133.66 +        System.out.println("Warmup 2 is done.");
  133.67 +        for (int i = 0; i < 100; i++) {
  133.68 +            t.arr = new int[100];
  133.69 +            r = t.test(r);
  133.70 +        }
  133.71 +        System.out.println("Warmup is done.");
  133.72 +        for (int i = 0; i < 100; i++) {
  133.73 +            t.arr = new int[1000000];
  133.74 +            t.arr = null;
  133.75 +            r = t.test(r);
  133.76 +        }
  133.77 +    }
  133.78 +}

   134.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   134.2 +++ b/test/compiler/6837094/Test.java	Mon Jul 27 17:23:52 2009 -0400
   134.3 @@ -0,0 +1,94 @@
   134.4 +/*
   134.5 + * Copyright 2009 Google Inc.  All Rights Reserved.
   134.6 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   134.7 + *
   134.8 + * This code is free software; you can redistribute it and/or modify it
   134.9 + * under the terms of the GNU General Public License version 2 only, as
  134.10 + * published by the Free Software Foundation.
  134.11 + *
  134.12 + * This code is distributed in the hope that it will be useful, but WITHOUT
  134.13 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  134.14 + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  134.15 + * version 2 for more details (a copy is included in the LICENSE file that
  134.16 + * accompanied this code).
  134.17 + *
  134.18 + * You should have received a copy of the GNU General Public License version
  134.19 + * 2 along with this work; if not, write to the Free Software Foundation,
  134.20 + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  134.21 + *
  134.22 + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  134.23 + * CA 95054 USA or visit www.sun.com if you need additional information or
  134.24 + * have any questions.
  134.25 + *
  134.26 + */
  134.27 +
  134.28 +/**
  134.29 + * @test
  134.30 + * @bug 6837094
  134.31 + * @summary False positive for "meet not symmetric" failure
  134.32 + *
  134.33 + * @run main/othervm -Xbatch -XX:CompileOnly=Test.collectIs,Test$Factory$1.getArray,Test$Factory$2.getArray Test
  134.34 + */
  134.35 +
  134.36 +import java.util.Set;
  134.37 +import java.util.HashSet;
  134.38 +
  134.39 +public class Test {
  134.40 +
  134.41 +  private interface Factory<M extends Interface> {
  134.42 +    Factory<Child0> Zero = new Factory<Child0>() {
  134.43 +      public Child0[] getArray() { return new Child0[1]; }
  134.44 +    };
  134.45 +
  134.46 +    Factory<Child1> One = new Factory<Child1>() {
  134.47 +      public Child1[] getArray() { return new Child1[1]; }
  134.48 +    };
  134.49 +
  134.50 +    M[] getArray();
  134.51 +  }
  134.52 +
  134.53 +  /**
  134.54 +   * C2 asserts when compiling this method. Bimorphic inlining happens at
  134.55 +   * getArray call site. A Phi in the catch block tries to join the meet type
  134.56 +   * from he inline site (Parent[]) with the type expected by CI (Interface[]).
  134.57 +   *
  134.58 +   * C2 throws an assert when it doesn't need to.
  134.59 +   */
  134.60 +  private static <I extends Interface> void collectIs(
  134.61 +      Factory<I> factory, Set<Interface> s) {
  134.62 +    for (I i : factory.getArray()) {
  134.63 +      try {
  134.64 +        s.add(i);
  134.65 +      } catch (Exception e) {
  134.66 +      }
  134.67 +    }
  134.68 +  }
  134.69 +
  134.70 +  static public void main(String argv[]) {
  134.71 +    Set<Interface> s = new HashSet();
  134.72 +
  134.73 +    for (int i = 0; i < 25000; i++) {
  134.74 +      collectIs(Factory.Zero, s);
  134.75 +      collectIs(Factory.One, s);
  134.76 +    }
  134.77 +  }
  134.78 +}
  134.79 +
  134.80 +/**
  134.81 + * Establish necessary class hierarchy
  134.82 + */
  134.83 +
  134.84 +interface Interface {
  134.85 +}
  134.86 +
  134.87 +class Parent {
  134.88 +}
  134.89 +
  134.90 +class Child0 extends Parent implements Interface {
  134.91 +}
  134.92 +
  134.93 +class Child1 extends Parent implements Interface {
  134.94 +}
  134.95 +
  134.96 +class Child2 extends Parent implements Interface {
  134.97 +}

   135.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   135.2 +++ b/test/compiler/6849574/Test.java	Mon Jul 27 17:23:52 2009 -0400
   135.3 @@ -0,0 +1,44 @@
   135.4 +/*
   135.5 + * Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
   135.6 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   135.7 + *
   135.8 + * This code is free software; you can redistribute it and/or modify it
   135.9 + * under the terms of the GNU General Public License version 2 only, as
  135.10 + * published by the Free Software Foundation.
  135.11 + *
  135.12 + * This code is distributed in the hope that it will be useful, but WITHOUT
  135.13 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  135.14 + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  135.15 + * version 2 for more details (a copy is included in the LICENSE file that
  135.16 + * accompanied this code).
  135.17 + *
  135.18 + * You should have received a copy of the GNU General Public License version
  135.19 + * 2 along with this work; if not, write to the Free Software Foundation,
  135.20 + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  135.21 + *
  135.22 + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  135.23 + * CA 95054 USA or visit www.sun.com if you need additional information or
  135.24 + * have any questions.
  135.25 + *
  135.26 + */
  135.27 +
  135.28 +/**
  135.29 + * @test
  135.30 + * @bug 6849574
  135.31 + * @summary VM crash using NonBlockingHashMap (high_scale_lib)
  135.32 + *
  135.33 + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+VerifyBeforeGC Test
  135.34 + */
  135.35 +
  135.36 +import java.util.concurrent.atomic.*;
  135.37 +
  135.38 +public class Test extends Thread {
  135.39 +
  135.40 +    public static void main(String[] args) {
  135.41 +        AtomicReferenceArray a = new AtomicReferenceArray(10000);
  135.42 +        for (int i = 0; i < 100000; i++) {
  135.43 +            a.getAndSet(9999, new Object());
  135.44 +            if (i > 99990) System.gc();
  135.45 +        }
  135.46 +    }
  135.47 +}

   136.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   136.2 +++ b/test/compiler/6851282/Test.java	Mon Jul 27 17:23:52 2009 -0400
   136.3 @@ -0,0 +1,124 @@
   136.4 +/*
   136.5 + * Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
   136.6 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   136.7 + *
   136.8 + * This code is free software; you can redistribute it and/or modify it
   136.9 + * under the terms of the GNU General Public License version 2 only, as
  136.10 + * published by the Free Software Foundation.
  136.11 + *
  136.12 + * This code is distributed in the hope that it will be useful, but WITHOUT
  136.13 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  136.14 + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  136.15 + * version 2 for more details (a copy is included in the LICENSE file that
  136.16 + * accompanied this code).
  136.17 + *
  136.18 + * You should have received a copy of the GNU General Public License version
  136.19 + * 2 along with this work; if not, write to the Free Software Foundation,
  136.20 + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  136.21 + *
  136.22 + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  136.23 + * CA 95054 USA or visit www.sun.com if you need additional information or
  136.24 + * have any questions.
  136.25 + *
  136.26 + */
  136.27 +
  136.28 +/**
  136.29 + * @test
  136.30 + * @bug 6851282
  136.31 + * @summary JIT miscompilation results in null entry in array when using CompressedOops
  136.32 + *
  136.33 + * @run main/othervm -XX:+IgnoreUnrecognizedVMOptions -XX:+UseCompressedOops Test
  136.34 + */
  136.35 +
  136.36 +import java.util.ArrayList;
  136.37 +import java.util.List;
  136.38 +
  136.39 +public class Test {
  136.40 +  void foo(A a, A[] as) {
  136.41 +    for (A a1 : as) {
  136.42 +      B[] filtered = a.c(a1);
  136.43 +      for (B b : filtered) {
  136.44 +        if (b == null) {
  136.45 +          System.out.println("bug: b == null");
  136.46 +          System.exit(97);
  136.47 +        }
  136.48 +      }
  136.49 +    }
  136.50 +  }
  136.51 +
  136.52 +  public static void main(String[] args) {
  136.53 +    List<A> as = new ArrayList<A>();
  136.54 +    for (int i = 0; i < 5000; i++) {
  136.55 +      List<B> bs = new ArrayList<B>();
  136.56 +      for (int j = i; j < i + 1000; j++)
  136.57 +        bs.add(new B(j));
  136.58 +      as.add(new A(bs.toArray(new B[0])));
  136.59 +    }
  136.60 +    new Test().foo(as.get(0), as.subList(1, as.size()).toArray(new A[0]));
  136.61 +  }
  136.62 +}
  136.63 +
  136.64 +class A {
  136.65 +  final B[] bs;
  136.66 +
  136.67 +  public A(B[] bs) {
  136.68 +    this.bs = bs;
  136.69 +  }
  136.70 +
  136.71 +  final B[] c(final A a) {
  136.72 +    return new BoxedArray<B>(bs).filter(new Function<B, Boolean>() {
  136.73 +      public Boolean apply(B arg) {
  136.74 +        for (B b : a.bs) {
  136.75 +          if (b.d == arg.d)
  136.76 +            return true;
  136.77 +        }
  136.78 +        return false;
  136.79 +      }
  136.80 +    });
  136.81 +  }
  136.82 +}
  136.83 +
  136.84 +class BoxedArray<T> {
  136.85 +
  136.86 +  private final T[] array;
  136.87 +
  136.88 +  BoxedArray(T[] array) {
  136.89 +    this.array = array;
  136.90 +  }
  136.91 +
  136.92 +  public T[] filter(Function<T, Boolean> function) {
  136.93 +    boolean[] include = new boolean[array.length];
  136.94 +    int len = 0;
  136.95 +    int i = 0;
  136.96 +    while (i < array.length) {
  136.97 +      if (function.apply(array[i])) {
  136.98 +        include[i] = true;
  136.99 +        len += 1;
 136.100 +      }
 136.101 +      i += 1;
 136.102 +    }
 136.103 +    T[] result = (T[]) java.lang.reflect.Array.newInstance(array.getClass().getComponentType(), len);
 136.104 +    len = 0;
 136.105 +    i = 0;
 136.106 +    while (len < result.length) {
 136.107 +      if (include[i]) {
 136.108 +        result[len] = array[i];
 136.109 +        len += 1;
 136.110 +      }
 136.111 +      i += 1;
 136.112 +    }
 136.113 +    return result;
 136.114 +  }
 136.115 +}
 136.116 +
 136.117 +interface Function<T, R> {
 136.118 +  R apply(T arg);
 136.119 +}
 136.120 +
 136.121 +class B {
 136.122 +  final int d;
 136.123 +  public B(int d) {
 136.124 +    this.d = d;
 136.125 +  }
 136.126 +}
 136.127 +

   137.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   137.2 +++ b/test/compiler/6855164/Test.java	Mon Jul 27 17:23:52 2009 -0400
   137.3 @@ -0,0 +1,55 @@
   137.4 +/*
   137.5 + * Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
   137.6 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   137.7 + *
   137.8 + * This code is free software; you can redistribute it and/or modify it
   137.9 + * under the terms of the GNU General Public License version 2 only, as
  137.10 + * published by the Free Software Foundation.
  137.11 + *
  137.12 + * This code is distributed in the hope that it will be useful, but WITHOUT
  137.13 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  137.14 + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  137.15 + * version 2 for more details (a copy is included in the LICENSE file that
  137.16 + * accompanied this code).
  137.17 + *
  137.18 + * You should have received a copy of the GNU General Public License version
  137.19 + * 2 along with this work; if not, write to the Free Software Foundation,
  137.20 + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  137.21 + *
  137.22 + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  137.23 + * CA 95054 USA or visit www.sun.com if you need additional information or
  137.24 + * have any questions.
  137.25 + */
  137.26 +
  137.27 +/*
  137.28 + * @test
  137.29 + * @bug 6855164
  137.30 + * @summary SIGSEGV during compilation of method involving loop over CharSequence
  137.31 + * @run main/othervm -Xbatch Test
  137.32 + */
  137.33 +
  137.34 +public class Test{
  137.35 +    public static void main(String[] args) throws Exception {
  137.36 +        StringBuffer builder = new StringBuffer();
  137.37 +
  137.38 +        for(int i = 0; i < 100; i++)
  137.39 +            builder.append("I am the very model of a modern major general\n");
  137.40 +
  137.41 +        for(int j = 0; j < builder.length(); j++){
  137.42 +            previousSpaceIndex(builder, j);
  137.43 +        }
  137.44 +    }
  137.45 +
  137.46 +    private static final int previousSpaceIndex(CharSequence sb, int seek) {
  137.47 +        seek--;
  137.48 +        while (seek > 0) {
  137.49 +            if (sb.charAt(seek) == ' ') {
  137.50 +                while (seek > 0 && sb.charAt(seek - 1) == ' ')
  137.51 +                    seek--;
  137.52 +                return seek;
  137.53 +            }
  137.54 +            seek--;
  137.55 +        }
  137.56 +        return 0;
  137.57 +    }
  137.58 +}

   138.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   138.2 +++ b/test/compiler/6857159/Test6857159.java	Mon Jul 27 17:23:52 2009 -0400
   138.3 @@ -0,0 +1,68 @@
   138.4 +/*
   138.5 + * Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
   138.6 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   138.7 + *
   138.8 + * This code is free software; you can redistribute it and/or modify it
   138.9 + * under the terms of the GNU General Public License version 2 only, as
  138.10 + * published by the Free Software Foundation.
  138.11 + *
  138.12 + * This code is distributed in the hope that it will be useful, but WITHOUT
  138.13 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  138.14 + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  138.15 + * version 2 for more details (a copy is included in the LICENSE file that
  138.16 + * accompanied this code).
  138.17 + *
  138.18 + * You should have received a copy of the GNU General Public License version
  138.19 + * 2 along with this work; if not, write to the Free Software Foundation,
  138.20 + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  138.21 + *
  138.22 + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  138.23 + * CA 95054 USA or visit www.sun.com if you need additional information or
  138.24 + * have any questions.
  138.25 + *
  138.26 + */
  138.27 +
  138.28 +/**
  138.29 + * @test
  138.30 + * @bug 6857159
  138.31 + * @summary local schedule failed with checkcast of Thread.currentThread()
  138.32 + *
  138.33 + * @run shell Test6857159.sh
  138.34 + */
  138.35 +
  138.36 +public class Test6857159 extends Thread {
  138.37 +    static class ct0 extends Test6857159 {
  138.38 +        public void message() {
  138.39 +            // System.out.println("message");
  138.40 +        }
  138.41 +
  138.42 +        public void run() {
  138.43 +             message();
  138.44 +             ct0 ct = (ct0) Thread.currentThread();
  138.45 +             ct.message();
  138.46 +        }
  138.47 +    }
  138.48 +    static class ct1 extends ct0 {
  138.49 +        public void message() {
  138.50 +            // System.out.println("message");
  138.51 +        }
  138.52 +    }
  138.53 +    static class ct2 extends ct0 {
  138.54 +        public void message() {
  138.55 +            // System.out.println("message");
  138.56 +        }
  138.57 +    }
  138.58 +
  138.59 +    public static void main(String[] args) throws Exception {
  138.60 +        for (int i = 0; i < 100000; i++) {
  138.61 +            Thread t = null;
  138.62 +            switch (i % 3) {
  138.63 +              case 0: t = new ct0(); break;
  138.64 +              case 1: t = new ct1(); break;
  138.65 +              case 2: t = new ct2(); break;
  138.66 +            }
  138.67 +            t.start();
  138.68 +            t.join();
  138.69 +        }
  138.70 +    }
  138.71 +}

   139.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   139.2 +++ b/test/compiler/6857159/Test6857159.sh	Mon Jul 27 17:23:52 2009 -0400
   139.3 @@ -0,0 +1,65 @@
   139.4 +#!/bin/sh
   139.5 +# 
   139.6 +# Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
   139.7 +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   139.8 +# 
   139.9 +# This code is free software; you can redistribute it and/or modify it
  139.10 +# under the terms of the GNU General Public License version 2 only, as
  139.11 +# published by the Free Software Foundation.
  139.12 +# 
  139.13 +# This code is distributed in the hope that it will be useful, but WITHOUT
  139.14 +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  139.15 +# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  139.16 +# version 2 for more details (a copy is included in the LICENSE file that
  139.17 +# accompanied this code).
  139.18 +# 
  139.19 +# You should have received a copy of the GNU General Public License version
  139.20 +# 2 along with this work; if not, write to the Free Software Foundation,
  139.21 +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  139.22 +# 
  139.23 +# Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  139.24 +# CA 95054 USA or visit www.sun.com if you need additional information or
  139.25 +# have any questions.
  139.26 +# 
  139.27 +# 
  139.28 +
  139.29 +if [ "${TESTSRC}" = "" ]
  139.30 +then
  139.31 +  echo "TESTSRC not set.  Test cannot execute.  Failed."
  139.32 +  exit 1
  139.33 +fi
  139.34 +echo "TESTSRC=${TESTSRC}"
  139.35 +if [ "${TESTJAVA}" = "" ]
  139.36 +then
  139.37 +  echo "TESTJAVA not set.  Test cannot execute.  Failed."
  139.38 +  exit 1
  139.39 +fi
  139.40 +echo "TESTJAVA=${TESTJAVA}"
  139.41 +if [ "${TESTCLASSES}" = "" ]
  139.42 +then
  139.43 +  echo "TESTCLASSES not set.  Test cannot execute.  Failed."
  139.44 +  exit 1
  139.45 +fi
  139.46 +echo "TESTCLASSES=${TESTCLASSES}"
  139.47 +echo "CLASSPATH=${CLASSPATH}"
  139.48 +
  139.49 +set -x
  139.50 +
  139.51 +cp ${TESTSRC}/Test6857159.java .
  139.52 +cp ${TESTSRC}/Test6857159.sh .
  139.53 +
  139.54 +${TESTJAVA}/bin/javac -d . Test6857159.java
  139.55 +
  139.56 +${TESTJAVA}/bin/java  ${TESTVMOPTS} -Xbatch -XX:+PrintCompilation -XX:CompileOnly=Test6857159\$ct.run Test6857159 > test.out 2>&1
  139.57 +
  139.58 +grep "COMPILE SKIPPED" test.out
  139.59 +
  139.60 +result=$?
  139.61 +if [ $result -eq 1 ]
  139.62 +then
  139.63 +  echo "Passed"
  139.64 +  exit 0
  139.65 +else
  139.66 +  echo "Failed"
  139.67 +  exit 1
  139.68 +fi

   140.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   140.2 +++ b/test/compiler/6859338/Test6859338.java	Mon Jul 27 17:23:52 2009 -0400
   140.3 @@ -0,0 +1,51 @@
   140.4 +/*
   140.5 + * Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
   140.6 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   140.7 + *
   140.8 + * This code is free software; you can redistribute it and/or modify it
   140.9 + * under the terms of the GNU General Public License version 2 only, as
  140.10 + * published by the Free Software Foundation.
  140.11 + *
  140.12 + * This code is distributed in the hope that it will be useful, but WITHOUT
  140.13 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  140.14 + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  140.15 + * version 2 for more details (a copy is included in the LICENSE file that
  140.16 + * accompanied this code).
  140.17 + *
  140.18 + * You should have received a copy of the GNU General Public License version
  140.19 + * 2 along with this work; if not, write to the Free Software Foundation,
  140.20 + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  140.21 + *
  140.22 + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  140.23 + * CA 95054 USA or visit www.sun.com if you need additional information or
  140.24 + * have any questions.
  140.25 + *
  140.26 + */
  140.27 +
  140.28 +/**
  140.29 + * @test
  140.30 + * @bug 6859338
  140.31 + * @summary Assertion failure in sharedRuntime.cpp
  140.32 + *
  140.33 + * @run main/othervm -XX:+IgnoreUnrecognizedVMOptions  -XX:-InlineObjectHash -Xbatch -XX:-ProfileInterpreter Test6859338
  140.34 + */
  140.35 +
  140.36 +public class Test6859338 {
  140.37 +    static Object[] o = new Object[] { new Object(), null };
  140.38 +    public static void main(String[] args) {
  140.39 +        int total = 0;
  140.40 +        try {
  140.41 +            // Exercise the implicit null check in the unverified entry point
  140.42 +            for (int i = 0; i < 40000; i++) {
  140.43 +                int limit = o.length;
  140.44 +                if (i < 20000) limit = 1;
  140.45 +                for (int j = 0; j < limit; j++) {
  140.46 +                    total += o[j].hashCode();
  140.47 +                }
  140.48 +            }
  140.49 +
  140.50 +        } catch (NullPointerException e) {
  140.51 +            // this is expected.  A true failure causes a crash
  140.52 +        }
  140.53 +    }
  140.54 +}

   141.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   141.2 +++ b/test/compiler/6860469/Test.java	Mon Jul 27 17:23:52 2009 -0400
   141.3 @@ -0,0 +1,71 @@
   141.4 +/*
   141.5 + * Copyright 2009 Google Inc.  All Rights Reserved.
   141.6 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   141.7 + *
   141.8 + * This code is free software; you can redistribute it and/or modify it
   141.9 + * under the terms of the GNU General Public License version 2 only, as
  141.10 + * published by the Free Software Foundation.
  141.11 + *
  141.12 + * This code is distributed in the hope that it will be useful, but WITHOUT
  141.13 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  141.14 + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  141.15 + * version 2 for more details (a copy is included in the LICENSE file that
  141.16 + * accompanied this code).
  141.17 + *
  141.18 + * You should have received a copy of the GNU General Public License version
  141.19 + * 2 along with this work; if not, write to the Free Software Foundation,
  141.20 + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  141.21 + *
  141.22 + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  141.23 + * CA 95054 USA or visit www.sun.com if you need additional information or
  141.24 + * have any questions.
  141.25 + *
  141.26 + */
  141.27 +
  141.28 +/**
  141.29 + * @test
  141.30 + * @bug 6860469
  141.31 + * @summary remix_address_expressions reshapes address expression with bad control
  141.32 + *
  141.33 + * @run main/othervm -Xcomp -XX:CompileOnly=Test.C Test
  141.34 + */
  141.35 +
  141.36 +public class Test {
  141.37 +
  141.38 +  private static final int H = 16;
  141.39 +  private static final int F = 9;
  141.40 +
  141.41 +  static int[] fl = new int[1 << F];
  141.42 +
  141.43 +  static int C(int ll, int f) {
  141.44 +    int max = -1;
  141.45 +    int min = H + 1;
  141.46 +
  141.47 +    if (ll != 0) {
  141.48 +      if (ll < min) {
  141.49 +        min = ll;
  141.50 +      }
  141.51 +      if (ll > max) {
  141.52 +        max = ll;
  141.53 +      }
  141.54 +    }
  141.55 +
  141.56 +    if (f > max) {
  141.57 +      f = max;
  141.58 +    }
  141.59 +    if (min > f) {
  141.60 +      min = f;
  141.61 +    }
  141.62 +
  141.63 +    for (int mc = 1 >> max - f; mc <= 0; mc++) {
  141.64 +      int i = mc << (32 - f);
  141.65 +      fl[i] = max;
  141.66 +    }
  141.67 +
  141.68 +    return min;
  141.69 +  }
  141.70 +
  141.71 +  public static void main(String argv[]) {
  141.72 +    C(0, 10);
  141.73 +  }
  141.74 +}

src/share/vm/classfile/classFileParser.cpp		file \| annotate \| diff \| comparison \| revisions
src/share/vm/includeDB_core		file \| annotate \| diff \| comparison \| revisions
src/share/vm/oops/objArrayKlass.cpp		file \| annotate \| diff \| comparison \| revisions

Mercurial > jdk8-mips64-public > hotspot / changeset

changeset

Merge