Mercurial > jdk8-mips64-public > hotspot / changeset

     1.1 --- a/src/share/vm/gc_implementation/parallelScavenge/psOldGen.cpp	Mon Jan 26 12:47:21 2009 -0800
     1.2 +++ b/src/share/vm/gc_implementation/parallelScavenge/psOldGen.cpp	Tue Jan 27 18:13:59 2009 -0800
     1.3 @@ -116,7 +116,7 @@
     1.4    // ObjectSpace stuff
     1.5    //
     1.6  
     1.7 -  _object_space = new MutableSpace();
     1.8 +  _object_space = new MutableSpace(virtual_space()->alignment());
     1.9  
    1.10    if (_object_space == NULL)
    1.11      vm_exit_during_initialization("Could not allocate an old gen space");
    1.12 @@ -385,10 +385,10 @@
    1.13    start_array()->set_covered_region(new_memregion);
    1.14    Universe::heap()->barrier_set()->resize_covered_region(new_memregion);
    1.15  
    1.16 -  HeapWord* const virtual_space_high = (HeapWord*) virtual_space()->high();
    1.17 -
    1.18    // ALWAYS do this last!!
    1.19 -  object_space()->set_end(virtual_space_high);
    1.20 +  object_space()->initialize(new_memregion,
    1.21 +                             SpaceDecorator::DontClear,
    1.22 +                             SpaceDecorator::DontMangle);
    1.23  
    1.24    assert(new_word_size == heap_word_size(object_space()->capacity_in_bytes()),
    1.25      "Sanity");

     2.1 --- a/src/share/vm/gc_implementation/parallelScavenge/psVirtualspace.cpp	Mon Jan 26 12:47:21 2009 -0800
     2.2 +++ b/src/share/vm/gc_implementation/parallelScavenge/psVirtualspace.cpp	Tue Jan 27 18:13:59 2009 -0800
     2.3 @@ -78,7 +78,7 @@
     2.4    _special = false;
     2.5  }
     2.6  
     2.7 -bool PSVirtualSpace::expand_by(size_t bytes, bool pre_touch) {
     2.8 +bool PSVirtualSpace::expand_by(size_t bytes) {
     2.9    assert(is_aligned(bytes), "arg not aligned");
    2.10    DEBUG_ONLY(PSVirtualSpaceVerifier this_verifier(this));
    2.11  
    2.12 @@ -92,15 +92,6 @@
    2.13      _committed_high_addr += bytes;
    2.14    }
    2.15  
    2.16 -  if (pre_touch || AlwaysPreTouch) {
    2.17 -    for (char* curr = base_addr;
    2.18 -         curr < _committed_high_addr;
    2.19 -         curr += os::vm_page_size()) {
    2.20 -      char tmp = *curr;
    2.21 -      *curr = 0;
    2.22 -    }
    2.23 -  }
    2.24 -
    2.25    return result;
    2.26  }
    2.27  
    2.28 @@ -255,7 +246,7 @@
    2.29    DEBUG_ONLY(verify());
    2.30  }
    2.31  
    2.32 -bool PSVirtualSpaceHighToLow::expand_by(size_t bytes, bool pre_touch) {
    2.33 +bool PSVirtualSpaceHighToLow::expand_by(size_t bytes) {
    2.34    assert(is_aligned(bytes), "arg not aligned");
    2.35    DEBUG_ONLY(PSVirtualSpaceVerifier this_verifier(this));
    2.36  
    2.37 @@ -269,15 +260,6 @@
    2.38      _committed_low_addr -= bytes;
    2.39    }
    2.40  
    2.41 -  if (pre_touch || AlwaysPreTouch) {
    2.42 -    for (char* curr = base_addr;
    2.43 -         curr < _committed_high_addr;
    2.44 -         curr += os::vm_page_size()) {
    2.45 -      char tmp = *curr;
    2.46 -      *curr = 0;
    2.47 -    }
    2.48 -  }
    2.49 -
    2.50    return result;
    2.51  }
    2.52  

     3.1 --- a/src/share/vm/gc_implementation/parallelScavenge/psVirtualspace.hpp	Mon Jan 26 12:47:21 2009 -0800
     3.2 +++ b/src/share/vm/gc_implementation/parallelScavenge/psVirtualspace.hpp	Tue Jan 27 18:13:59 2009 -0800
     3.3 @@ -80,7 +80,7 @@
     3.4    inline  void   set_reserved(char* low_addr, char* high_addr, bool special);
     3.5    inline  void   set_reserved(ReservedSpace rs);
     3.6    inline  void   set_committed(char* low_addr, char* high_addr);
     3.7 -  virtual bool   expand_by(size_t bytes, bool pre_touch = false);
     3.8 +  virtual bool   expand_by(size_t bytes);
     3.9    virtual bool   shrink_by(size_t bytes);
    3.10    virtual size_t expand_into(PSVirtualSpace* space, size_t bytes);
    3.11    void           release();
    3.12 @@ -127,7 +127,7 @@
    3.13    PSVirtualSpaceHighToLow(ReservedSpace rs, size_t alignment);
    3.14    PSVirtualSpaceHighToLow(ReservedSpace rs);
    3.15  
    3.16 -  virtual bool   expand_by(size_t bytes, bool pre_touch = false);
    3.17 +  virtual bool   expand_by(size_t bytes);
    3.18    virtual bool   shrink_by(size_t bytes);
    3.19    virtual size_t expand_into(PSVirtualSpace* space, size_t bytes);
    3.20  

     4.1 --- a/src/share/vm/gc_implementation/parallelScavenge/psYoungGen.cpp	Mon Jan 26 12:47:21 2009 -0800
     4.2 +++ b/src/share/vm/gc_implementation/parallelScavenge/psYoungGen.cpp	Tue Jan 27 18:13:59 2009 -0800
     4.3 @@ -64,12 +64,12 @@
     4.4    }
     4.5  
     4.6    if (UseNUMA) {
     4.7 -    _eden_space = new MutableNUMASpace();
     4.8 +    _eden_space = new MutableNUMASpace(virtual_space()->alignment());
     4.9    } else {
    4.10 -    _eden_space = new MutableSpace();
    4.11 +    _eden_space = new MutableSpace(virtual_space()->alignment());
    4.12    }
    4.13 -  _from_space = new MutableSpace();
    4.14 -  _to_space   = new MutableSpace();
    4.15 +  _from_space = new MutableSpace(virtual_space()->alignment());
    4.16 +  _to_space   = new MutableSpace(virtual_space()->alignment());
    4.17  
    4.18    if (_eden_space == NULL || _from_space == NULL || _to_space == NULL) {
    4.19      vm_exit_during_initialization("Could not allocate a young gen space");

     5.1 --- a/src/share/vm/gc_implementation/shared/mutableNUMASpace.cpp	Mon Jan 26 12:47:21 2009 -0800
     5.2 +++ b/src/share/vm/gc_implementation/shared/mutableNUMASpace.cpp	Tue Jan 27 18:13:59 2009 -0800
     5.3 @@ -27,7 +27,7 @@
     5.4  # include "incls/_mutableNUMASpace.cpp.incl"
     5.5  
     5.6  
     5.7 -MutableNUMASpace::MutableNUMASpace() {
     5.8 +MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment) {
     5.9    _lgrp_spaces = new (ResourceObj::C_HEAP) GrowableArray<LGRPSpace*>(0, true);
    5.10    _page_size = os::vm_page_size();
    5.11    _adaptation_cycles = 0;
    5.12 @@ -221,7 +221,7 @@
    5.13          }
    5.14        }
    5.15        if (!found) {
    5.16 -        lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i]));
    5.17 +        lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], alignment()));
    5.18        }
    5.19      }
    5.20  
    5.21 @@ -443,10 +443,10 @@
    5.22    // Is there bottom?
    5.23    if (new_region.start() < intersection.start()) { // Yes
    5.24      // Try to coalesce small pages into a large one.
    5.25 -    if (UseLargePages && page_size() >= os::large_page_size()) {
    5.26 -      HeapWord* p = (HeapWord*)round_to((intptr_t) intersection.start(), os::large_page_size());
    5.27 +    if (UseLargePages && page_size() >= alignment()) {
    5.28 +      HeapWord* p = (HeapWord*)round_to((intptr_t) intersection.start(), alignment());
    5.29        if (new_region.contains(p)
    5.30 -          && pointer_delta(p, new_region.start(), sizeof(char)) >= os::large_page_size()) {
    5.31 +          && pointer_delta(p, new_region.start(), sizeof(char)) >= alignment()) {
    5.32          if (intersection.contains(p)) {
    5.33            intersection = MemRegion(p, intersection.end());
    5.34          } else {
    5.35 @@ -462,10 +462,10 @@
    5.36    // Is there top?
    5.37    if (intersection.end() < new_region.end()) { // Yes
    5.38      // Try to coalesce small pages into a large one.
    5.39 -    if (UseLargePages && page_size() >= os::large_page_size()) {
    5.40 -      HeapWord* p = (HeapWord*)round_down((intptr_t) intersection.end(), os::large_page_size());
    5.41 +    if (UseLargePages && page_size() >= alignment()) {
    5.42 +      HeapWord* p = (HeapWord*)round_down((intptr_t) intersection.end(), alignment());
    5.43        if (new_region.contains(p)
    5.44 -          && pointer_delta(new_region.end(), p, sizeof(char)) >= os::large_page_size()) {
    5.45 +          && pointer_delta(new_region.end(), p, sizeof(char)) >= alignment()) {
    5.46          if (intersection.contains(p)) {
    5.47            intersection = MemRegion(intersection.start(), p);
    5.48          } else {
    5.49 @@ -504,12 +504,12 @@
    5.50              // That's the only case we have to make an additional bias_region() call.
    5.51              HeapWord* start = invalid_region->start();
    5.52              HeapWord* end = invalid_region->end();
    5.53 -            if (UseLargePages && page_size() >= os::large_page_size()) {
    5.54 -              HeapWord *p = (HeapWord*)round_down((intptr_t) start, os::large_page_size());
    5.55 +            if (UseLargePages && page_size() >= alignment()) {
    5.56 +              HeapWord *p = (HeapWord*)round_down((intptr_t) start, alignment());
    5.57                if (new_region.contains(p)) {
    5.58                  start = p;
    5.59                }
    5.60 -              p = (HeapWord*)round_to((intptr_t) end, os::large_page_size());
    5.61 +              p = (HeapWord*)round_to((intptr_t) end, alignment());
    5.62                if (new_region.contains(end)) {
    5.63                  end = p;
    5.64                }
    5.65 @@ -526,7 +526,8 @@
    5.66  
    5.67  void MutableNUMASpace::initialize(MemRegion mr,
    5.68                                    bool clear_space,
    5.69 -                                  bool mangle_space) {
    5.70 +                                  bool mangle_space,
    5.71 +                                  bool setup_pages) {
    5.72    assert(clear_space, "Reallocation will destory data!");
    5.73    assert(lgrp_spaces()->length() > 0, "There should be at least one space");
    5.74  
    5.75 @@ -538,7 +539,7 @@
    5.76  
    5.77    // Compute chunk sizes
    5.78    size_t prev_page_size = page_size();
    5.79 -  set_page_size(UseLargePages ? os::large_page_size() : os::vm_page_size());
    5.80 +  set_page_size(UseLargePages ? alignment() : os::vm_page_size());
    5.81    HeapWord* rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
    5.82    HeapWord* rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
    5.83    size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
    5.84 @@ -666,7 +667,7 @@
    5.85      }
    5.86  
    5.87      // Clear space (set top = bottom) but never mangle.
    5.88 -    s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle);
    5.89 +    s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle, MutableSpace::DontSetupPages);
    5.90  
    5.91      set_adaptation_cycles(samples_count());
    5.92    }

     6.1 --- a/src/share/vm/gc_implementation/shared/mutableNUMASpace.hpp	Mon Jan 26 12:47:21 2009 -0800
     6.2 +++ b/src/share/vm/gc_implementation/shared/mutableNUMASpace.hpp	Tue Jan 27 18:13:59 2009 -0800
     6.3 @@ -82,8 +82,8 @@
     6.4      char* last_page_scanned()            { return _last_page_scanned; }
     6.5      void set_last_page_scanned(char* p)  { _last_page_scanned = p;    }
     6.6     public:
     6.7 -    LGRPSpace(int l) : _lgrp_id(l), _last_page_scanned(NULL), _allocation_failed(false) {
     6.8 -      _space = new MutableSpace();
     6.9 +    LGRPSpace(int l, size_t alignment) : _lgrp_id(l), _last_page_scanned(NULL), _allocation_failed(false) {
    6.10 +      _space = new MutableSpace(alignment);
    6.11        _alloc_rate = new AdaptiveWeightedAverage(NUMAChunkResizeWeight);
    6.12      }
    6.13      ~LGRPSpace() {
    6.14 @@ -183,10 +183,10 @@
    6.15  
    6.16   public:
    6.17    GrowableArray<LGRPSpace*>* lgrp_spaces() const     { return _lgrp_spaces;       }
    6.18 -  MutableNUMASpace();
    6.19 +  MutableNUMASpace(size_t alignment);
    6.20    virtual ~MutableNUMASpace();
    6.21    // Space initialization.
    6.22 -  virtual void initialize(MemRegion mr, bool clear_space, bool mangle_space);
    6.23 +  virtual void initialize(MemRegion mr, bool clear_space, bool mangle_space, bool setup_pages = SetupPages);
    6.24    // Update space layout if necessary. Do all adaptive resizing job.
    6.25    virtual void update();
    6.26    // Update allocation rate averages.

     7.1 --- a/src/share/vm/gc_implementation/shared/mutableSpace.cpp	Mon Jan 26 12:47:21 2009 -0800
     7.2 +++ b/src/share/vm/gc_implementation/shared/mutableSpace.cpp	Tue Jan 27 18:13:59 2009 -0800
     7.3 @@ -25,7 +25,10 @@
     7.4  # include "incls/_precompiled.incl"
     7.5  # include "incls/_mutableSpace.cpp.incl"
     7.6  
     7.7 -MutableSpace::MutableSpace(): ImmutableSpace(), _top(NULL) {
     7.8 +MutableSpace::MutableSpace(size_t alignment): ImmutableSpace(), _top(NULL), _alignment(alignment) {
     7.9 +  assert(MutableSpace::alignment() >= 0 &&
    7.10 +         MutableSpace::alignment() % os::vm_page_size() == 0,
    7.11 +         "Space should be aligned");
    7.12    _mangler = new MutableSpaceMangler(this);
    7.13  }
    7.14  
    7.15 @@ -33,16 +36,88 @@
    7.16    delete _mangler;
    7.17  }
    7.18  
    7.19 +void MutableSpace::numa_setup_pages(MemRegion mr, bool clear_space) {
    7.20 +  if (!mr.is_empty()) {
    7.21 +    size_t page_size = UseLargePages ? alignment() : os::vm_page_size();
    7.22 +    HeapWord *start = (HeapWord*)round_to((intptr_t) mr.start(), page_size);
    7.23 +    HeapWord *end =  (HeapWord*)round_down((intptr_t) mr.end(), page_size);
    7.24 +    if (end > start) {
    7.25 +      size_t size = pointer_delta(end, start, sizeof(char));
    7.26 +      if (clear_space) {
    7.27 +        // Prefer page reallocation to migration.
    7.28 +        os::free_memory((char*)start, size);
    7.29 +      }
    7.30 +      os::numa_make_global((char*)start, size);
    7.31 +    }
    7.32 +  }
    7.33 +}
    7.34 +
    7.35 +void MutableSpace::pretouch_pages(MemRegion mr) {
    7.36 +  for (volatile char *p = (char*)mr.start(); p < (char*)mr.end(); p += os::vm_page_size()) {
    7.37 +    char t = *p; *p = t;
    7.38 +  }
    7.39 +}
    7.40 +
    7.41  void MutableSpace::initialize(MemRegion mr,
    7.42                                bool clear_space,
    7.43 -                              bool mangle_space) {
    7.44 -  HeapWord* bottom = mr.start();
    7.45 -  HeapWord* end    = mr.end();
    7.46 +                              bool mangle_space,
    7.47 +                              bool setup_pages) {
    7.48  
    7.49 -  assert(Universe::on_page_boundary(bottom) && Universe::on_page_boundary(end),
    7.50 +  assert(Universe::on_page_boundary(mr.start()) && Universe::on_page_boundary(mr.end()),
    7.51           "invalid space boundaries");
    7.52 -  set_bottom(bottom);
    7.53 -  set_end(end);
    7.54 +
    7.55 +  if (setup_pages && (UseNUMA || AlwaysPreTouch)) {
    7.56 +    // The space may move left and right or expand/shrink.
    7.57 +    // We'd like to enforce the desired page placement.
    7.58 +    MemRegion head, tail;
    7.59 +    if (last_setup_region().is_empty()) {
    7.60 +      // If it's the first initialization don't limit the amount of work.
    7.61 +      head = mr;
    7.62 +      tail = MemRegion(mr.end(), mr.end());
    7.63 +    } else {
    7.64 +      // Is there an intersection with the address space?
    7.65 +      MemRegion intersection = last_setup_region().intersection(mr);
    7.66 +      if (intersection.is_empty()) {
    7.67 +        intersection = MemRegion(mr.end(), mr.end());
    7.68 +      }
    7.69 +      // All the sizes below are in words.
    7.70 +      size_t head_size = 0, tail_size = 0;
    7.71 +      if (mr.start() <= intersection.start()) {
    7.72 +        head_size = pointer_delta(intersection.start(), mr.start());
    7.73 +      }
    7.74 +      if(intersection.end() <= mr.end()) {
    7.75 +        tail_size = pointer_delta(mr.end(), intersection.end());
    7.76 +      }
    7.77 +      // Limit the amount of page manipulation if necessary.
    7.78 +      if (NUMASpaceResizeRate > 0 && !AlwaysPreTouch) {
    7.79 +        const size_t change_size = head_size + tail_size;
    7.80 +        const float setup_rate_words = NUMASpaceResizeRate >> LogBytesPerWord;
    7.81 +        head_size = MIN2((size_t)(setup_rate_words * head_size / change_size),
    7.82 +                         head_size);
    7.83 +        tail_size = MIN2((size_t)(setup_rate_words * tail_size / change_size),
    7.84 +                         tail_size);
    7.85 +      }
    7.86 +      head = MemRegion(intersection.start() - head_size, intersection.start());
    7.87 +      tail = MemRegion(intersection.end(), intersection.end() + tail_size);
    7.88 +    }
    7.89 +    assert(mr.contains(head) && mr.contains(tail), "Sanity");
    7.90 +
    7.91 +    if (UseNUMA) {
    7.92 +      numa_setup_pages(head, clear_space);
    7.93 +      numa_setup_pages(tail, clear_space);
    7.94 +    }
    7.95 +
    7.96 +    if (AlwaysPreTouch) {
    7.97 +      pretouch_pages(head);
    7.98 +      pretouch_pages(tail);
    7.99 +    }
   7.100 +
   7.101 +    // Remember where we stopped so that we can continue later.
   7.102 +    set_last_setup_region(MemRegion(head.start(), tail.end()));
   7.103 +  }
   7.104 +
   7.105 +  set_bottom(mr.start());
   7.106 +  set_end(mr.end());
   7.107  
   7.108    if (clear_space) {
   7.109      clear(mangle_space);

     8.1 --- a/src/share/vm/gc_implementation/shared/mutableSpace.hpp	Mon Jan 26 12:47:21 2009 -0800
     8.2 +++ b/src/share/vm/gc_implementation/shared/mutableSpace.hpp	Tue Jan 27 18:13:59 2009 -0800
     8.3 @@ -25,7 +25,10 @@
     8.4  // A MutableSpace is a subtype of ImmutableSpace that supports the
     8.5  // concept of allocation. This includes the concepts that a space may
     8.6  // be only partially full, and the querry methods that go with such
     8.7 -// an assumption.
     8.8 +// an assumption. MutableSpace is also responsible for minimizing the
     8.9 +// page allocation time by having the memory pretouched (with
    8.10 +// AlwaysPretouch) and for optimizing page placement on NUMA systems
    8.11 +// by make the underlying region interleaved (with UseNUMA).
    8.12  //
    8.13  // Invariant: (ImmutableSpace +) bottom() <= top() <= end()
    8.14  // top() is inclusive and end() is exclusive.
    8.15 @@ -37,15 +40,23 @@
    8.16  
    8.17    // Helper for mangling unused space in debug builds
    8.18    MutableSpaceMangler* _mangler;
    8.19 -
    8.20 +  // The last region which page had been setup to be interleaved.
    8.21 +  MemRegion _last_setup_region;
    8.22 +  size_t _alignment;
    8.23   protected:
    8.24    HeapWord* _top;
    8.25  
    8.26    MutableSpaceMangler* mangler() { return _mangler; }
    8.27  
    8.28 +  void numa_setup_pages(MemRegion mr, bool clear_space);
    8.29 +  void pretouch_pages(MemRegion mr);
    8.30 +
    8.31 +  void set_last_setup_region(MemRegion mr) { _last_setup_region = mr;   }
    8.32 +  MemRegion last_setup_region() const      { return _last_setup_region; }
    8.33 +
    8.34   public:
    8.35    virtual ~MutableSpace();
    8.36 -  MutableSpace();
    8.37 +  MutableSpace(size_t page_size);
    8.38  
    8.39    // Accessors
    8.40    HeapWord* top() const                    { return _top;    }
    8.41 @@ -57,13 +68,20 @@
    8.42    virtual void set_bottom(HeapWord* value) { _bottom = value; }
    8.43    virtual void set_end(HeapWord* value)    { _end = value; }
    8.44  
    8.45 +  size_t alignment()                       { return _alignment; }
    8.46 +
    8.47    // Returns a subregion containing all objects in this space.
    8.48    MemRegion used_region() { return MemRegion(bottom(), top()); }
    8.49  
    8.50 +  static const bool SetupPages = true;
    8.51 +  static const bool DontSetupPages = false;
    8.52 +
    8.53    // Initialization
    8.54    virtual void initialize(MemRegion mr,
    8.55                            bool clear_space,
    8.56 -                          bool mangle_space);
    8.57 +                          bool mangle_space,
    8.58 +                          bool setup_pages = SetupPages);
    8.59 +
    8.60    virtual void clear(bool mangle_space);
    8.61    // Does the usual initialization but optionally resets top to bottom.
    8.62  #if 0  // MANGLE_SPACE