ysr@777: /* xdono@1279: * Copyright 2001-2009 Sun Microsystems, Inc. All Rights Reserved. ysr@777: * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ysr@777: * ysr@777: * This code is free software; you can redistribute it and/or modify it ysr@777: * under the terms of the GNU General Public License version 2 only, as ysr@777: * published by the Free Software Foundation. ysr@777: * ysr@777: * This code is distributed in the hope that it will be useful, but WITHOUT ysr@777: * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ysr@777: * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ysr@777: * version 2 for more details (a copy is included in the LICENSE file that ysr@777: * accompanied this code). ysr@777: * ysr@777: * You should have received a copy of the GNU General Public License version ysr@777: * 2 along with this work; if not, write to the Free Software Foundation, ysr@777: * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ysr@777: * ysr@777: * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, ysr@777: * CA 95054 USA or visit www.sun.com if you need additional information or ysr@777: * have any questions. ysr@777: * ysr@777: */ ysr@777: ysr@777: #include "incls/_precompiled.incl" ysr@777: #include "incls/_sparsePRT.cpp.incl" ysr@777: ysr@777: #define SPARSE_PRT_VERBOSE 0 ysr@777: iveresov@1696: #define UNROLL_CARD_LOOPS 1 ysr@777: ysr@777: void SparsePRT::init_iterator(SparsePRTIter* sprt_iter) { ysr@777: sprt_iter->init(this); ysr@777: } ysr@777: johnc@1242: void SparsePRTEntry::init(RegionIdx_t region_ind) { ysr@777: _region_ind = region_ind; ysr@777: _next_index = NullEntry; iveresov@1696: ysr@777: #if UNROLL_CARD_LOOPS iveresov@1696: assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry"); iveresov@1696: for (int i = 0; i < cards_num(); i += UnrollFactor) { iveresov@1696: _cards[i] = NullEntry; iveresov@1696: _cards[i + 1] = NullEntry; iveresov@1696: _cards[i + 2] = NullEntry; iveresov@1696: _cards[i + 3] = NullEntry; iveresov@1696: } ysr@777: #else iveresov@1696: for (int i = 0; i < cards_num(); i++) johnc@1242: _cards[i] = NullEntry; ysr@777: #endif ysr@777: } ysr@777: johnc@1242: bool SparsePRTEntry::contains_card(CardIdx_t card_index) const { ysr@777: #if UNROLL_CARD_LOOPS iveresov@1696: assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry"); iveresov@1696: for (int i = 0; i < cards_num(); i += UnrollFactor) { iveresov@1696: if (_cards[i] == card_index || iveresov@1696: _cards[i + 1] == card_index || iveresov@1696: _cards[i + 2] == card_index || iveresov@1696: _cards[i + 3] == card_index) return true; iveresov@1696: } ysr@777: #else iveresov@1696: for (int i = 0; i < cards_num(); i++) { ysr@777: if (_cards[i] == card_index) return true; ysr@777: } ysr@777: #endif ysr@777: // Otherwise, we're full. ysr@777: return false; ysr@777: } ysr@777: ysr@777: int SparsePRTEntry::num_valid_cards() const { ysr@777: int sum = 0; ysr@777: #if UNROLL_CARD_LOOPS iveresov@1696: assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry"); iveresov@1696: for (int i = 0; i < cards_num(); i += UnrollFactor) { iveresov@1696: sum += (_cards[i] != NullEntry); iveresov@1696: sum += (_cards[i + 1] != NullEntry); iveresov@1696: sum += (_cards[i + 2] != NullEntry); iveresov@1696: sum += (_cards[i + 3] != NullEntry); iveresov@1696: } ysr@777: #else iveresov@1696: for (int i = 0; i < cards_num(); i++) { iveresov@1696: sum += (_cards[i] != NullEntry); ysr@777: } ysr@777: #endif ysr@777: // Otherwise, we're full. ysr@777: return sum; ysr@777: } ysr@777: johnc@1242: SparsePRTEntry::AddCardResult SparsePRTEntry::add_card(CardIdx_t card_index) { ysr@777: #if UNROLL_CARD_LOOPS iveresov@1696: assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry"); iveresov@1696: CardIdx_t c; iveresov@1696: for (int i = 0; i < cards_num(); i += UnrollFactor) { iveresov@1696: c = _cards[i]; iveresov@1696: if (c == card_index) return found; iveresov@1696: if (c == NullEntry) { _cards[i] = card_index; return added; } iveresov@1696: c = _cards[i + 1]; iveresov@1696: if (c == card_index) return found; iveresov@1696: if (c == NullEntry) { _cards[i + 1] = card_index; return added; } iveresov@1696: c = _cards[i + 2]; iveresov@1696: if (c == card_index) return found; iveresov@1696: if (c == NullEntry) { _cards[i + 2] = card_index; return added; } iveresov@1696: c = _cards[i + 3]; iveresov@1696: if (c == card_index) return found; iveresov@1696: if (c == NullEntry) { _cards[i + 3] = card_index; return added; } iveresov@1696: } ysr@777: #else iveresov@1696: for (int i = 0; i < cards_num(); i++) { johnc@1242: CardIdx_t c = _cards[i]; ysr@777: if (c == card_index) return found; iveresov@1696: if (c == NullEntry) { _cards[i] = card_index; return added; } ysr@777: } ysr@777: #endif ysr@777: // Otherwise, we're full. ysr@777: return overflow; ysr@777: } ysr@777: johnc@1242: void SparsePRTEntry::copy_cards(CardIdx_t* cards) const { ysr@777: #if UNROLL_CARD_LOOPS iveresov@1696: assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry"); iveresov@1696: for (int i = 0; i < cards_num(); i += UnrollFactor) { iveresov@1696: cards[i] = _cards[i]; iveresov@1696: cards[i + 1] = _cards[i + 1]; iveresov@1696: cards[i + 2] = _cards[i + 2]; iveresov@1696: cards[i + 3] = _cards[i + 3]; iveresov@1696: } ysr@777: #else iveresov@1696: for (int i = 0; i < cards_num(); i++) { ysr@777: cards[i] = _cards[i]; ysr@777: } ysr@777: #endif ysr@777: } ysr@777: ysr@777: void SparsePRTEntry::copy_cards(SparsePRTEntry* e) const { ysr@777: copy_cards(&e->_cards[0]); ysr@777: } ysr@777: ysr@777: // ---------------------------------------------------------------------- ysr@777: ysr@777: RSHashTable::RSHashTable(size_t capacity) : ysr@777: _capacity(capacity), _capacity_mask(capacity-1), ysr@777: _occupied_entries(0), _occupied_cards(0), iveresov@1696: _entries((SparsePRTEntry*)NEW_C_HEAP_ARRAY(char, SparsePRTEntry::size() * capacity)), johnc@1242: _buckets(NEW_C_HEAP_ARRAY(int, capacity)), ysr@777: _free_list(NullEntry), _free_region(0) ysr@777: { ysr@777: clear(); ysr@777: } ysr@777: ysr@777: RSHashTable::~RSHashTable() { ysr@777: if (_entries != NULL) { ysr@777: FREE_C_HEAP_ARRAY(SparsePRTEntry, _entries); ysr@777: _entries = NULL; ysr@777: } ysr@777: if (_buckets != NULL) { johnc@1242: FREE_C_HEAP_ARRAY(int, _buckets); ysr@777: _buckets = NULL; ysr@777: } ysr@777: } ysr@777: ysr@777: void RSHashTable::clear() { ysr@777: _occupied_entries = 0; ysr@777: _occupied_cards = 0; ysr@777: guarantee(_entries != NULL, "INV"); ysr@777: guarantee(_buckets != NULL, "INV"); johnc@1242: johnc@1242: guarantee(_capacity <= ((size_t)1 << (sizeof(int)*BitsPerByte-1)) - 1, johnc@1242: "_capacity too large"); johnc@1242: ysr@777: // This will put -1 == NullEntry in the key field of all entries. iveresov@1696: memset(_entries, NullEntry, _capacity * SparsePRTEntry::size()); iveresov@1696: memset(_buckets, NullEntry, _capacity * sizeof(int)); ysr@777: _free_list = NullEntry; ysr@777: _free_region = 0; ysr@777: } ysr@777: johnc@1242: bool RSHashTable::add_card(RegionIdx_t region_ind, CardIdx_t card_index) { ysr@777: SparsePRTEntry* e = entry_for_region_ind_create(region_ind); ysr@777: assert(e != NULL && e->r_ind() == region_ind, ysr@777: "Postcondition of call above."); ysr@777: SparsePRTEntry::AddCardResult res = e->add_card(card_index); ysr@777: if (res == SparsePRTEntry::added) _occupied_cards++; ysr@777: #if SPARSE_PRT_VERBOSE ysr@777: gclog_or_tty->print_cr(" after add_card[%d]: valid-cards = %d.", iveresov@1696: pointer_delta(e, _entries, SparsePRTEntry::size()), iveresov@1696: e->num_valid_cards()); ysr@777: #endif ysr@777: assert(e->num_valid_cards() > 0, "Postcondition"); ysr@777: return res != SparsePRTEntry::overflow; ysr@777: } ysr@777: johnc@1242: bool RSHashTable::get_cards(RegionIdx_t region_ind, CardIdx_t* cards) { johnc@1242: int ind = (int) (region_ind & capacity_mask()); johnc@1242: int cur_ind = _buckets[ind]; ysr@777: SparsePRTEntry* cur; ysr@777: while (cur_ind != NullEntry && ysr@777: (cur = entry(cur_ind))->r_ind() != region_ind) { ysr@777: cur_ind = cur->next_index(); ysr@777: } ysr@777: ysr@777: if (cur_ind == NullEntry) return false; ysr@777: // Otherwise... ysr@777: assert(cur->r_ind() == region_ind, "Postcondition of loop + test above."); ysr@777: assert(cur->num_valid_cards() > 0, "Inv"); ysr@777: cur->copy_cards(cards); ysr@777: return true; ysr@777: } ysr@777: iveresov@1696: SparsePRTEntry* RSHashTable::get_entry(RegionIdx_t region_ind) { iveresov@1696: int ind = (int) (region_ind & capacity_mask()); iveresov@1696: int cur_ind = _buckets[ind]; iveresov@1696: SparsePRTEntry* cur; iveresov@1696: while (cur_ind != NullEntry && iveresov@1696: (cur = entry(cur_ind))->r_ind() != region_ind) { iveresov@1696: cur_ind = cur->next_index(); iveresov@1696: } iveresov@1696: iveresov@1696: if (cur_ind == NullEntry) return NULL; iveresov@1696: // Otherwise... iveresov@1696: assert(cur->r_ind() == region_ind, "Postcondition of loop + test above."); iveresov@1696: assert(cur->num_valid_cards() > 0, "Inv"); iveresov@1696: return cur; iveresov@1696: } iveresov@1696: johnc@1242: bool RSHashTable::delete_entry(RegionIdx_t region_ind) { johnc@1242: int ind = (int) (region_ind & capacity_mask()); johnc@1242: int* prev_loc = &_buckets[ind]; johnc@1242: int cur_ind = *prev_loc; ysr@777: SparsePRTEntry* cur; ysr@777: while (cur_ind != NullEntry && ysr@777: (cur = entry(cur_ind))->r_ind() != region_ind) { ysr@777: prev_loc = cur->next_index_addr(); ysr@777: cur_ind = *prev_loc; ysr@777: } ysr@777: ysr@777: if (cur_ind == NullEntry) return false; ysr@777: // Otherwise, splice out "cur". ysr@777: *prev_loc = cur->next_index(); ysr@777: _occupied_cards -= cur->num_valid_cards(); ysr@777: free_entry(cur_ind); ysr@777: _occupied_entries--; ysr@777: return true; ysr@777: } ysr@777: johnc@1242: SparsePRTEntry* johnc@1242: RSHashTable::entry_for_region_ind(RegionIdx_t region_ind) const { ysr@777: assert(occupied_entries() < capacity(), "Precondition"); johnc@1242: int ind = (int) (region_ind & capacity_mask()); johnc@1242: int cur_ind = _buckets[ind]; ysr@777: SparsePRTEntry* cur; ysr@777: while (cur_ind != NullEntry && ysr@777: (cur = entry(cur_ind))->r_ind() != region_ind) { ysr@777: cur_ind = cur->next_index(); ysr@777: } ysr@777: ysr@777: if (cur_ind != NullEntry) { ysr@777: assert(cur->r_ind() == region_ind, "Loop postcondition + test"); ysr@777: return cur; ysr@777: } else { ysr@777: return NULL; ysr@777: } ysr@777: } ysr@777: johnc@1242: SparsePRTEntry* johnc@1242: RSHashTable::entry_for_region_ind_create(RegionIdx_t region_ind) { ysr@777: SparsePRTEntry* res = entry_for_region_ind(region_ind); ysr@777: if (res == NULL) { johnc@1242: int new_ind = alloc_entry(); ysr@777: assert(0 <= new_ind && (size_t)new_ind < capacity(), "There should be room."); ysr@777: res = entry(new_ind); ysr@777: res->init(region_ind); ysr@777: // Insert at front. johnc@1242: int ind = (int) (region_ind & capacity_mask()); ysr@777: res->set_next_index(_buckets[ind]); ysr@777: _buckets[ind] = new_ind; ysr@777: _occupied_entries++; ysr@777: } ysr@777: return res; ysr@777: } ysr@777: johnc@1242: int RSHashTable::alloc_entry() { johnc@1242: int res; ysr@777: if (_free_list != NullEntry) { ysr@777: res = _free_list; ysr@777: _free_list = entry(res)->next_index(); ysr@777: return res; ysr@777: } else if ((size_t) _free_region+1 < capacity()) { ysr@777: res = _free_region; ysr@777: _free_region++; ysr@777: return res; ysr@777: } else { ysr@777: return NullEntry; ysr@777: } ysr@777: } ysr@777: johnc@1242: void RSHashTable::free_entry(int fi) { ysr@777: entry(fi)->set_next_index(_free_list); ysr@777: _free_list = fi; ysr@777: } ysr@777: ysr@777: void RSHashTable::add_entry(SparsePRTEntry* e) { ysr@777: assert(e->num_valid_cards() > 0, "Precondition."); ysr@777: SparsePRTEntry* e2 = entry_for_region_ind_create(e->r_ind()); ysr@777: e->copy_cards(e2); ysr@777: _occupied_cards += e2->num_valid_cards(); ysr@777: assert(e2->num_valid_cards() > 0, "Postcondition."); ysr@777: } ysr@777: johnc@1242: CardIdx_t /* RSHashTable:: */ RSHashTableIter::find_first_card_in_list() { johnc@1242: CardIdx_t res; ysr@777: while (_bl_ind != RSHashTable::NullEntry) { ysr@777: res = _rsht->entry(_bl_ind)->card(0); ysr@777: if (res != SparsePRTEntry::NullEntry) { ysr@777: return res; ysr@777: } else { ysr@777: _bl_ind = _rsht->entry(_bl_ind)->next_index(); ysr@777: } ysr@777: } ysr@777: // Otherwise, none found: ysr@777: return SparsePRTEntry::NullEntry; ysr@777: } ysr@777: johnc@1242: size_t /* RSHashTable:: */ RSHashTableIter::compute_card_ind(CardIdx_t ci) { ysr@777: return ysr@777: _heap_bot_card_ind tonyp@1377: + (_rsht->entry(_bl_ind)->r_ind() * HeapRegion::CardsPerRegion) ysr@777: + ci; ysr@777: } ysr@777: ysr@777: bool /* RSHashTable:: */ RSHashTableIter::has_next(size_t& card_index) { ysr@777: _card_ind++; johnc@1242: CardIdx_t ci; iveresov@1696: if (_card_ind < SparsePRTEntry::cards_num() && ysr@777: ((ci = _rsht->entry(_bl_ind)->card(_card_ind)) != ysr@777: SparsePRTEntry::NullEntry)) { ysr@777: card_index = compute_card_ind(ci); ysr@777: return true; ysr@777: } ysr@777: // Otherwise, must find the next valid entry. ysr@777: _card_ind = 0; ysr@777: ysr@777: if (_bl_ind != RSHashTable::NullEntry) { ysr@777: _bl_ind = _rsht->entry(_bl_ind)->next_index(); ysr@777: ci = find_first_card_in_list(); ysr@777: if (ci != SparsePRTEntry::NullEntry) { ysr@777: card_index = compute_card_ind(ci); ysr@777: return true; ysr@777: } ysr@777: } ysr@777: // If we didn't return above, must go to the next non-null table index. ysr@777: _tbl_ind++; ysr@777: while ((size_t)_tbl_ind < _rsht->capacity()) { ysr@777: _bl_ind = _rsht->_buckets[_tbl_ind]; ysr@777: ci = find_first_card_in_list(); ysr@777: if (ci != SparsePRTEntry::NullEntry) { ysr@777: card_index = compute_card_ind(ci); ysr@777: return true; ysr@777: } ysr@777: // Otherwise, try next entry. ysr@777: _tbl_ind++; ysr@777: } ysr@777: // Otherwise, there were no entry. ysr@777: return false; ysr@777: } ysr@777: johnc@1242: bool RSHashTable::contains_card(RegionIdx_t region_index, CardIdx_t card_index) const { ysr@777: SparsePRTEntry* e = entry_for_region_ind(region_index); ysr@777: return (e != NULL && e->contains_card(card_index)); ysr@777: } ysr@777: ysr@777: size_t RSHashTable::mem_size() const { johnc@1242: return sizeof(this) + iveresov@1696: capacity() * (SparsePRTEntry::size() + sizeof(int)); ysr@777: } ysr@777: ysr@777: // ---------------------------------------------------------------------- ysr@777: ysr@777: SparsePRT* SparsePRT::_head_expanded_list = NULL; ysr@777: ysr@777: void SparsePRT::add_to_expanded_list(SparsePRT* sprt) { ysr@777: // We could expand multiple times in a pause -- only put on list once. ysr@777: if (sprt->expanded()) return; ysr@777: sprt->set_expanded(true); ysr@777: SparsePRT* hd = _head_expanded_list; ysr@777: while (true) { ysr@777: sprt->_next_expanded = hd; ysr@777: SparsePRT* res = ysr@777: (SparsePRT*) ysr@777: Atomic::cmpxchg_ptr(sprt, &_head_expanded_list, hd); ysr@777: if (res == hd) return; ysr@777: else hd = res; ysr@777: } ysr@777: } ysr@777: johnc@1242: ysr@777: SparsePRT* SparsePRT::get_from_expanded_list() { ysr@777: SparsePRT* hd = _head_expanded_list; ysr@777: while (hd != NULL) { ysr@777: SparsePRT* next = hd->next_expanded(); ysr@777: SparsePRT* res = ysr@777: (SparsePRT*) ysr@777: Atomic::cmpxchg_ptr(next, &_head_expanded_list, hd); ysr@777: if (res == hd) { ysr@777: hd->set_next_expanded(NULL); ysr@777: return hd; ysr@777: } else { ysr@777: hd = res; ysr@777: } ysr@777: } ysr@777: return NULL; ysr@777: } ysr@777: ysr@777: ysr@777: void SparsePRT::cleanup_all() { ysr@777: // First clean up all expanded tables so they agree on next and cur. ysr@777: SparsePRT* sprt = get_from_expanded_list(); ysr@777: while (sprt != NULL) { ysr@777: sprt->cleanup(); ysr@777: sprt = get_from_expanded_list(); ysr@777: } ysr@777: } ysr@777: ysr@777: ysr@777: SparsePRT::SparsePRT(HeapRegion* hr) : ysr@777: _expanded(false), _next_expanded(NULL) ysr@777: { ysr@777: _cur = new RSHashTable(InitialCapacity); ysr@777: _next = _cur; ysr@777: } ysr@777: johnc@1242: ysr@777: SparsePRT::~SparsePRT() { ysr@777: assert(_next != NULL && _cur != NULL, "Inv"); ysr@777: if (_cur != _next) { delete _cur; } ysr@777: delete _next; ysr@777: } ysr@777: ysr@777: ysr@777: size_t SparsePRT::mem_size() const { ysr@777: // We ignore "_cur" here, because it either = _next, or else it is ysr@777: // on the deleted list. ysr@777: return sizeof(this) + _next->mem_size(); ysr@777: } ysr@777: johnc@1242: bool SparsePRT::add_card(RegionIdx_t region_id, CardIdx_t card_index) { ysr@777: #if SPARSE_PRT_VERBOSE ysr@777: gclog_or_tty->print_cr(" Adding card %d from region %d to region %d sparse.", ysr@777: card_index, region_id, _hr->hrs_index()); ysr@777: #endif ysr@777: if (_next->occupied_entries() * 2 > _next->capacity()) { ysr@777: expand(); ysr@777: } ysr@777: return _next->add_card(region_id, card_index); ysr@777: } ysr@777: johnc@1242: bool SparsePRT::get_cards(RegionIdx_t region_id, CardIdx_t* cards) { ysr@777: return _next->get_cards(region_id, cards); ysr@777: } ysr@777: iveresov@1696: SparsePRTEntry* SparsePRT::get_entry(RegionIdx_t region_id) { iveresov@1696: return _next->get_entry(region_id); iveresov@1696: } iveresov@1696: johnc@1242: bool SparsePRT::delete_entry(RegionIdx_t region_id) { ysr@777: return _next->delete_entry(region_id); ysr@777: } ysr@777: ysr@777: void SparsePRT::clear() { ysr@777: // If they differ, _next is bigger then cur, so next has no chance of ysr@777: // being the initial size. ysr@777: if (_next != _cur) { ysr@777: delete _next; ysr@777: } ysr@777: ysr@777: if (_cur->capacity() != InitialCapacity) { ysr@777: delete _cur; ysr@777: _cur = new RSHashTable(InitialCapacity); ysr@777: } else { ysr@777: _cur->clear(); ysr@777: } ysr@777: _next = _cur; ysr@777: } ysr@777: ysr@777: void SparsePRT::cleanup() { apetrusenko@1480: // Make sure that the current and next tables agree. apetrusenko@1480: if (_cur != _next) { apetrusenko@1480: delete _cur; apetrusenko@1480: } ysr@777: _cur = _next; tonyp@1052: set_expanded(false); ysr@777: } ysr@777: ysr@777: void SparsePRT::expand() { ysr@777: RSHashTable* last = _next; ysr@777: _next = new RSHashTable(last->capacity() * 2); ysr@777: ysr@777: #if SPARSE_PRT_VERBOSE ysr@777: gclog_or_tty->print_cr(" Expanded sparse table for %d to %d.", ysr@777: _hr->hrs_index(), _next->capacity()); ysr@777: #endif ysr@777: for (size_t i = 0; i < last->capacity(); i++) { ysr@777: SparsePRTEntry* e = last->entry((int)i); ysr@777: if (e->valid_entry()) { ysr@777: #if SPARSE_PRT_VERBOSE ysr@777: gclog_or_tty->print_cr(" During expansion, transferred entry for %d.", ysr@777: e->r_ind()); ysr@777: #endif ysr@777: _next->add_entry(e); ysr@777: } ysr@777: } apetrusenko@1480: if (last != _cur) { apetrusenko@1480: delete last; apetrusenko@1480: } ysr@777: add_to_expanded_list(this); ysr@777: }