Index: source/Utility/ConstString.cpp =================================================================== --- source/Utility/ConstString.cpp +++ source/Utility/ConstString.cpp @@ -18,6 +18,7 @@ #include "llvm/Support/FormatProviders.h" // for format_provider #include "llvm/Support/RWMutex.h" #include "llvm/Support/Threading.h" +#include "llvm/Support/xxhash.h" #include // for min #include @@ -31,14 +32,35 @@ class Pool { public: - typedef const char *StringPoolValueType; - typedef llvm::StringMap - StringPool; - typedef llvm::StringMapEntry StringPoolEntryType; + struct Entry { + uint64_t hash; + size_t key_len; + std::atomic value; + std::atomic next; + // string follows + void setValue(const char * val) { + value.store(val, std::memory_order_release); + } + const char * cstr() const { + return reinterpret_cast(this + 1); + } + llvm::StringRef str() const { + return llvm::StringRef(cstr(), key_len); + } + }; + + Pool() { + for (auto & a : m_string_pools) + a.store(nullptr, std::memory_order_release); + } + + typedef std::atomic StringPool; + typedef Entry StringPoolEntryType; static StringPoolEntryType & GetStringMapEntryFromKeyData(const char *keyData) { - return StringPoolEntryType::GetStringMapEntryFromKeyData(keyData); + StringPoolEntryType * p = reinterpret_cast(const_cast(keyData)); + return *(p - 1); } static size_t GetConstCStringLength(const char *ccstr) { @@ -46,32 +68,22 @@ // Since the entry is read only, and we derive the entry entirely from the // pointer, we don't need the lock. const StringPoolEntryType &entry = GetStringMapEntryFromKeyData(ccstr); - return entry.getKey().size(); + return entry.key_len; } return 0; } - StringPoolValueType GetMangledCounterpart(const char *ccstr) const { + const char * GetMangledCounterpart(const char *ccstr) const { if (ccstr != nullptr) { - const uint8_t h = hash(llvm::StringRef(ccstr)); - llvm::sys::SmartScopedReader rlock(m_string_pools[h].m_mutex); - return GetStringMapEntryFromKeyData(ccstr).getValue(); + return GetStringMapEntryFromKeyData(ccstr).value; } return nullptr; } bool SetMangledCounterparts(const char *key_ccstr, const char *value_ccstr) { if (key_ccstr != nullptr && value_ccstr != nullptr) { - { - const uint8_t h = hash(llvm::StringRef(key_ccstr)); - llvm::sys::SmartScopedWriter wlock(m_string_pools[h].m_mutex); - GetStringMapEntryFromKeyData(key_ccstr).setValue(value_ccstr); - } - { - const uint8_t h = hash(llvm::StringRef(value_ccstr)); - llvm::sys::SmartScopedWriter wlock(m_string_pools[h].m_mutex); - GetStringMapEntryFromKeyData(value_ccstr).setValue(key_ccstr); - } + GetStringMapEntryFromKeyData(key_ccstr).setValue(value_ccstr); + GetStringMapEntryFromKeyData(value_ccstr).setValue(key_ccstr); return true; } return false; @@ -91,21 +103,37 @@ const char *GetConstCStringWithStringRef(const llvm::StringRef &string_ref) { if (string_ref.data()) { - const uint8_t h = hash(string_ref); - - { - llvm::sys::SmartScopedReader rlock(m_string_pools[h].m_mutex); - auto it = m_string_pools[h].m_string_map.find(string_ref); - if (it != m_string_pools[h].m_string_map.end()) - return it->getKeyData(); + const uint64_t h = hash(string_ref); + while (true) { + auto * b = &bucket(h); + Entry * bref; + while ((bref = b->load())) { + if (bref->hash > h) + break; + if (bref->hash == h) { + // Hash collision, do a secondary sort on string + llvm::StringRef bstringref = bref->str(); + if (string_ref == bstringref) + return bref->cstr(); + if (bstringref > string_ref) + break; + } + b = &bref->next; + } + Entry * new_entry = reinterpret_cast( + malloc(sizeof(Entry) + string_ref.size() + 1)); + new_entry->hash = h; + new_entry->key_len = string_ref.size(); + new_entry->value.store(nullptr, std::memory_order_relaxed); + new_entry->next.store(bref, std::memory_order_relaxed); + char * dest = const_cast(new_entry->cstr()); + memcpy(dest, string_ref.data(), string_ref.size()); + dest[string_ref.size()] = 0; + if (b->compare_exchange_strong(bref, new_entry)) { + return dest; + } + free(new_entry); } - - llvm::sys::SmartScopedWriter wlock(m_string_pools[h].m_mutex); - StringPoolEntryType &entry = - *m_string_pools[h] - .m_string_map.insert(std::make_pair(string_ref, nullptr)) - .first; - return entry.getKeyData(); } return nullptr; } @@ -114,30 +142,11 @@ GetConstCStringAndSetMangledCounterPart(const char *demangled_cstr, const char *mangled_ccstr) { if (demangled_cstr != nullptr) { - const char *demangled_ccstr = nullptr; - - { - llvm::StringRef string_ref(demangled_cstr); - const uint8_t h = hash(string_ref); - llvm::sys::SmartScopedWriter wlock(m_string_pools[h].m_mutex); + const char *demangled_ccstr = GetConstCStringWithStringRef(demangled_cstr); - // Make string pool entry with the mangled counterpart already set - StringPoolEntryType &entry = - *m_string_pools[h] - .m_string_map.insert(std::make_pair(string_ref, mangled_ccstr)) - .first; - - // Extract the const version of the demangled_cstr - demangled_ccstr = entry.getKeyData(); - } - - { - // Now assign the demangled const string as the counterpart of the - // mangled const string... - const uint8_t h = hash(llvm::StringRef(mangled_ccstr)); - llvm::sys::SmartScopedWriter wlock(m_string_pools[h].m_mutex); - GetStringMapEntryFromKeyData(mangled_ccstr).setValue(demangled_ccstr); - } + // Now assign the demangled const string as the counterpart of the + // mangled const string... + GetStringMapEntryFromKeyData(mangled_ccstr).setValue(demangled_ccstr); // Return the constant demangled C string return demangled_ccstr; @@ -161,26 +170,27 @@ //------------------------------------------------------------------ size_t MemorySize() const { size_t mem_size = sizeof(Pool); + /* for (const auto &pool : m_string_pools) { llvm::sys::SmartScopedReader rlock(pool.m_mutex); for (const auto &entry : pool.m_string_map) mem_size += sizeof(StringPoolEntryType) + entry.getKey().size(); } + */ return mem_size; } protected: - uint8_t hash(const llvm::StringRef &s) const { - uint32_t h = llvm::HashString(s); - return ((h >> 24) ^ (h >> 16) ^ (h >> 8) ^ h) & 0xff; + static const unsigned hash_bits = 20; + uint64_t hash(const llvm::StringRef &s) const { + return llvm::xxHash64(s); } - struct PoolEntry { - mutable llvm::sys::SmartRWMutex m_mutex; - StringPool m_string_map; - }; + std::atomic & bucket(uint64_t hash) { + return m_string_pools[hash >> (64 - hash_bits)]; + } - std::array m_string_pools; + std::atomic m_string_pools[1 << hash_bits]; }; //----------------------------------------------------------------------