diff --git a/CMakeLists.txt b/CMakeLists.txt index 844ead632..0a31a5cff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,7 @@ option(HAVE_ICU "BUild with ICU library" OFF) option(HAVE_TCMALLOC "Build with tcmalloc library" ON) option(HAVE_LZ4 "Build with lz4 library" ON) option(HAVE_OPENMP "Build with OpenMP" ON) +option(HAVE_ANKERL "Build with ankerl::unordered_dense::* library" OFF) option(BUILD_TESTS "Build test suite" OFF) option(HAVE_WORD_ALIGNMENT "Build some structures as packed with 4-byte alignment" OFF) option(USE_UNITY_BUILD "Enable Unity Build" OFF) diff --git a/cmake/config.h.in b/cmake/config.h.in index b90d8b5d6..7390be97d 100644 --- a/cmake/config.h.in +++ b/cmake/config.h.in @@ -8,6 +8,7 @@ #cmakedefine HAVE_ICU 1 #cmakedefine HAVE_OVERPASS_XML 1 #cmakedefine HAVE_WORD_ALIGNMENT 1 +#cmakedefine HAVE_ANKERL 0 #cmakedefine OSMIUM_WITH_LZ4 1 #cmakedefine HAVE_LIBOSMIUM ${HAVE_LIBOSMIUM} #cmakedefine PACKAGE_VERSION "${PACKAGE_VERSION}" diff --git a/src/overpass_api/core/datatypes.h b/src/overpass_api/core/datatypes.h index dc480b769..6b187bf79 100644 --- a/src/overpass_api/core/datatypes.h +++ b/src/overpass_api/core/datatypes.h @@ -1289,19 +1289,19 @@ struct Tags_By_Id_Cache { public: template - std::unordered_map< typename T::Id_Type, kv_pairs> & get() = delete; + osm3s::unordered_map< typename T::Id_Type, kv_pairs> & get() = delete; private: - std::unordered_map< Node_Skeleton::Id_Type, kv_pairs > t_by_id_node; - std::unordered_map< Way_Skeleton::Id_Type, kv_pairs > t_by_id_way; - std::unordered_map< Relation_Skeleton::Id_Type, kv_pairs > t_by_id_relation; - std::unordered_map< Area_Skeleton::Id_Type, kv_pairs > t_by_id_area; + osm3s::unordered_map< Node_Skeleton::Id_Type, kv_pairs > t_by_id_node; + osm3s::unordered_map< Way_Skeleton::Id_Type, kv_pairs > t_by_id_way; + osm3s::unordered_map< Relation_Skeleton::Id_Type, kv_pairs > t_by_id_relation; + osm3s::unordered_map< Area_Skeleton::Id_Type, kv_pairs > t_by_id_area; }; -template<> inline std::unordered_map< Node_Skeleton::Id_Type, kv_pairs > & Tags_By_Id_Cache::get< Node_Skeleton >() { return t_by_id_node; } -template<> inline std::unordered_map< Way_Skeleton::Id_Type, kv_pairs > & Tags_By_Id_Cache::get< Way_Skeleton >() { return t_by_id_way; } -template<> inline std::unordered_map< Relation_Skeleton::Id_Type, kv_pairs > & Tags_By_Id_Cache::get< Relation_Skeleton >() { return t_by_id_relation; } -template<> inline std::unordered_map< Area_Skeleton::Id_Type, kv_pairs > & Tags_By_Id_Cache::get< Area_Skeleton >() { return t_by_id_area; } +template<> inline osm3s::unordered_map< Node_Skeleton::Id_Type, kv_pairs > & Tags_By_Id_Cache::get< Node_Skeleton >() { return t_by_id_node; } +template<> inline osm3s::unordered_map< Way_Skeleton::Id_Type, kv_pairs > & Tags_By_Id_Cache::get< Way_Skeleton >() { return t_by_id_way; } +template<> inline osm3s::unordered_map< Relation_Skeleton::Id_Type, kv_pairs > & Tags_By_Id_Cache::get< Relation_Skeleton >() { return t_by_id_relation; } +template<> inline osm3s::unordered_map< Area_Skeleton::Id_Type, kv_pairs > & Tags_By_Id_Cache::get< Area_Skeleton >() { return t_by_id_area; } using user_id_name_t = std::vector< std::pair< uint32, std::string > >; diff --git a/src/overpass_api/data/collect_items.h b/src/overpass_api/data/collect_items.h index 84b28894e..7f1908891 100644 --- a/src/overpass_api/data/collect_items.h +++ b/src/overpass_api/data/collect_items.h @@ -748,9 +748,14 @@ std::vector< Index > get_indexes_ Random_File< typename Skeleton::Id_Type, Index > current(rman.get_transaction()->random_index (current_skeleton_file_properties< Skeleton >())); for (auto it = ids.begin(); it != ids.end(); ++it) - result.push_back(current.get(it->val())); + { + auto v = current.get(it->val()); + if (result.empty() || !(result.back() == v)) + result.push_back(v); + } - std::sort(result.begin(), result.end()); + if (!std::is_sorted(result.begin(), result.end())) + std::sort(result.begin(), result.end()); result.erase(std::unique(result.begin(), result.end()), result.end()); if (rman.get_desired_timestamp() != NOW || get_attic_idxs) @@ -774,7 +779,8 @@ std::vector< Index > get_indexes_ for (const auto & it : idx_list_db.as_discrete(idx_list_ids)) result.push_back(it.object()); - std::sort(result.begin(), result.end()); + if (!std::is_sorted(result.begin(), result.end())) + std::sort(result.begin(), result.end()); result.erase(std::unique(result.begin(), result.end()), result.end()); } diff --git a/src/overpass_api/data/collect_members.h b/src/overpass_api/data/collect_members.h index 0b37b594f..68dc1df60 100644 --- a/src/overpass_api/data/collect_members.h +++ b/src/overpass_api/data/collect_members.h @@ -365,7 +365,7 @@ void filter_attic_elements // Confirm elements that are backed by meta data // Update element's expiration timestamp if a meta exists that is older than the current // expiration date and younger than timestamp - std::unordered_map< Index, std::unordered_map< typename Skeleton::Id_Type, std::pair< timestamp_t, timestamp_t > > > + osm3s::unordered_map< Index, osm3s::unordered_map< typename Skeleton::Id_Type, std::pair< timestamp_t, timestamp_t > > > timestamp_by_id_by_idx; for (typename std::map< Index, std::vector< Skeleton > >::const_iterator it = current.begin(); it != current.end(); ++it) @@ -390,7 +390,7 @@ void filter_attic_elements (attic_meta_file_properties< Skeleton >())); { - std::unordered_map< typename Skeleton::Id_Type, std::pair< timestamp_t, timestamp_t > > * entry = nullptr; + osm3s::unordered_map< typename Skeleton::Id_Type, std::pair< timestamp_t, timestamp_t > > * entry = nullptr; for (auto it = attic_meta_db.discrete_begin(idx_set.begin(), idx_set.end()); it != attic_meta_db.discrete_end(); ++it) { @@ -418,7 +418,7 @@ void filter_attic_elements (current_meta_file_properties< Skeleton >())); { - std::unordered_map< typename Skeleton::Id_Type, std::pair< timestamp_t, timestamp_t > > * entry = nullptr; + osm3s::unordered_map< typename Skeleton::Id_Type, std::pair< timestamp_t, timestamp_t > > * entry = nullptr; for (auto it = meta_db.discrete_begin(idx_set.begin(), idx_set.end()); it != meta_db.discrete_end(); ++it) { diff --git a/src/overpass_api/data/filter_ids_by_tags.h b/src/overpass_api/data/filter_ids_by_tags.h index 2629342a0..8133a76b6 100644 --- a/src/overpass_api/data/filter_ids_by_tags.h +++ b/src/overpass_api/data/filter_ids_by_tags.h @@ -241,7 +241,7 @@ struct Tag_Entry_Listener_Value_Regex : public Tag_Entry_Listener< Id_Type > std::string value_; std::vector< Regular_Expression* > conditions_; const std::vector< Id_Type >* old_ids_; - std::map< Id_Type, std::pair< timestamp_t, timestamp_t > > timestamps; + osm3s::unordered_map< Id_Type, std::pair< timestamp_t, timestamp_t > > timestamps; }; @@ -295,8 +295,7 @@ struct Tag_Entry_Listener_Key_Regex : public Tag_Entry_Listener< Id_Type > void commit_ids() { - for (typename std::map< Id_Type, std::pair< timestamp_t, timestamp_t > >::const_iterator it = timestamps.begin(); - it != timestamps.end(); ++it) + for (auto it = timestamps.begin(); it != timestamps.end(); ++it) { if (0 < it->second.first && it->second.first <= it->second.second) new_ids_.push_back(it->first); @@ -310,7 +309,7 @@ struct Tag_Entry_Listener_Key_Regex : public Tag_Entry_Listener< Id_Type > Regular_Expression* value_; const std::vector< Id_Type >* old_ids_; std::vector< Id_Type > new_ids_; - std::map< Id_Type, std::pair< timestamp_t, timestamp_t > > timestamps; + osm3s::unordered_map< Id_Type, std::pair< timestamp_t, timestamp_t > > timestamps; }; diff --git a/src/overpass_api/data/set_comparison.cc b/src/overpass_api/data/set_comparison.cc index 8bc25db34..03eb95dc1 100644 --- a/src/overpass_api/data/set_comparison.cc +++ b/src/overpass_api/data/set_comparison.cc @@ -368,12 +368,12 @@ std::vector< typename Skeleton::Id_Type > find_still_existing_skeletons template< typename Index, typename Skeleton > -std::map< typename Skeleton::Id_Type, OSM_Element_Metadata_Skeleton< typename Skeleton::Id_Type > > +osm3s::unordered_map< typename Skeleton::Id_Type, OSM_Element_Metadata_Skeleton< typename Skeleton::Id_Type > > find_meta_elements (Resource_Manager& rman, timestamp_t timestamp, const std::vector< Index >& idx_set, const std::vector< typename Skeleton::Id_Type >& searched_ids) { - std::map< typename Skeleton::Id_Type, OSM_Element_Metadata_Skeleton< typename Skeleton::Id_Type > > result; + osm3s::unordered_map< typename Skeleton::Id_Type, OSM_Element_Metadata_Skeleton< typename Skeleton::Id_Type > > result; Block_Backend< Index, OSM_Element_Metadata_Skeleton< typename Skeleton::Id_Type >, typename std::vector< Index >::const_iterator > @@ -475,14 +475,14 @@ void Set_Comparison::clear_nodes(Resource_Manager& rman, bool add_deletion_infor std::vector< Node_Skeleton::Id_Type > found_ids = find_still_existing_skeletons< Uint32_Index, Node_Skeleton >( rman, rman.get_desired_timestamp(), req, searched_ids); - std::map< Node_Skeleton::Id_Type, OSM_Element_Metadata_Skeleton< Node::Id_Type > > found_meta + osm3s::unordered_map< Node_Skeleton::Id_Type, OSM_Element_Metadata_Skeleton< Node::Id_Type > > found_meta = find_meta_elements< Uint32_Index, Node_Skeleton >(rman, rman.get_diff_to_timestamp(), req, searched_ids); for (std::vector< Node_With_Context >::const_iterator it = nodes.begin(); it != nodes.end(); ++it) { if (it->idx.val() != 0xffu) { - std::map< Node_Skeleton::Id_Type, OSM_Element_Metadata_Skeleton< Node::Id_Type > >::const_iterator + osm3s::unordered_map< Node_Skeleton::Id_Type, OSM_Element_Metadata_Skeleton< Node::Id_Type > >::const_iterator meta_it = found_meta.find(it->elem.id); result.different_nodes.push_back(std::make_pair(*it, Node_With_Context(monobound_binary_search(found_ids, it->elem.id) ? 0xfdu : 0xffu, @@ -511,10 +511,10 @@ void Set_Comparison::clear_nodes(Resource_Manager& rman, bool add_deletion_infor { if (it->first.idx.val() == 0xffu) { - it->first.idx = std::binary_search(found_ids.begin(), found_ids.end(), it->second.elem.id) ? 0xfdu : 0xffu; + it->first.idx = monobound_binary_search(found_ids, it->second.elem.id) ? 0xfdu : 0xffu; it->first.elem = Node_Skeleton(it->second.elem.id); - std::map< Node_Skeleton::Id_Type, OSM_Element_Metadata_Skeleton< Node::Id_Type > >::const_iterator + osm3s::unordered_map< Node_Skeleton::Id_Type, OSM_Element_Metadata_Skeleton< Node::Id_Type > >::const_iterator meta_it = found_meta.find(it->second.elem.id); if (meta_it != found_meta.end()) it->first.meta = meta_it->second; @@ -604,14 +604,14 @@ void Set_Comparison::clear_ways(Resource_Manager& rman, bool add_deletion_inform std::vector< Way_Skeleton::Id_Type > found_ids = find_still_existing_skeletons< Uint31_Index, Way_Skeleton >( rman, rman.get_desired_timestamp(), req, searched_ids); - std::map< Way_Skeleton::Id_Type, OSM_Element_Metadata_Skeleton< Way::Id_Type > > found_meta + osm3s::unordered_map< Way_Skeleton::Id_Type, OSM_Element_Metadata_Skeleton< Way::Id_Type > > found_meta = find_meta_elements< Uint31_Index, Way_Skeleton >(rman, rman.get_diff_to_timestamp(), req, searched_ids); for (std::vector< Way_With_Context >::const_iterator it = ways.begin(); it != ways.end(); ++it) { if (it->idx.val() != 0xffu) { - std::map< Way_Skeleton::Id_Type, OSM_Element_Metadata_Skeleton< Way::Id_Type > >::const_iterator + osm3s::unordered_map< Way_Skeleton::Id_Type, OSM_Element_Metadata_Skeleton< Way::Id_Type > >::const_iterator meta_it = found_meta.find(it->elem.id); result.different_ways.push_back(std::make_pair(*it, Way_With_Context(monobound_binary_search(found_ids, it->elem.id) ? 0xfdu : 0xffu, @@ -644,7 +644,7 @@ void Set_Comparison::clear_ways(Resource_Manager& rman, bool add_deletion_inform it->first.idx = monobound_binary_search(found_ids, it->second.elem.id) ? 0xfdu : 0xffu; it->first.elem = Way_Skeleton(it->second.elem.id); - std::map< Way_Skeleton::Id_Type, OSM_Element_Metadata_Skeleton< Way::Id_Type > >::const_iterator + osm3s::unordered_map< Way_Skeleton::Id_Type, OSM_Element_Metadata_Skeleton< Way::Id_Type > >::const_iterator meta_it = found_meta.find(it->second.elem.id); if (meta_it != found_meta.end()) it->first.meta = meta_it->second; @@ -735,7 +735,8 @@ void Set_Comparison::clear_relations(Resource_Manager& rman, bool add_deletion_i std::vector< Relation_Skeleton::Id_Type > found_ids = find_still_existing_skeletons< Uint31_Index, Relation_Skeleton >( rman, rman.get_diff_to_timestamp(), req, searched_ids); - std::map< Relation_Skeleton::Id_Type, OSM_Element_Metadata_Skeleton< Relation::Id_Type > > found_meta + + osm3s::unordered_map< Relation_Skeleton::Id_Type, OSM_Element_Metadata_Skeleton< Relation::Id_Type > > found_meta = find_meta_elements< Uint31_Index, Relation_Skeleton >( rman, rman.get_diff_to_timestamp(), req, searched_ids); @@ -743,7 +744,7 @@ void Set_Comparison::clear_relations(Resource_Manager& rman, bool add_deletion_i { if (it->idx.val() != 0xffu) { - std::map< Relation_Skeleton::Id_Type, OSM_Element_Metadata_Skeleton< Relation::Id_Type > >::const_iterator + osm3s::unordered_map< Relation_Skeleton::Id_Type, OSM_Element_Metadata_Skeleton< Relation::Id_Type > >::const_iterator meta_it = found_meta.find(it->elem.id); result.different_relations.push_back(std::make_pair(*it, Relation_With_Context(monobound_binary_search(found_ids, it->elem.id) ? 0xfdu : 0xffu, @@ -776,7 +777,7 @@ void Set_Comparison::clear_relations(Resource_Manager& rman, bool add_deletion_i it->first.idx = monobound_binary_search(found_ids, it->second.elem.id) ? 0xfdu : 0xffu; it->first.elem = Relation_Skeleton(it->second.elem.id); - std::map< Relation_Skeleton::Id_Type, OSM_Element_Metadata_Skeleton< Relation::Id_Type > >::const_iterator + osm3s::unordered_map< Relation_Skeleton::Id_Type, OSM_Element_Metadata_Skeleton< Relation::Id_Type > >::const_iterator meta_it = found_meta.find(it->second.elem.id); if (meta_it != found_meta.end()) it->first.meta = meta_it->second; diff --git a/src/overpass_api/data/tag_store.h b/src/overpass_api/data/tag_store.h index d5dd81eff..70c036e73 100644 --- a/src/overpass_api/data/tag_store.h +++ b/src/overpass_api/data/tag_store.h @@ -56,7 +56,7 @@ class Tag_Store const std::vector< std::pair< std::string, std::string > >* get(const Index& index, const Object& elem); private: - std::unordered_map< typename Object::Id_Type, std::vector< std::pair< std::string, std::string > > > tags_by_id; + osm3s::unordered_map< typename Object::Id_Type, std::vector< std::pair< std::string, std::string > > > tags_by_id; Resource_Manager* rman = nullptr; Transaction* transaction = nullptr; bool use_index = false; @@ -99,7 +99,7 @@ class Tag_Store< Uint31_Index, Derived_Structure > template< class Id_Type > void collect_attic_tags - (std::unordered_map< Id_Type, std::vector< std::pair< std::string, std::string > > >& tags_by_id, + (osm3s::unordered_map< Id_Type, std::vector< std::pair< std::string, std::string > > >& tags_by_id, const Block_Backend< Tag_Index_Local, Id_Type >& current_items_db, typename Block_Backend< Tag_Index_Local, Id_Type >::Range_Iterator& current_tag_it, const Block_Backend< Tag_Index_Local, Attic< Id_Type > >& attic_items_db, @@ -206,7 +206,7 @@ void collect_attic_tags template< class Id_Type > void collect_attic_tags - (std::unordered_map< Id_Type, std::vector< std::pair< std::string, std::string > > >& tags_by_id, + (osm3s::unordered_map< Id_Type, std::vector< std::pair< std::string, std::string > > >& tags_by_id, const Block_Backend< Tag_Index_Local, Id_Type >& current_items_db, typename Block_Backend< Tag_Index_Local, Id_Type >::Range_Iterator& current_tag_it, const Block_Backend< Tag_Index_Local, Attic< Id_Type > >& attic_items_db, @@ -263,7 +263,7 @@ void collect_tags template< class Id_Type > void collect_tags_single - (std::unordered_map< Id_Type, std::vector< std::pair< std::string, std::string > > >& tags_by_id, + (osm3s::unordered_map< Id_Type, std::vector< std::pair< std::string, std::string > > >& tags_by_id, const Block_Backend< Tag_Index_Local, Id_Type >& items_db, typename Block_Backend< Tag_Index_Local, Id_Type >::Range_Iterator& tag_it, const Id_Type id, uint32 coarse_index) @@ -305,7 +305,7 @@ void collect_tags_single template< class Id_Type > void collect_tags - (std::unordered_map< Id_Type, std::vector< std::pair< std::string, std::string > > >& tags_by_id, + (osm3s::unordered_map< Id_Type, std::vector< std::pair< std::string, std::string > > >& tags_by_id, const Block_Backend< Tag_Index_Local, Id_Type >& items_db, typename Block_Backend< Tag_Index_Local, Id_Type >::Range_Iterator& tag_it, const std::vector< Id_Type >& ids, uint32 coarse_index) @@ -350,7 +350,7 @@ void collect_tags template< class Id_Type > void collect_tags_framed - (std::unordered_map< Id_Type, std::vector< std::pair< std::string, std::string > > >& tags_by_id, + (osm3s::unordered_map< Id_Type, std::vector< std::pair< std::string, std::string > > >& tags_by_id, const Block_Backend< Tag_Index_Local, Id_Type >& items_db, typename Block_Backend< Tag_Index_Local, Id_Type >::Range_Iterator& tag_it, std::map< uint32, std::vector< Id_Type > >& ids_by_coarse, diff --git a/src/template_db/types.h b/src/template_db/types.h index 74ceba01f..7516cc1b4 100644 --- a/src/template_db/types.h +++ b/src/template_db/types.h @@ -19,6 +19,11 @@ #ifndef DE__OSM3S___TEMPLATE_DB__TYPES_H #define DE__OSM3S___TEMPLATE_DB__TYPES_H +#ifdef HAVE_CONFIG_H +#include +#undef VERSION +#endif + #include #include #include @@ -34,14 +39,29 @@ #include #include #include +#include #include +#include "unordered_dense.h" + #ifdef NATIVE_LARGE_FILES #define ftruncate64 ftruncate #define lseek64 lseek #define open64 open #endif +namespace osm3s { + template< class Key, class T > +#ifndef HAVE_ANKERL + using unordered_map = std::unordered_map< Key, T>; +#else + using unordered_map = ankerl::unordered_dense::map< Key, T>; +#endif + + template< class Key, class T > + using map = std::map< Key, T>; +} + typedef unsigned int uint; typedef char int8;