Two-threads bidirectional A*

coding/coding_tests/var_record_reader_test.cpp

@@ -52,13 +52,13 @@ UNIT_TEST(VarRecordReader_Simple)
   MemReader reader(&data[0], data.size());
   VarRecordReader<MemReader> recordReader(reader);
 
-  auto r = recordReader.ReadRecord(0);
+  auto r = recordReader.ReadRecord(0, 0);
   TEST_EQUAL(string(r.begin(), r.end()), "abc", ());
 
-  r = recordReader.ReadRecord(6 + longStringSize);
+  r = recordReader.ReadRecord(6 + longStringSize, 0);
   TEST_EQUAL(string(r.begin(), r.end()), "defg", ());
 
-  r = recordReader.ReadRecord(4);
+  r = recordReader.ReadRecord(4, 0);
   TEST_EQUAL(string(r.begin(), r.end()), longString, ());
 
   vector<pair<uint64_t, string>> forEachCalls;

coding/file_reader.cpp

@@ -57,6 +57,7 @@ class FileReader::FileReaderData
 
   uint64_t Size() const { return m_fileData.Size(); }
 
+//  std::mutex mut;
   void Read(uint64_t pos, void * p, size_t size)
   {
 #if LOG_FILE_READER_STATS
@@ -65,7 +66,8 @@ class FileReader::FileReaderData
       LOG(LINFO, ("FileReader", m_fileData.GetName(), m_readerCache.GetStatsStr()));
     }
 #endif
-
+    // TODO This cache should not be accessed from different threads.
+//    std::lock_guard guard(mut);
     return m_readerCache.Read(m_fileData, pos, p, size);
   }
 

coding/var_record_reader.hpp

@@ -1,5 +1,7 @@
 #pragma once
 
+#include "indexer/feature.hpp"
+
 #include "coding/byte_stream.hpp"
 #include "coding/reader.hpp"
 #include "coding/varint.hpp"
@@ -18,14 +20,15 @@ class VarRecordReader
 public:
   VarRecordReader(ReaderT const & reader) : m_reader(reader) {}
 
-  std::vector<uint8_t> ReadRecord(uint64_t const pos) const
+  std::vector<uint8_t> ReadRecord(uint64_t const pos, uint32_t index) const
   {
     ReaderSource source(m_reader);
     ASSERT_LESS(pos, source.Size(), ());
     source.Skip(pos);
     uint32_t const recordSize = ReadVarUint<uint32_t>(source);
     std::vector<uint8_t> buffer(recordSize);
     source.Read(buffer.data(), recordSize);
+    CheckFeatureTypeData(buffer, pos, index, recordSize);
     return buffer;
   }
 

data/countries.txt

@@ -1,5 +1,5 @@
 {
- "v": 200607,
+ "v": 200920,
  "id": "Countries",
  "g": [
   {

generator/generator_tests_support/routing_helpers.hpp

@@ -31,7 +31,7 @@ class TestGeometryLoader : public GeometryLoader
 {
 public:
   // GeometryLoader overrides:
-  void Load(uint32_t featureId, routing::RoadGeometry & road) override;
+  void Load(uint32_t featureId, bool isOutgoing, routing::RoadGeometry & road) override;
 
   void AddRoad(uint32_t featureId, bool oneWay, float speed,
                routing::RoadGeometry::Points const & points);

generator/restriction_collector.cpp

@@ -129,8 +129,8 @@ bool RestrictionCollector::ParseRestrictions(std::string const & path)
 Joint::Id RestrictionCollector::GetFirstCommonJoint(uint32_t firstFeatureId,
                                                     uint32_t secondFeatureId) const
 {
-  uint32_t const firstLen = m_indexGraph->GetGeometry().GetRoad(firstFeatureId).GetPointsCount();
-  uint32_t const secondLen = m_indexGraph->GetGeometry().GetRoad(secondFeatureId).GetPointsCount();
+  uint32_t const firstLen = m_indexGraph->GetGeometry().GetRoad(firstFeatureId, true /* isOutgoing */).GetPointsCount();
+  uint32_t const secondLen = m_indexGraph->GetGeometry().GetRoad(secondFeatureId, true /* isOutgoing */).GetPointsCount();
 
   auto const firstRoad = m_indexGraph->GetRoad(firstFeatureId);
   auto const secondRoad = m_indexGraph->GetRoad(secondFeatureId);
@@ -155,7 +155,7 @@ bool RestrictionCollector::FeatureHasPointWithCoords(uint32_t featureId,
                                                      m2::PointD const & coords) const
 {
   CHECK(m_indexGraph, ());
-  auto const & roadGeometry = m_indexGraph->GetGeometry().GetRoad(featureId);
+  auto const & roadGeometry = m_indexGraph->GetGeometry().GetRoad(featureId, true /* isOutgoing */);
   uint32_t const pointsCount = roadGeometry.GetPointsCount();
   for (uint32_t i = 0; i < pointsCount; ++i)
   {
@@ -245,7 +245,7 @@ bool RestrictionCollector::CheckAndProcessUTurn(Restriction::Type & restrictionT
 
     uint32_t & featureId = featureIds.back();
 
-    auto const & road = m_indexGraph->GetGeometry().GetRoad(featureId);
+    auto const & road = m_indexGraph->GetGeometry().GetRoad(featureId, true /* isOutgoing */);
     // Can not do UTurn from feature to the same feature if it is one way.
     if (road.IsOneWay())
       return false;

generator/routing_index_generator.cpp

@@ -173,7 +173,7 @@ class IndexGraphWrapper final
   routing::Segment GetFinishSegment() const { return {}; }
   bool ConvertToReal(routing::Segment const & /* segment */) const { return false; }
   routing::RouteWeight HeuristicCostEstimate(routing::Segment const & /* from */,
-                                             ms::LatLon const & /* to */)
+                                             ms::LatLon const & /* to */, bool)
   {
     CHECK(false, ("This method exists only for compatibility with IndexGraphStarterJoints"));
     return routing::GetAStarWeightZero<routing::RouteWeight>();
@@ -202,9 +202,9 @@ class IndexGraphWrapper final
   routing::RouteWeight GetAStarWeightEpsilon() { return routing::RouteWeight(0.0); }
   // @}
 
-  ms::LatLon const & GetPoint(routing::Segment const & s, bool forward)
+  ms::LatLon const & GetPoint(routing::Segment const & s, bool forward, bool isOutgoing)
   {
-    return m_graph.GetPoint(s, forward);
+    return m_graph.GetPoint(s, forward, isOutgoing);
   }
 
   void GetEdgesList(routing::Segment const & child, bool isOutgoing,
@@ -234,7 +234,7 @@ class IndexGraphWrapper final
   }
 
   template <typename Vertex>
-  routing::RouteWeight HeuristicCostEstimate(Vertex const & /* from */, m2::PointD const & /* to */)
+  routing::RouteWeight HeuristicCostEstimate(Vertex const & /* from */, m2::PointD const & /* to */, bool)
   {
     CHECK(false, ("This method should not be use, it is just for compatibility with "
                   "IndexGraphStarterJoints."));
@@ -256,7 +256,7 @@ class DijkstraWrapperJoints : public routing::IndexGraphStarterJoints<IndexGraph
   {
   }
 
-  Weight HeuristicCostEstimate(Vertex const & /* from */, Vertex const & /* to */) override
+  Weight HeuristicCostEstimate(Vertex const & /* from */, Vertex const & /* to */, bool isOutgoing) override
   {
     return routing::GetAStarWeightZero<Weight>();
   }

indexer/classificator.hpp

@@ -185,6 +185,7 @@ class Classificator
   uint32_t GetIndexForType(uint32_t t) const { return m_mapping.GetIndex(t); }
   // Throws std::out_of_range exception.
   uint32_t GetTypeForIndex(uint32_t i) const { return m_mapping.GetType(i); }
+  bool IsIndexOk(uint32_t ind) const { return m_mapping.IsIndexOk(ind); }
   bool IsTypeValid(uint32_t t) const { return m_mapping.HasIndex(t); }
 
   inline uint32_t GetCoastType() const { return m_coastType; }

indexer/data_source.cpp

@@ -134,6 +134,7 @@ unique_ptr<FeatureType> FeaturesLoaderGuard::GetOriginalOrEditedFeatureByIndex(u
 
 unique_ptr<FeatureType> FeaturesLoaderGuard::GetFeatureByIndex(uint32_t index) const
 {
+  CHECK(m_handle.IsAlive(), ());
   if (!m_handle.IsAlive())
     return {};
 
@@ -149,6 +150,7 @@ unique_ptr<FeatureType> FeaturesLoaderGuard::GetFeatureByIndex(uint32_t index) c
 
 unique_ptr<FeatureType> FeaturesLoaderGuard::GetOriginalFeatureByIndex(uint32_t index) const
 {
+  CHECK(m_handle.IsAlive(), ());
   return m_handle.IsAlive() ? m_source->GetOriginalFeature(index) : nullptr;
 }
 

indexer/feature.cpp

@@ -196,6 +196,13 @@ FeatureType::FeatureType(SharedLoadInfo const * loadInfo, vector<uint8_t> && buf
   m_header = Header(m_data);
 }
 
+FeatureType::FeatureType(feature::SharedLoadInfo const * loadInfo, std::vector<uint8_t> && buffer,
+                         feature::MetadataIndex const * metadataIndex, uint32_t offset)
+  : FeatureType(loadInfo, std::move(buffer), metadataIndex)
+{
+  m_offset = offset;
+}
+
 FeatureType::FeatureType(osm::MapObject const & emo)
 {
   HeaderGeomType headerGeomType = HeaderGeomType::Point;
@@ -275,19 +282,22 @@ void FeatureType::ParseTypes()
 
   size_t const count = GetTypesCount();
   uint32_t index = 0;
+  uint32_t type = 0;
+  size_t i = 0;
   try
   {
-    for (size_t i = 0; i < count; ++i)
+    for (i = 0; i < count; ++i)
     {
       index = ReadVarUint<uint32_t>(source);
-      m_types[i] = c.GetTypeForIndex(index);
+      type = c.GetTypeForIndex(index);
+      m_types[i] = type;
     }
   }
   catch (std::out_of_range const & ex)
   {
     LOG(LERROR, ("Incorrect type index for feature.FeatureID:", m_id, ". Incorrect index:", index,
                  ". Loaded feature types:", m_types, ". Total count of types:", count,
-                 ". Header:", m_header));
+                 ". Header:", m_header, "type:", type, "ex:", ex.what(), "i:", i));
     throw;
   }
 
@@ -815,3 +825,41 @@ feature::Metadata & FeatureType::GetMetadata()
   ParseMetadata();
   return m_metadata;
 }
+
+void CheckFeatureTypeData(std::vector<uint8_t> const & data, uint64_t offset, uint32_t fid,
+                          uint32_t recordSize)
+{
+  static std::atomic<size_t> counter = 0;
+  ++counter;
+  CHECK(!data.empty(), ("offset:", offset, "fid:", fid, "counter:", counter, "recordSize:", recordSize));
+
+  auto const header = data[0];
+  auto const typesOffset = sizeof(header);
+  Classificator & c = classif();
+  ArrayByteSource source(data.data() + typesOffset);
+
+  size_t const count = (header & feature::HEADER_MASK_TYPE) + 1;
+  uint32_t index = 0;
+  uint32_t type = 0;
+  size_t i = 0;
+  try
+  {
+    for (i = 0; i < count; ++i)
+    {
+      index = ReadVarUint<uint32_t>(source);
+      type = c.GetTypeForIndex(index);
+    }
+  }
+  catch (std::out_of_range const & ex)
+  {
+    LOG(LCRITICAL, ("Incorrect type index:", index, ". header:", header, ". Size of data:", data,
+                    ". count:", count, "type:", type, "ex:", ex.what(), "i:", i,
+                    "Is index ok:", c.IsIndexOk(index) ? "true" : "false", "counter:", counter.load(), "fid:", fid, "offset:", offset));
+    for (auto const i : data)
+      LOG(LINFO, (i));
+    throw;
+  }
+//  LOG(LINFO, ("Correct type index:", index, ". header:", header, ". Size of data:", data,
+//      ". count:", count, "type:", type, "i:", i,
+//      "Is index ok:", c.IsIndexOk(index) ? "true" : "false"));
+}

indexer/feature.hpp

@@ -37,6 +37,8 @@ class FeatureType
   FeatureType(feature::SharedLoadInfo const * loadInfo, std::vector<uint8_t> && buffer,
               feature::MetadataIndex const * metadataIndex);
   FeatureType(osm::MapObject const & emo);
+  FeatureType(feature::SharedLoadInfo const * loadInfo, std::vector<uint8_t> && buffer,
+              feature::MetadataIndex const * metadataIndex, uint32_t offset);
 
   feature::GeomType GetGeomType() const;
   FeatureParamsBase & GetParams() { return m_params; }
@@ -255,5 +257,10 @@ class FeatureType
 
   InnerGeomStat m_innerStats;
 
+  uint32_t m_offset = 0;
+
   DISALLOW_COPY_AND_MOVE(FeatureType);
 };
+
+void CheckFeatureTypeData(std::vector<uint8_t> const & data, uint64_t offset, uint32_t index,
+                          uint32_t recordSize);

indexer/features_offsets_table.cpp

@@ -19,6 +19,8 @@ using namespace std;
 
 namespace feature
 {
+  bool FeaturesOffsetsTable::m_log = false;
+
   void FeaturesOffsetsTable::Builder::PushOffset(uint32_t const offset)
   {
     ASSERT(m_offsets.empty() || m_offsets.back() < offset, ());
@@ -28,10 +30,14 @@ namespace feature
   FeaturesOffsetsTable::FeaturesOffsetsTable(succinct::elias_fano::elias_fano_builder & builder)
       : m_table(&builder)
   {
+    if (m_log)
+      LOG(LINFO, ("FeaturesOffsetsTable::FeaturesOffsetsTable()"));
   }
 
   FeaturesOffsetsTable::FeaturesOffsetsTable(string const & filePath)
   {
+    if (m_log)
+      LOG(LINFO, ("FeaturesOffsetsTable::FeaturesOffsetsTable() filePath:", filePath));
     m_pReader.reset(new MmapReader(filePath));
     succinct::mapper::map(m_table, reinterpret_cast<char const *>(m_pReader->Data()));
   }
@@ -68,6 +74,9 @@ namespace feature
   // static
   unique_ptr<FeaturesOffsetsTable> FeaturesOffsetsTable::Load(FilesContainerR const & cont)
   {
+    if (m_log)
+      LOG(LINFO, ("FeaturesOffsetsTable::Load()"));
+
     unique_ptr<FeaturesOffsetsTable> table(new FeaturesOffsetsTable());
 
     table->m_file.Open(cont.GetFileName());
@@ -90,6 +99,8 @@ namespace feature
 
   void FeaturesOffsetsTable::Save(string const & filePath)
   {
+    if (m_log)
+      LOG(LINFO, ("FeaturesOffsetsTable::Save()"));
     LOG(LINFO, ("Saving features offsets table to ", filePath));
     string const fileNameTmp = filePath + EXTENSION_TMP;
     succinct::mapper::freeze(m_table, fileNameTmp.c_str());
@@ -98,12 +109,16 @@ namespace feature
 
   uint32_t FeaturesOffsetsTable::GetFeatureOffset(size_t index) const
   {
+    if (m_log)
+      LOG(LINFO, (threads::GetCurrentThreadID(), "FeaturesOffsetsTable::GetFeatureOffset()", index));
     ASSERT_LESS(index, size(), ("Index out of bounds", index, size()));
     return static_cast<uint32_t>(m_table.select(index));
   }
 
   size_t FeaturesOffsetsTable::GetFeatureIndexbyOffset(uint32_t offset) const
   {
+    if (m_log)
+      LOG(LINFO, ("FeaturesOffsetsTable::GetFeatureIndexbyOffset()", offset));
     ASSERT_GREATER(size(), 0, ("We must not ask empty table"));
     ASSERT_LESS_OR_EQUAL(offset, m_table.select(size() - 1), ("Offset out of bounds", offset,
                                                      m_table.select(size() - 1)));

indexer/features_offsets_table.hpp

@@ -93,6 +93,8 @@ namespace feature
     ///         can be used in benchmarks, logging, etc.
     // size_t byte_size() { return static_cast<size_t>(succinct::mapper::size_of(m_table)); }
 
+    static void SetLog(bool flag) { m_log = flag; }
+
   private:
     FeaturesOffsetsTable(succinct::elias_fano::elias_fano_builder & builder);
     FeaturesOffsetsTable(std::string const & filePath);
@@ -105,6 +107,7 @@ namespace feature
 
     detail::MappedFile m_file;
     detail::MappedFile::Handle m_handle;
+    static bool m_log;
   };
 
   // Builds feature offsets table in an mwm or rebuilds an existing

indexer/features_vector.cpp

@@ -1,4 +1,5 @@
 #include "features_vector.hpp"
+#include "feature.hpp"
 #include "features_offsets_table.hpp"
 #include "data_factory.hpp"
 
@@ -7,9 +8,12 @@
 
 std::unique_ptr<FeatureType> FeaturesVector::GetByIndex(uint32_t index) const
 {
-  auto const ftOffset = m_table ? m_table->GetFeatureOffset(index) : index;
-  return std::make_unique<FeatureType>(&m_loadInfo, m_recordReader->ReadRecord(ftOffset),
-                                       m_metaidx.get());
+  uint32_t ftOffset = 0;
+  ftOffset = m_table ? m_table->GetFeatureOffset(index) : index;
+//  if (index == 333004 || index == 468000 || index == 664105)
+//    LOG(LCRITICAL, (index, ftOffset));
+  return std::make_unique<FeatureType>(&m_loadInfo, m_recordReader->ReadRecord(ftOffset, index),
+                                       m_metaidx.get(), ftOffset);
 }
 
 size_t FeaturesVector::GetNumFeatures() const

indexer/types_mapping.hpp

@@ -17,10 +17,12 @@ class IndexAndTypeMapping
   // Throws std::out_of_range exception.
   uint32_t GetType(uint32_t ind) const
   {
-    ASSERT_LESS ( ind, m_types.size(), () );
+    ASSERT_LESS(ind, m_types.size(), ());
     return m_types.at(ind);
   }
 
+  bool IsIndexOk(uint32_t ind) const { return ind < m_types.size(); }
+
   uint32_t GetIndex(uint32_t t) const;
 
   /// For Debug purposes only.

map/framework.cpp

@@ -1481,6 +1481,7 @@ void Framework::MemoryWarning()
 
 void Framework::EnterBackground()
 {
+  LOG(LINFO, ("Framework::EnterBackground()"));
   m_startBackgroundTime = base::Timer::LocalTime();
   settings::Set("LastEnterBackground", m_startBackgroundTime);
 
@@ -1507,6 +1508,7 @@ void Framework::EnterBackground()
 
 void Framework::EnterForeground()
 {
+  LOG(LINFO, ("Framework::EnterForeground()"));
   m_startForegroundTime = base::Timer::LocalTime();
   if (m_drapeEngine != nullptr && m_startBackgroundTime != 0.0)
   {