diff --git a/coding/internal/xmlparser.hpp b/coding/internal/xmlparser.hpp index b453e3dbed..effc3d5e53 100644 --- a/coding/internal/xmlparser.hpp +++ b/coding/internal/xmlparser.hpp @@ -7,6 +7,9 @@ #pragma clang diagnostic ignored "-Wunused-parameter" #endif +#include +#include + #define XML_STATIC #include "3party/expat/expat_impl.h" @@ -76,14 +79,15 @@ public: m_charData.append(pszData, nLength); } - void PrintError() + std::string GetErrorMessage() { - if (BaseT::GetErrorCode() != XML_ERROR_NONE) - { - LOG(LDEBUG, ("XML parse error at line", - BaseT::GetCurrentLineNumber(), - "and byte", BaseT::GetCurrentByteIndex())); - } + if (BaseT::GetErrorCode() == XML_ERROR_NONE) + return {}; + + std::stringstream s; + s << "XML parse error at line " << BaseT::GetCurrentLineNumber() + << " and byte " << BaseT::GetCurrentByteIndex(); + return s.str(); } private: diff --git a/coding/parse_xml.hpp b/coding/parse_xml.hpp index 5a129fbe02..8afa49170e 100644 --- a/coding/parse_xml.hpp +++ b/coding/parse_xml.hpp @@ -3,55 +3,25 @@ #include "coding/internal/xmlparser.hpp" #include "base/assert.hpp" +#include "base/exception.hpp" #include #include +#include -template -uint64_t ParseXMLSequence(SequenceT & source, XMLDispatcherT & dispatcher, bool useCharData = false) -{ - // Create the parser - XmlParser parser(dispatcher, useCharData); - if (!parser.Create()) - return 0; +DECLARE_EXCEPTION(XmlParseError, RootException); - uint32_t const BUFFER_SIZE = 16 * 1024; - - uint64_t res = 0; - uint64_t readed; - do - { - char * buffer = static_cast(parser.GetBuffer(BUFFER_SIZE)); - ASSERT(buffer, ()); - - readed = source.Read(buffer, BUFFER_SIZE); - if (readed == 0) - return res; - - if (!parser.ParseBuffer(static_cast(readed), false)) - { - parser.PrintError(); - return res; - } - - res += readed; - } while (readed == BUFFER_SIZE); - - return res; -} - -template -class ParserXMLSequence +template +class XMLSequenceParser { public: - ParserXMLSequence(SequenceT & source, XMLDispatcherT & dispatcher) + XMLSequenceParser(Sequence & source, XMLDispatcher & dispatcher, bool useCharData = false) : m_res(0) - , m_readed(0) + , m_numRead(0) , m_source(source) - , m_parser(dispatcher, false /* useCharData */) + , m_parser(dispatcher, useCharData) { CHECK(m_parser.Create(), ()); - } bool Read() @@ -59,51 +29,58 @@ public: char * buffer = static_cast(m_parser.GetBuffer(kBufferSize)); ASSERT(buffer, ()); - m_readed = m_source.Read(buffer, kBufferSize); - if (m_readed == 0) + m_numRead = m_source.Read(buffer, kBufferSize); + if (m_numRead == 0) return false; - if (!m_parser.ParseBuffer(static_cast(m_readed), false)) - { - m_parser.PrintError(); - return false; - } + if (!m_parser.ParseBuffer(static_cast(m_numRead), false)) + MYTHROW(XmlParseError, (m_parser.GetErrorMessage())); - m_res += m_readed; - return m_readed == kBufferSize; + m_res += m_numRead; + return m_numRead == kBufferSize; } private: uint32_t static const kBufferSize = 16 * 1024; - uint64_t m_res; - uint64_t m_readed; - SequenceT & m_source; - XmlParser m_parser; + uint64_t m_res = 0; + uint64_t m_numRead = 0; + Sequence & m_source; + XmlParser m_parser; }; -namespace +template +class SequenceAdapter { - -template class SequenceAdapter -{ - SourceT & m_source; public: - SequenceAdapter(SourceT & source) : m_source(source) {} + SequenceAdapter(Source & source) : m_source(source) {} + uint64_t Read(void * p, uint64_t size) { size_t const correctSize = static_cast(std::min(size, m_source.Size())); m_source.Read(p, correctSize); return correctSize; } + +private: + Source & m_source; }; -} -template -bool ParseXML(SourceT & source, XMLDispatcherT & dispatcher, bool useCharData = false) +template +bool ParseXML(Source & source, XMLDispatcher & dispatcher, bool useCharData = false) { - uint64_t const size = source.Size(); - SequenceAdapter adapter(source); - return (ParseXMLSequence(adapter, dispatcher, useCharData) == size); + SequenceAdapter adapter(source); + XMLSequenceParser parser(adapter, dispatcher, useCharData); + try + { + while (parser.Read()) /* empty */; + } + catch (std::exception const & e) + { + LOG(LERROR, (e.what())); + return false; + } + + return true; } diff --git a/generator/osm_source.cpp b/generator/osm_source.cpp index 07a3601970..6936543c17 100644 --- a/generator/osm_source.cpp +++ b/generator/osm_source.cpp @@ -22,7 +22,6 @@ #include "defines.hpp" using namespace std; -using namespace base::thread_pool::computational; namespace generator { @@ -110,23 +109,21 @@ void AddElementToCache(cache::IntermediateDataWriter & cache, OsmElement & eleme void BuildIntermediateDataFromXML(SourceReader & stream, cache::IntermediateDataWriter & cache, TownsDumper & towns) { - XMLSource parser([&](OsmElement * element) + ProcessorOsmElementsFromXml processorOsmElementsFromXml(stream); + OsmElement element; + while (processorOsmElementsFromXml.TryRead(element)) { - towns.CheckElement(*element); - AddElementToCache(cache, *element); - }); - ParseXMLSequence(stream, parser); + towns.CheckElement(element); + AddElementToCache(cache, element); + } } void ProcessOsmElementsFromXML(SourceReader & stream, function processor) { - ProcessorXmlElementsFromXml processorXmlElementsFromXml(stream); + ProcessorOsmElementsFromXml processorOsmElementsFromXml(stream); OsmElement element; - while (processorXmlElementsFromXml.TryRead(element)) - { + while (processorOsmElementsFromXml.TryRead(element)) processor(&element); - element = {}; - } } void BuildIntermediateDataFromO5M(SourceReader & stream, cache::IntermediateDataWriter & cache, @@ -147,15 +144,14 @@ void ProcessOsmElementsFromO5M(SourceReader & stream, function(buffer), size); }) + , m_dataset([&](uint8_t * buffer, size_t size) { + return m_stream.Read(reinterpret_cast(buffer), size); + }) , m_pos(m_dataset.begin()) { } @@ -176,7 +172,14 @@ bool ProcessorOsmElementsFromO5M::TryRead(OsmElement & element) } }; - auto entity = *m_pos; + element = {}; + + // Be careful, we could call Nodes(), Members(), Tags() from O5MSource::Entity + // only once (!). Because these functions read data from file simultaneously with + // iterating in loop. Furthermore, into Tags() method calls Nodes.Skip() and Members.Skip(), + // thus first call of Nodes (Members) after Tags() will not return any results. + // So don not reorder the "for" loops (!). + auto const entity = *m_pos; element.m_id = entity.id; switch (entity.type) { @@ -211,13 +214,13 @@ bool ProcessorOsmElementsFromO5M::TryRead(OsmElement & element) return true; } -ProcessorXmlElementsFromXml::ProcessorXmlElementsFromXml(SourceReader & stream) +ProcessorOsmElementsFromXml::ProcessorOsmElementsFromXml(SourceReader & stream) : m_xmlSource([&, this](auto * element) { m_queue.emplace(*element); }) , m_parser(stream, m_xmlSource) { } -bool ProcessorXmlElementsFromXml::TryReadFromQueue(OsmElement & element) +bool ProcessorOsmElementsFromXml::TryReadFromQueue(OsmElement & element) { if (m_queue.empty()) return false; @@ -227,16 +230,12 @@ bool ProcessorXmlElementsFromXml::TryReadFromQueue(OsmElement & element) return true; } -bool ProcessorXmlElementsFromXml::TryRead(OsmElement & element) +bool ProcessorOsmElementsFromXml::TryRead(OsmElement & element) { - if (TryReadFromQueue(element)) - return true; - - while (m_parser.Read()) - { + do { if (TryReadFromQueue(element)) return true; - } + } while (m_parser.Read()); return TryReadFromQueue(element); } diff --git a/generator/osm_source.hpp b/generator/osm_source.hpp index 70e012d917..eb51ee4eda 100644 --- a/generator/osm_source.hpp +++ b/generator/osm_source.hpp @@ -1,27 +1,19 @@ #pragma once -#include "generator/feature_generator.hpp" -#include "generator/final_processor_intermediate_mwm.hpp" -#include "generator/processor_interface.hpp" #include "generator/generate_info.hpp" #include "generator/intermediate_data.hpp" #include "generator/osm_o5m_source.hpp" #include "generator/osm_xml_source.hpp" -#include "generator/translator_collection.hpp" #include "generator/translator_interface.hpp" #include "coding/parse_xml.hpp" -#include "base/thread_pool_computational.hpp" -#include "base/thread_safe_queue.hpp" - #include #include #include #include #include #include -#include struct OsmElement; class FeatureParams; @@ -149,10 +141,11 @@ private: osm::O5MSource::Iterator m_pos; }; -class ProcessorXmlElementsFromXml : public ProcessorOsmElementsInterface +class ProcessorOsmElementsFromXml : public ProcessorOsmElementsInterface { public: - explicit ProcessorXmlElementsFromXml(SourceReader & stream); + explicit ProcessorOsmElementsFromXml(SourceReader & stream); + // ProcessorOsmElementsInterface overrides: bool TryRead(OsmElement & element) override; @@ -160,7 +153,7 @@ private: bool TryReadFromQueue(OsmElement & element); XMLSource m_xmlSource; - ParserXMLSequence m_parser; + XMLSequenceParser m_parser; std::queue m_queue; }; } // namespace generator diff --git a/generator/raw_generator.cpp b/generator/raw_generator.cpp index 2d909566e1..ba2447cb80 100644 --- a/generator/raw_generator.cpp +++ b/generator/raw_generator.cpp @@ -165,7 +165,7 @@ bool RawGenerator::GenerateFilteredFeatures() sourceProcessor = std::make_unique(reader); break; case feature::GenerateInfo::OsmSourceType::XML: - sourceProcessor = std::make_unique(reader); + sourceProcessor = std::make_unique(reader); break; } CHECK(sourceProcessor, ());