From 84b4d78443907ad1afba64dec8027993598797ed Mon Sep 17 00:00:00 2001 From: vng Date: Mon, 5 Sep 2011 13:12:19 +0300 Subject: [PATCH] Add override support in FilesContaiterW. Make it's interface more simple. --- coding/coding_tests/file_container_test.cpp | 61 +++++++----- coding/file_container.cpp | 97 +++++++++++++------- coding/file_container.hpp | 18 ++-- generator/feature_sorter.cpp | 6 +- generator/generator.pro | 2 - generator/generator_tool/generator_tool.cpp | 4 - generator/mwm_rect_updater.cpp | 61 ------------ generator/mwm_rect_updater.hpp | 5 - indexer/index_builder.cpp | 2 +- indexer/indexer_tests/index_builder_test.cpp | 4 +- indexer/search_index_builder.cpp | 8 +- 11 files changed, 120 insertions(+), 148 deletions(-) delete mode 100644 generator/mwm_rect_updater.cpp delete mode 100644 generator/mwm_rect_updater.hpp diff --git a/coding/coding_tests/file_container_test.cpp b/coding/coding_tests/file_container_test.cpp index b6285e7d63..80ff239a74 100644 --- a/coding/coding_tests/file_container_test.cpp +++ b/coding/coding_tests/file_container_test.cpp @@ -47,7 +47,7 @@ UNIT_TEST(FilesContainer_Smoke) for (size_t i = 0; i < ARRAY_SIZE(arrAppend); ++i) { { - FilesContainerW writer(fName, FileWriter::OP_APPEND); + FilesContainerW writer(fName, FileWriter::OP_WRITE_EXISTING); FileWriter w = writer.GetWriter(strings::to_string(arrAppend[i])); WriteVarUint(w, arrAppend[i]); @@ -111,7 +111,7 @@ UNIT_TEST(FilesContainer_Shared) CheckInvariant(reader, "2", test64); - FilesContainerW writer(fName, FileWriter::OP_APPEND); + FilesContainerW writer(fName, FileWriter::OP_WRITE_EXISTING); FileWriter w = writer.GetWriter("3"); ReaderSource src(r1); @@ -132,6 +132,29 @@ UNIT_TEST(FilesContainer_Shared) FileWriter::DeleteFileX(fName); } +namespace +{ + void CheckContainer(string const & fName, + char const * key[], char const * value[], size_t count) + { + FilesContainerR reader(fName); + + for (size_t i = 0; i < count; ++i) + { + FilesContainerR::ReaderT r = reader.GetReader(key[i]); + + size_t const szBuffer = 100; + size_t const szS = strlen(value[i]); + + char s[szBuffer] = { 0 }; + ASSERT_LESS ( szS, szBuffer, () ); + r.Read(0, s, szS); + + TEST(strcmp(value[i], s) == 0, (s)); + } + } +} + UNIT_TEST(FilesContainer_RewriteExisting) { string const fName = "file_container.tmp"; @@ -139,7 +162,6 @@ UNIT_TEST(FilesContainer_RewriteExisting) char const * key[] = { "3", "2", "1" }; char const * value[] = { "prolog", "data", "epilog" }; - char const * buffer = "xxxx"; // fill container { @@ -153,33 +175,30 @@ UNIT_TEST(FilesContainer_RewriteExisting) } // re-write middle file in container + char const * buffer1 = "xxxxxxx"; { FilesContainerW writer(fName, FileWriter::OP_WRITE_EXISTING); - FileWriter w = writer.GetExistingWriter(key[1]); + FileWriter w = writer.GetWriter(key[1]); - w.Write(buffer, strlen(buffer)); + w.Write(buffer1, strlen(buffer1)); } // check container files + char const * value1[] = { value[0], buffer1, value[2] }; + CheckContainer(fName, key, value1, 3); + + // re-write end file in container + char const * buffer2 = "yyyyyyyyyyyyyy"; { - FilesContainerR reader(fName); + FilesContainerW writer(fName, FileWriter::OP_WRITE_EXISTING); + FileWriter w = writer.GetWriter(key[2]); - for (size_t i = 0; i < ARRAY_SIZE(key); ++i) - { - FilesContainerR::ReaderT r = reader.GetReader(key[i]); - char s[10] = { 0 }; - r.Read(0, s, strlen(value[i])); - - if (i == 1) - { - TEST(strcmp(buffer, s) == 0, (s)); - } - else - { - TEST(strcmp(value[i], s) == 0, (s)); - } - } + w.Write(buffer2, strlen(buffer2)); } + // check container files + char const * value2[] = { value[0], buffer1, buffer2 }; + CheckContainer(fName, key, value2, 3); + FileWriter::DeleteFileX(fName); } diff --git a/coding/file_container.cpp b/coding/file_container.cpp index cfe3d957fb..7e5cacc994 100644 --- a/coding/file_container.cpp +++ b/coding/file_container.cpp @@ -3,6 +3,7 @@ #include "file_container.hpp" #include "varint.hpp" #include "write_to_sink.hpp" +#include "internal/file_data.hpp" ///////////////////////////////////////////////////////////////////////////// @@ -75,34 +76,45 @@ bool FilesContainerR::IsReaderExist(Tag const & tag) const FilesContainerW::FilesContainerW(string const & fName, FileWriter::Op op) : m_name(fName), m_bFinished(false) { + Open(op); +} + +void FilesContainerW::Open(FileWriter::Op op) +{ + m_bNeedRewrite = true; + switch (op) { - case FileWriter::OP_WRITE_TRUNCATE: // default usage + case FileWriter::OP_WRITE_TRUNCATE: break; - case FileWriter::OP_APPEND: - m_bNeedRewrite = true; // need to override service info after appending - // 'break' doesn't present! - case FileWriter::OP_WRITE_EXISTING: { // read an existing service info - FileReader reader(fName); + FileReader reader(m_name); ReadInfo(reader); // Important: in append mode we should sort info-vector by offsets sort(m_info.begin(), m_info.end(), LessOffset()); + break; } + + default: + ASSERT ( false, ("Unsupperted options") ); + break; } if (m_info.empty()) - { - // leave space for offset to service info - FileWriter writer(fName); - uint64_t skip = 0; - writer.Write(&skip, sizeof(skip)); - m_bNeedRewrite = false; - } + StartNew(); +} + +void FilesContainerW::StartNew() +{ + // leave space for offset to service info + FileWriter writer(m_name); + uint64_t skip = 0; + writer.Write(&skip, sizeof(skip)); + m_bNeedRewrite = false; } FilesContainerW::~FilesContainerW() @@ -123,6 +135,40 @@ uint64_t FilesContainerW::SaveCurrentSize() FileWriter FilesContainerW::GetWriter(Tag const & tag) { ASSERT(!m_bFinished, ()); + + InfoContainer::const_iterator it = find_if(m_info.begin(), m_info.end(), EqualTag(tag)); + if (it != m_info.end()) + { + if (it+1 == m_info.end()) + { + m_info.pop_back(); + + if (m_info.empty()) + StartNew(); + else + m_bNeedRewrite = true; + } + else + { + { + FilesContainerR contR(m_name); + FilesContainerW contW(m_name + ".tmp"); + + for (size_t i = 0; i < m_info.size(); ++i) + { + if (m_info[i].m_tag != it->m_tag) + contW.Write(contR.GetReader(m_info[i].m_tag), m_info[i].m_tag); + } + } + + my::DeleteFileX(m_name); + if (!my::RenameFileX(m_name + ".tmp", m_name)) + MYTHROW(RootException, ("Can't rename file", m_name, "Sharing violation or disk error!")); + + Open(FileWriter::OP_WRITE_EXISTING); + } + } + if (m_bNeedRewrite) { m_bNeedRewrite = false; @@ -143,28 +189,13 @@ FileWriter FilesContainerW::GetWriter(Tag const & tag) } } -FileWriter FilesContainerW::GetExistingWriter(Tag const & tag) +void FilesContainerW::Write(string const & fPath, Tag const & tag) { - InfoContainer::const_iterator i = find_if(m_info.begin(), m_info.end(), EqualTag(tag)); - - if (i != m_info.end()) - { - FileWriter writer(m_name, FileWriter::OP_WRITE_EXISTING); - writer.Seek(i->m_offset); - return writer; - } - else - MYTHROW(Writer::OpenException, (tag)); + Write(new FileReader(fPath), tag); } -void FilesContainerW::Append(string const & fPath, Tag const & tag) +void FilesContainerW::Write(ModelReaderPtr reader, Tag const & tag) { - Append(new FileReader(fPath), tag); -} - -void FilesContainerW::Append(ModelReaderPtr reader, Tag const & tag) -{ - ASSERT(!m_bFinished, ()); uint64_t const bufferSize = 4*1024; char buffer[bufferSize]; @@ -183,9 +214,8 @@ void FilesContainerW::Append(ModelReaderPtr reader, Tag const & tag) } } -void FilesContainerW::Append(vector const & buffer, Tag const & tag) +void FilesContainerW::Write(vector const & buffer, Tag const & tag) { - ASSERT(!m_bFinished, ()); if (!buffer.empty()) GetWriter(tag).Write(&buffer[0], buffer.size()); } @@ -193,6 +223,7 @@ void FilesContainerW::Append(vector const & buffer, Tag const & tag) void FilesContainerW::Finish() { ASSERT(!m_bFinished, ()); + { uint64_t const curr = SaveCurrentSize(); FileWriter writer(m_name, FileWriter::OP_WRITE_EXISTING); diff --git a/coding/file_container.hpp b/coding/file_container.hpp index 3b877608bf..45ccad51ae 100644 --- a/coding/file_container.hpp +++ b/coding/file_container.hpp @@ -111,24 +111,18 @@ public: FileWriter GetWriter(Tag const & tag); - /// @todo Subclass from FileWriter to check write bounds (avoid overrun). - /// Return this object with additional check. - /// @precondition Container should be constructed with OP_WRITE_EXISTING. - FileWriter GetExistingWriter(Tag const & tag); - - /// @name Append to existing container. - /// @precondition Container should be constructed with OP_APPEND. - //@{ - void Append(string const & fPath, Tag const & tag); - void Append(ModelReaderPtr reader, Tag const & tag); - void Append(vector const & buffer, Tag const & tag); - //@} + void Write(string const & fPath, Tag const & tag); + void Write(ModelReaderPtr reader, Tag const & tag); + void Write(vector const & buffer, Tag const & tag); void Finish(); private: uint64_t SaveCurrentSize(); + void Open(FileWriter::Op op); + void StartNew(); + string m_name; bool m_bNeedRewrite; bool m_bFinished; diff --git a/generator/feature_sorter.cpp b/generator/feature_sorter.cpp index d9f9b4f4ee..ada213972a 100644 --- a/generator/feature_sorter.cpp +++ b/generator/feature_sorter.cpp @@ -119,7 +119,7 @@ namespace feature // assume like we close files m_datFile.Flush(); - m_writer.Append(m_datFile.GetName(), DATA_FILE_TAG); + m_writer.Write(m_datFile.GetName(), DATA_FILE_TAG); for (size_t i = 0; i < m_header.GetScalesCount(); ++i) { @@ -136,8 +136,8 @@ namespace feature string trgPostfix = TRIANGLE_FILE_TAG; trgPostfix += postfix; - m_writer.Append(geomFile, geoPostfix); - m_writer.Append(trgFile, trgPostfix); + m_writer.Write(geomFile, geoPostfix); + m_writer.Write(trgFile, trgPostfix); FileWriter::DeleteFileX(geomFile); FileWriter::DeleteFileX(trgFile); diff --git a/generator/generator.pro b/generator/generator.pro index 4b3ce5f46d..424232ed2e 100644 --- a/generator/generator.pro +++ b/generator/generator.pro @@ -24,7 +24,6 @@ SOURCES += \ borders_generator.cpp \ osm_xml_parser.cpp \ borders_loader.cpp \ - mwm_rect_updater.cpp \ dumper.cpp \ unpack_mwm.cpp \ feature_builder.cpp \ @@ -49,7 +48,6 @@ HEADERS += \ borders_generator.hpp \ osm_xml_parser.hpp \ borders_loader.hpp \ - mwm_rect_updater.hpp \ feature_emitter_iface.hpp \ dumper.hpp \ generate_info.hpp \ diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index 02c08774b3..5afabe2ab7 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -5,7 +5,6 @@ #include "../borders_generator.hpp" #include "../classif_routine.hpp" #include "../dumper.hpp" -#include "../mwm_rect_updater.hpp" #include "../statistics.hpp" #include "../unpack_mwm.hpp" #include "../generate_info.hpp" @@ -194,9 +193,6 @@ int main(int argc, char ** argv) } } - //if (FLAGS_split_by_polygons) - // UpdateMWMRectsFromBoundaries(path); - // Create http update list for countries and corresponding files if (FLAGS_generate_update) { diff --git a/generator/mwm_rect_updater.cpp b/generator/mwm_rect_updater.cpp deleted file mode 100644 index 41ac62884f..0000000000 --- a/generator/mwm_rect_updater.cpp +++ /dev/null @@ -1,61 +0,0 @@ -#include "mwm_rect_updater.hpp" -#include "borders_loader.hpp" - -#include "../indexer/data_header.hpp" - -#include "../coding/file_container.hpp" - -#include "../base/logging.hpp" - -#include "../defines.hpp" - - -namespace -{ - class DoUpdateRect - { - string const & m_basePath; - public: - DoUpdateRect(string const & path) : m_basePath(path) {} - - void operator() (m2::RectD const & r, borders::CountryPolygons const & c) - { - string name = m_basePath + c.m_name + DATA_FILE_EXTENSION; - - feature::DataHeader h; - - try - { - { - FilesContainerR contR(name); - h.Load(contR.GetReader(HEADER_FILE_TAG)); - } - - h.SetBounds(r); - - { - FilesContainerW contW(name, FileWriter::OP_WRITE_EXISTING); - FileWriter w = contW.GetExistingWriter(HEADER_FILE_TAG); - h.Save(w); - } - } - catch (FileReader::OpenException const &) - { - // country from countries list not found - } - catch (FileWriter::OpenException const &) - { - // something real bad happend - LOG(LERROR, ("Can't open existing file for writing")); - } - } - }; -} - -void UpdateMWMRectsFromBoundaries(string const & dataPath) -{ - borders::CountriesContainerT countries; - borders::LoadCountriesList(dataPath, countries); - - countries.ForEachWithRect(DoUpdateRect(dataPath)); -} diff --git a/generator/mwm_rect_updater.hpp b/generator/mwm_rect_updater.hpp deleted file mode 100644 index c8417feabf..0000000000 --- a/generator/mwm_rect_updater.hpp +++ /dev/null @@ -1,5 +0,0 @@ -#pragma once - -#include "../std/string.hpp" - -void UpdateMWMRectsFromBoundaries(string const & dataPath); diff --git a/indexer/index_builder.cpp b/indexer/index_builder.cpp index 4bb9966adf..96b8190cae 100644 --- a/indexer/index_builder.cpp +++ b/indexer/index_builder.cpp @@ -19,7 +19,7 @@ namespace indexer FeaturesVector featuresVector(readCont, header); - FilesContainerW writeCont(datFile, FileWriter::OP_APPEND); + FilesContainerW writeCont(datFile, FileWriter::OP_WRITE_EXISTING); FileWriter writer = writeCont.GetWriter(INDEX_FILE_TAG); BuildIndex(header.GetLastScale() + 1, featuresVector, writer, tmpFile); diff --git a/indexer/indexer_tests/index_builder_test.cpp b/indexer/indexer_tests/index_builder_test.cpp index 0ee762b6cc..07f8c037e3 100644 --- a/indexer/indexer_tests/index_builder_test.cpp +++ b/indexer/indexer_tests/index_builder_test.cpp @@ -50,9 +50,9 @@ UNIT_TEST(BuildIndexTest) for (size_t i = 0; i < tags.size(); ++i) { if (tags[i] != INDEX_FILE_TAG) - containerWriter.Append(originalContainer.GetReader(tags[i]), tags[i]); + containerWriter.Write(originalContainer.GetReader(tags[i]), tags[i]); } - containerWriter.Append(serialIndex, INDEX_FILE_TAG); + containerWriter.Write(serialIndex, INDEX_FILE_TAG); } { diff --git a/indexer/search_index_builder.cpp b/indexer/search_index_builder.cpp index ca43d36c35..346b65ffc0 100644 --- a/indexer/search_index_builder.cpp +++ b/indexer/search_index_builder.cpp @@ -113,16 +113,16 @@ bool indexer::BuildSearchIndexFromDatFile(string const & datFile) FeaturesVector featuresVector(readCont, header); - FilesContainerW writeCont(datFile, FileWriter::OP_APPEND); + FilesContainerW writeCont(datFile, FileWriter::OP_WRITE_EXISTING); - vector serialTrie; + vector serialTrie; { - MemWriter > writer(serialTrie); + MemWriter > writer(serialTrie); BuildSearchIndex(featuresVector, writer); reverse(serialTrie.begin(), serialTrie.end()); } - writeCont.GetWriter(SEARCH_INDEX_FILE_TAG).Write(&serialTrie[0], serialTrie.size()); + writeCont.Write(serialTrie, SEARCH_INDEX_FILE_TAG); } catch (Reader::Exception const & e) {