From b33ae08c7968744e969f9e75dcd0e8bd7a58f400 Mon Sep 17 00:00:00 2001 From: vng Date: Sun, 2 Jan 2011 22:22:07 +0200 Subject: [PATCH] OP_APPEND functionality in FilesContainerW. --- coding/coding_tests/file_container_test.cpp | 27 ++++++ coding/file_container.cpp | 96 ++++++++++++++++++--- coding/file_container.hpp | 16 +++- 3 files changed, 123 insertions(+), 16 deletions(-) diff --git a/coding/coding_tests/file_container_test.cpp b/coding/coding_tests/file_container_test.cpp index 4e2a4d9907..4662b91fc4 100644 --- a/coding/coding_tests/file_container_test.cpp +++ b/coding/coding_tests/file_container_test.cpp @@ -11,6 +11,7 @@ UNIT_TEST(FileContainer_Smoke) string const fName = "file_container.tmp"; size_t const count = 10; + // fill container one by one { FilesContainerW writer(fName); @@ -25,6 +26,7 @@ UNIT_TEST(FileContainer_Smoke) writer.Finish(); } + // read container one by one { FilesContainerR reader(fName); @@ -40,4 +42,29 @@ UNIT_TEST(FileContainer_Smoke) } } } + + // append to container + uint32_t const arrAppend[] = { 666, 777, 888 }; + for (size_t i = 0; i < ARRAY_SIZE(arrAppend); ++i) + { + { + FilesContainerW writer(fName, FileWriter::OP_APPEND); + + FileWriter w = writer.GetWriter(utils::to_string(arrAppend[i])); + WriteVarUint(w, arrAppend[i]); + + writer.Finish(); + } + + // read appended + { + FilesContainerR reader(fName); + + FileReader r = reader.GetReader(utils::to_string(arrAppend[i])); + ReaderSource src(r); + + uint32_t const test = ReadVarUint(src); + CHECK_EQUAL(arrAppend[i], test, ()); + } + } } diff --git a/coding/file_container.cpp b/coding/file_container.cpp index 9aeb9ce3af..1c431e820e 100644 --- a/coding/file_container.cpp +++ b/coding/file_container.cpp @@ -5,12 +5,15 @@ #include "write_to_sink.hpp" -FilesContainerR::FilesContainerR(string const & fName) -: m_source(fName) -{ - uint64_t offset = ReadPrimitiveFromPos(m_source, 0); +///////////////////////////////////////////////////////////////////////////// +// FilesContainerBase +///////////////////////////////////////////////////////////////////////////// - ReaderSource src(m_source); +void FilesContainerBase::ReadInfo(FileReader & reader) +{ + uint64_t offset = ReadPrimitiveFromPos(reader, 0); + + ReaderSource src(reader); src.Skip(offset); uint32_t const count = ReadVarUint(src); @@ -27,7 +30,19 @@ FilesContainerR::FilesContainerR(string const & fName) } } -FileReader FilesContainerR::GetReader(Tag const & tag) +///////////////////////////////////////////////////////////////////////////// +// FilesContainerR +///////////////////////////////////////////////////////////////////////////// + +FilesContainerR::FilesContainerR(string const & fName, + uint32_t logPageSize, + uint32_t logPageCount) +: m_source(fName, logPageSize, logPageCount) +{ + ReadInfo(m_source); +} + +FileReader FilesContainerR::GetReader(Tag const & tag) const { info_cont_t::const_iterator i = lower_bound(m_info.begin(), m_info.end(), tag, less_info()); @@ -38,12 +53,27 @@ FileReader FilesContainerR::GetReader(Tag const & tag) MYTHROW(Reader::OpenException, (tag)); } -FilesContainerW::FilesContainerW(string const & fName) +///////////////////////////////////////////////////////////////////////////// +// FilesContainerW +///////////////////////////////////////////////////////////////////////////// + +FilesContainerW::FilesContainerW(string const & fName, FileWriter::Op op) : m_name(fName) { - FileWriter writer(fName); - uint64_t skip = 0; - writer.Write(&skip, sizeof(skip)); + if (op == FileWriter::OP_APPEND) + { + FileReader reader(fName); + ReadInfo(reader); + m_needRewrite = true; + } + + if (m_info.empty()) + { + FileWriter writer(fName); + uint64_t skip = 0; + writer.Write(&skip, sizeof(skip)); + m_needRewrite = false; + } } uint64_t FilesContainerW::SaveCurrentSize() @@ -56,11 +86,51 @@ uint64_t FilesContainerW::SaveCurrentSize() FileWriter FilesContainerW::GetWriter(Tag const & tag) { - uint64_t const curr = SaveCurrentSize(); + if (m_needRewrite) + { + m_needRewrite = false; + ASSERT ( !m_info.empty(), () ); - m_info.push_back(Info(tag, curr)); + uint64_t const curr = m_info.back().m_offset + m_info.back().m_size; + m_info.push_back(Info(tag, curr)); - return FileWriter(m_name, FileWriter::OP_APPEND); + FileWriter writer(m_name, FileWriter::OP_WRITE_EXISTING); + writer.Seek(curr); + return writer; + } + else + { + uint64_t const curr = SaveCurrentSize(); + m_info.push_back(Info(tag, curr)); + return FileWriter(m_name, FileWriter::OP_APPEND); + } +} + +void FilesContainerW::Append(string const & fName, Tag const & tag) +{ + uint64_t const bufferSize = 4*1024; + char buffer[bufferSize]; + + FileReader reader(fName); + ReaderSource src(reader); + FileWriter writer = GetWriter(tag); + + uint64_t size = reader.Size(); + while (size > 0) + { + size_t const curr = static_cast(min(bufferSize, size)); + + src.Read(&buffer[0], curr); + writer.Write(&buffer[0], curr); + + size -= curr; + } +} + +void FilesContainerW::Append(vector const & buffer, Tag const & tag) +{ + if (!buffer.empty()) + GetWriter(tag).Write(&buffer[0], buffer.size()); } void FilesContainerW::Finish() diff --git a/coding/file_container.hpp b/coding/file_container.hpp index 01da1e670c..fca57f4459 100644 --- a/coding/file_container.hpp +++ b/coding/file_container.hpp @@ -39,6 +39,8 @@ protected: typedef vector info_cont_t; info_cont_t m_info; + + void ReadInfo(FileReader & reader); }; class FilesContainerR : public FilesContainerBase @@ -48,9 +50,11 @@ class FilesContainerR : public FilesContainerBase FileReader m_source; public: - FilesContainerR(string const & fName); + explicit FilesContainerR(string const & fName, + uint32_t logPageSize = 10, + uint32_t logPageCount = 10); - FileReader GetReader(Tag const & tag); + FileReader GetReader(Tag const & tag) const; }; class FilesContainerW : public FilesContainerBase @@ -61,10 +65,16 @@ class FilesContainerW : public FilesContainerBase uint64_t SaveCurrentSize(); + bool m_needRewrite; + public: - FilesContainerW(string const & fName); + FilesContainerW(string const & fName, + FileWriter::Op op = FileWriter::OP_WRITE_TRUNCATE); FileWriter GetWriter(Tag const & tag); + void Append(string const & fName, Tag const & tag); + void Append(vector const & buffer, Tag const & tag); + void Finish(); };