From 985d9afb431363a7fa211a0178152ced6ab15f46 Mon Sep 17 00:00:00 2001 From: cyber-toad Date: Fri, 9 Feb 2024 01:00:54 +0100 Subject: [PATCH] [bookmarks] Add code to export multiple files Signed-off-by: cyber-toad --- coding/coding_tests/zip_creator_test.cpp | 42 +++++++- coding/zip_creator.cpp | 116 ++++++++--------------- coding/zip_creator.hpp | 7 +- data/gpx_test_data/Üφήが1.gpx | 16 ++++ data/gpx_test_data/Üφήが2.gpx | 16 ++++ data/kml_test_data/kmz_index.kml | 10 ++ map/bookmark_manager.cpp | 101 ++++++++++++++++++-- map/map_tests/bookmarks_test.cpp | 83 +++++++++++++++- 8 files changed, 300 insertions(+), 91 deletions(-) create mode 100644 data/gpx_test_data/Üφήが1.gpx create mode 100644 data/gpx_test_data/Üφήが2.gpx create mode 100644 data/kml_test_data/kmz_index.kml diff --git a/coding/coding_tests/zip_creator_test.cpp b/coding/coding_tests/zip_creator_test.cpp index 5814e5fc91..b60d3883bb 100644 --- a/coding/coding_tests/zip_creator_test.cpp +++ b/coding/coding_tests/zip_creator_test.cpp @@ -15,7 +15,7 @@ namespace { void CreateAndTestZip(std::string const & filePath, std::string const & zipPath) { - TEST(CreateZipFromPathDeflatedAndDefaultCompression(filePath, zipPath), ()); + TEST(CreateZipFromFiles({filePath}, zipPath, CompressionLevel::DefaultCompression), ()); ZipFileReader::FileList files; ZipFileReader::FilesList(zipPath, files); @@ -50,6 +50,26 @@ void CreateAndTestZip(std::vector const & files, std::string const TEST(base::DeleteFileX(zipPath), ()); } +void CreateAndTestZipWithFolder(std::vector const & files, std::vector const & filesInArchive, std::string const & zipPath, + CompressionLevel compression) +{ + TEST(CreateZipFromFiles(files, filesInArchive, zipPath, compression), ()); + + ZipFileReader::FileList fileList; + ZipFileReader::FilesList(zipPath, fileList); + std::string const unzippedFile = "unzipped.tmp"; + for (size_t i = 0; i < files.size(); ++i) + { + TEST_EQUAL(fileList[i].second, FileReader(files[i]).Size(), ()); + + ZipFileReader::UnzipFile(zipPath, fileList[i].first, unzippedFile); + + TEST(base::IsEqualFiles(files[i], unzippedFile), ()); + TEST(base::DeleteFileX(unzippedFile), ()); + } + TEST(base::DeleteFileX(zipPath), ()); +} + std::vector GetCompressionLevels() { return {CompressionLevel::DefaultCompression, CompressionLevel::BestCompression, @@ -100,6 +120,26 @@ UNIT_TEST(CreateZip_MultipleFiles) CreateAndTestZip(fileData, "testzip.zip", compression); } +UNIT_TEST(CreateZip_MultipleFilesWithFolders) +{ + std::vector const fileData{"testf1", "testfile2", "testfile3_longname.txt.xml.csv"}; + std::vector const fileInArchiveData{"testf1", "f2/testfile2", "f3/testfile3_longname.txt.xml.csv"}; + SCOPE_GUARD(deleteFileGuard, [&fileData]() + { + for (auto const & file : fileData) + TEST(base::DeleteFileX(file), ()); + }); + + for (auto const & name : fileData) + { + FileWriter f(name); + f.Write(name.c_str(), name.size()); + } + + for (auto compression : GetCompressionLevels()) + CreateAndTestZipWithFolder(fileData, fileInArchiveData, "testzip.zip", compression); +} + UNIT_TEST(CreateZip_MultipleFilesSingleEmpty) { std::vector const fileData{"singleEmptyfile.txt"}; diff --git a/coding/zip_creator.cpp b/coding/zip_creator.cpp index 077082ac6e..11f576832b 100644 --- a/coding/zip_creator.cpp +++ b/coding/zip_creator.cpp @@ -2,16 +2,12 @@ #include "base/string_utils.hpp" -#include "coding/constants.hpp" #include "coding/internal/file_data.hpp" -#include "coding/reader.hpp" #include "base/file_name_utils.hpp" #include "base/logging.hpp" #include "base/scope_guard.hpp" -#include "3party/minizip/minizip.hpp" - #include #include #include @@ -39,20 +35,6 @@ public: zip::File Handle() const { return m_zipFileHandle; } }; -void CreateTMZip(zip::DateTime & res) -{ - time_t rawtime; - struct tm * timeinfo; - time(&rawtime); - timeinfo = localtime(&rawtime); - res.tm_sec = timeinfo->tm_sec; - res.tm_min = timeinfo->tm_min; - res.tm_hour = timeinfo->tm_hour; - res.tm_mday = timeinfo->tm_mday; - res.tm_mon = timeinfo->tm_mon; - res.tm_year = timeinfo->tm_year; -} - int GetCompressionLevel(CompressionLevel compression) { switch (compression) @@ -67,61 +49,23 @@ int GetCompressionLevel(CompressionLevel compression) } } // namespace -bool CreateZipFromPathDeflatedAndDefaultCompression(std::string const & filePath, - std::string const & zipFilePath) +void FillZipLocalDateTime(zip::DateTime & res) { - // Open zip file for writing. - SCOPE_GUARD(outFileGuard, [&zipFilePath]() { base::DeleteFileX(zipFilePath); }); - - ZipHandle zip(zipFilePath); - if (!zip.Handle()) - return false; - - zip::FileInfo zipInfo = {}; - CreateTMZip(zipInfo.tmz_date); - - std::string fileName = base::FileNameFromFullPath(filePath); - if (!strings::IsASCIIString(fileName)) - fileName = "OrganicMaps.kml"; - - if (zip::Code::Ok != zip::OpenNewFileInZip(zip.Handle(), fileName, zipInfo, "ZIP from OMaps", - Z_DEFLATED, Z_DEFAULT_COMPRESSION)) - { - return false; - } - - // Write source file into zip file. - try - { - base::FileData file(filePath, base::FileData::OP_READ); - uint64_t const fileSize = file.Size(); - - uint64_t currSize = 0; - std::array buffer; - while (currSize < fileSize) - { - auto const toRead = std::min(buffer.size(), static_cast(fileSize - currSize)); - file.Read(currSize, buffer.data(), toRead); - - if (zip::Code::Ok != zip::WriteInFileInZip(zip.Handle(), buffer, toRead)) - return false; - - currSize += toRead; - } - } - catch (Reader::Exception const & ex) - { - LOG(LERROR, ("Error reading file:", filePath, ex.Msg())); - return false; - } - - outFileGuard.release(); - return true; + auto const now = std::time(nullptr); + // Files inside .zip are using local date time format. + auto const * local = std::localtime(&now); + res.tm_sec = local->tm_sec; + res.tm_min = local->tm_min; + res.tm_hour = local->tm_hour; + res.tm_mday = local->tm_mday; + res.tm_mon = local->tm_mon; + res.tm_year = local->tm_year; } -bool CreateZipFromFiles(std::vector const & files, std::string const & zipFilePath, +bool CreateZipFromFiles(std::vector const & files, std::vector const & filesInArchive, std::string const & zipFilePath, CompressionLevel compression) { + ASSERT_EQUAL(files.size(), filesInArchive.size(), ("List of file names in archive doesn't match list of files")); SCOPE_GUARD(outFileGuard, [&zipFilePath]() { base::DeleteFileX(zipFilePath); }); ZipHandle zip(zipFilePath); @@ -129,34 +73,41 @@ bool CreateZipFromFiles(std::vector const & files, std::string cons return false; auto const compressionLevel = GetCompressionLevel(compression); - zip::FileInfo const fileInfo = {}; try { - for (auto const & filePath : files) + int suffix = 0; + for (size_t i = 0; i < files.size(); i++) { - if (zip::Code::Ok != zip::OpenNewFileInZip(zip.Handle(), filePath, fileInfo, "", - Z_DEFLATED, compressionLevel)) + auto const & filePath = files.at(i); + std::string fileInArchive = filesInArchive.at(i); + if (!strings::IsASCIIString(fileInArchive)) { - return false; + if (suffix == 0) + fileInArchive = "OrganicMaps.kml"; + else + fileInArchive = "OrganicMaps_" + std::to_string(suffix) + ".kml"; + ++suffix; } + zip::FileInfo fileInfo = {}; + FillZipLocalDateTime(fileInfo.tmz_date); + if (zip::Code::Ok != zip::OpenNewFileInZip(zip.Handle(), fileInArchive, fileInfo, "", Z_DEFLATED, compressionLevel)) + return false; base::FileData file(filePath, base::FileData::OP_READ); uint64_t const fileSize = file.Size(); uint64_t writtenSize = 0; - zip::Buffer buffer; + std::array buffer; while (writtenSize < fileSize) { - auto const filePartSize = - std::min(buffer.size(), static_cast(fileSize - writtenSize)); + auto const filePartSize = std::min(buffer.size(), static_cast(fileSize - writtenSize)); file.Read(writtenSize, buffer.data(), filePartSize); - if (zip::Code::Ok != zip::WriteInFileInZip(zip.Handle(), buffer, filePartSize)) return false; - writtenSize += filePartSize; } + zipCloseFileInZip(zip.Handle()); } } catch (std::exception const & e) @@ -168,3 +119,12 @@ bool CreateZipFromFiles(std::vector const & files, std::string cons outFileGuard.release(); return true; } + +bool CreateZipFromFiles(std::vector const & files, std::string const & zipFilePath, + CompressionLevel compression) +{ + std::vector filesInArchive; + for (auto const & file : files) + filesInArchive.push_back(base::FileNameFromFullPath(file)); + return CreateZipFromFiles(files, filesInArchive, zipFilePath, compression); +} diff --git a/coding/zip_creator.hpp b/coding/zip_creator.hpp index 6d76428fa3..14bc2d7bc1 100644 --- a/coding/zip_creator.hpp +++ b/coding/zip_creator.hpp @@ -2,6 +2,7 @@ #include #include +#include "3party/minizip/minizip.hpp" enum class CompressionLevel { @@ -12,8 +13,10 @@ enum class CompressionLevel Count }; -bool CreateZipFromPathDeflatedAndDefaultCompression(std::string const & filePath, - std::string const & zipFilePath); +void FillZipLocalDateTime(zip::DateTime & res); + +bool CreateZipFromFiles(std::vector const & files, std::vector const & filesInArchive, std::string const & zipFilePath, + CompressionLevel compression = CompressionLevel::DefaultCompression); bool CreateZipFromFiles(std::vector const & files, std::string const & zipFilePath, CompressionLevel compression = CompressionLevel::DefaultCompression); diff --git a/data/gpx_test_data/Üφήが1.gpx b/data/gpx_test_data/Üφήが1.gpx new file mode 100644 index 0000000000..084fbe2100 --- /dev/null +++ b/data/gpx_test_data/Üφήが1.gpx @@ -0,0 +1,16 @@ + + + + Welcome to my route + + +Some Üφήが1 route + +184.4 +Point 1 + + +186.9 +Point 2 + + \ No newline at end of file diff --git a/data/gpx_test_data/Üφήが2.gpx b/data/gpx_test_data/Üφήが2.gpx new file mode 100644 index 0000000000..2537a73b06 --- /dev/null +++ b/data/gpx_test_data/Üφήが2.gpx @@ -0,0 +1,16 @@ + + + + Welcome to my route + + +Some Üφήが2 route + +184.4 +Point 1 + + +186.9 +Point 2 + + \ No newline at end of file diff --git a/data/kml_test_data/kmz_index.kml b/data/kml_test_data/kmz_index.kml new file mode 100644 index 0000000000..a6e7fbdf30 --- /dev/null +++ b/data/kml_test_data/kmz_index.kml @@ -0,0 +1,10 @@ + + + +Organic Maps Bookmarks and Tracks +Some random routefiles/Some random route.kml +newfiles/new.kml +OrganicMaps_1files/OrganicMaps_1.kml +OrganicMaps_2files/OrganicMaps_2.kml + + \ No newline at end of file diff --git a/map/bookmark_manager.cpp b/map/bookmark_manager.cpp index 8cbccd4d07..b8098eec30 100644 --- a/map/bookmark_manager.cpp +++ b/map/bookmark_manager.cpp @@ -31,7 +31,6 @@ #include #include - namespace { std::string const kLastEditedBookmarkCategory = "LastBookmarkCategory"; @@ -74,12 +73,17 @@ public: m2::PointD m_globalCenter; }; -BookmarkManager::SharingResult GetFileForSharing(BookmarkManager::KMLDataCollectionPtr collection) +std::string GetFileNameForExport(BookmarkManager::KMLDataCollectionPtr::element_type::value_type const & kmlToShare) { - auto const & kmlToShare = collection->front(); std::string fileName = RemoveInvalidSymbols(kml::GetDefaultStr(kmlToShare.second->m_categoryData.m_name)); if (fileName.empty()) fileName = base::GetNameFromFullPathWithoutExt(kmlToShare.first); + return fileName; +} + +BookmarkManager::SharingResult ExportSingleFile(BookmarkManager::KMLDataCollectionPtr::element_type::value_type const & kmlToShare) +{ + std::string fileName = GetFileNameForExport(kmlToShare); auto const filePath = base::JoinPath(GetPlatform().TmpDir(), fileName + std::string{kKmlExtension}); SCOPE_GUARD(fileGuard, std::bind(&base::DeleteFileX, filePath)); @@ -90,12 +94,91 @@ BookmarkManager::SharingResult GetFileForSharing(BookmarkManager::KMLDataCollect return {{categoryId}, BookmarkManager::SharingResult::Code::FileError, "Bookmarks file does not exist."}; auto tmpFilePath = base::JoinPath(GetPlatform().TmpDir(), fileName + std::string{kKmzExtension}); - if (!CreateZipFromPathDeflatedAndDefaultCompression(filePath, tmpFilePath)) + + if (!CreateZipFromFiles({filePath}, tmpFilePath)) return {{categoryId}, BookmarkManager::SharingResult::Code::ArchiveError, "Could not create archive."}; return {{categoryId}, std::move(tmpFilePath)}; } +std::string BuildIndexFile(std::vector const & filesForIndex) +{ + std::string const filePath = base::JoinPath(GetPlatform().TmpDir(), "doc.kml"); + FileWriter fileWriter(filePath); + std::string content = "\n" + "\n" + "\n" + "Organic Maps Bookmarks and Tracks\n"; + for (auto const & fileName : filesForIndex) + { + content.append(""); + content.append(base::GetNameFromFullPathWithoutExt(fileName)); + content.append(""); + content.append(fileName); + content.append("\n"); + } + content.append("\n" + ""); + fileWriter.Write(content.c_str(), content.length()); + return filePath; +} + +BookmarkManager::SharingResult ExportMultipleFiles(BookmarkManager::KMLDataCollectionPtr collection) +{ + auto const kmzFileName = "OrganicMapsBackup_" + std::to_string(base::GenerateYYMMDD(time(nullptr))); + auto kmzFilePath = base::JoinPath(GetPlatform().TmpDir(), kmzFileName + std::string{kKmzExtension}); + auto const filesDir = "files/"; + kml::GroupIdCollection categoriesIds; + for (auto const & kmlToExport : *collection) + categoriesIds.push_back(kmlToExport.second->m_categoryData.m_id); + std::vector filesInArchive; + std::vector pathsForArchive; + + SCOPE_GUARD(deleteFileGuard, [&pathsForArchive]() + { + for (auto const & path : pathsForArchive) + base::DeleteFileX(path); + }); + + int suffix = 1; + for (auto const & kmlToExport : *collection) + { + std::string fileName = base::FilenameWithoutExt(GetFileNameForExport(kmlToExport)); + if (!strings::IsASCIIString(fileName)) + fileName = "OrganicMaps_" + std::to_string(suffix++); + auto const kmlPath = base::JoinPath(GetPlatform().TmpDir(), fileName + std::string{kKmlExtension}); + auto const filePathInArchive = filesDir + fileName + std::string{kKmlExtension}; + if (!SaveKmlFileSafe(*kmlToExport.second, kmlPath, KmlFileType::Text)) + continue; + pathsForArchive.push_back(kmlPath); + filesInArchive.push_back(filePathInArchive); + } + std::string indexFilePath; + try + { + indexFilePath = BuildIndexFile(filesInArchive); + } + catch (Writer::WriteException const & e) + { + LOG(LERROR, ("Error creating index file ", indexFilePath, " error ", e.Msg(), " cause ", e.what())); + return {std::move(categoriesIds), BookmarkManager::SharingResult::Code::ArchiveError, "Could not create archive."}; + } + filesInArchive.insert(filesInArchive.begin(), "doc.kml"); + pathsForArchive.insert(pathsForArchive.begin(), indexFilePath); + if (!CreateZipFromFiles(pathsForArchive, filesInArchive, kmzFilePath)) + return {std::move(categoriesIds), BookmarkManager::SharingResult::Code::ArchiveError, "Could not create archive."}; + return {std::move(categoriesIds), std::move(kmzFilePath)}; +} + +BookmarkManager::SharingResult GetFileForSharing(BookmarkManager::KMLDataCollectionPtr collection) +{ + if (collection->size() == 1) + return ExportSingleFile(collection->front()); + else + return ExportMultipleFiles(collection); +} + + std::string ToString(BookmarkManager::SortingType type) { switch (type) @@ -2613,10 +2696,16 @@ void BookmarkManager::PrepareFileForSharing(kml::GroupIdCollection && categories return; } - GetPlatform().RunTask(Platform::Thread::File, [collection = std::move(collection), handler = std::move(handler)]() mutable + if (m_testModeEnabled) { handler(GetFileForSharing(std::move(collection))); - }); + } + else + { + GetPlatform().RunTask(Platform::Thread::File, + [collection = std::move(collection), handler = std::move(handler)]() mutable + { handler(GetFileForSharing(std::move(collection))); }); + } } bool BookmarkManager::IsCategoryEmpty(kml::MarkGroupId categoryId) const diff --git a/map/map_tests/bookmarks_test.cpp b/map/map_tests/bookmarks_test.cpp index bb85245e70..5ca5535c0a 100644 --- a/map/map_tests/bookmarks_test.cpp +++ b/map/map_tests/bookmarks_test.cpp @@ -1,24 +1,25 @@ #include "testing/testing.hpp" +#include "map/bookmark_helpers.hpp" +#include "map/framework.hpp" + #include "drape_frontend/visual_params.hpp" #include "indexer/feature_utils.hpp" #include "indexer/mwm_set.hpp" -#include "map/bookmark_helpers.hpp" -#include "map/framework.hpp" - #include "platform/platform.hpp" #include "platform/preferred_languages.hpp" #include "coding/internal/file_data.hpp" #include "coding/string_utf8_multilang.hpp" +#include "coding/zip_reader.hpp" #include "base/file_name_utils.hpp" #include "base/scope_guard.hpp" #include -#include // strlen +#include // strlen #include #include #include @@ -1387,6 +1388,80 @@ UNIT_CLASS_TEST(Runner, Bookmarks_AutoSave) TEST(base::DeleteFileX(fileName2), ()); } +UNIT_CLASS_TEST(Runner, ExportAll) +{ + std::string const gpxFiles[] = { + GetPlatform().TestsDataPathForFile("gpx_test_data/route.gpx"), + GetPlatform().TestsDataPathForFile("gpx_test_data/points.gpx"), + GetPlatform().TestsDataPathForFile("gpx_test_data/Üφήが1.gpx"), + GetPlatform().TestsDataPathForFile("gpx_test_data/Üφήが2.gpx")}; + BookmarkManager bmManager(BM_CALLBACKS); + bmManager.EnableTestMode(true); + + BookmarkManager::KMLDataCollection kmlDataCollection; + for (auto const & file : gpxFiles) + kmlDataCollection.emplace_back(file, LoadKmlFile(file, KmlFileType::Gpx)); + + bmManager.CreateCategories(std::move(kmlDataCollection)); + TEST_EQUAL(bmManager.GetBmGroupsIdList().size(), 4, ()); + + auto categories = bmManager.GetBmGroupsIdList(); + auto checker = [](BookmarkManager::SharingResult const & result) + { + auto kmz = result.m_sharingPath; + ZipFileReader::FileList files; + ZipFileReader::FilesList(kmz, files); + TEST_EQUAL(files.size(), 5, ("5 files are expected in kmz")); + auto index = "doc.kml"; + std::vector expectedFiles = {"doc.kml", "files/new.kml", "files/Some random route.kml", + "files/OrganicMaps_1.kml", "files/OrganicMaps_2.kml"}; + for (auto const & file : files) + TEST(std::find(expectedFiles.begin(), expectedFiles.end(), file.first) != expectedFiles.end(), ()); + auto indexPath = base::JoinPath(GetPlatform().TmpDir(), index); + ZipFileReader::UnzipFile(kmz, index, indexPath); + std::string indexContent; + FileReader(indexPath).ReadAsString(indexContent); + std::string expectedIndexContent; + FileReader(GetPlatform().TestsDataPathForFile("kml_test_data/kmz_index.kml")) + .ReadAsString(expectedIndexContent); + TEST_EQUAL(expectedIndexContent, indexContent, ("Index content doesnt match expected value")); + auto tmpPath = base::JoinPath(GetPlatform().TmpDir(), "tmp.xml"); + for (auto const & file : files) + { + base::DeleteFileX(tmpPath); + ZipFileReader::UnzipFile(kmz, file.first, tmpPath); + } + TEST(base::DeleteFileX(kmz), ()); + TEST(base::DeleteFileX(indexPath), ()); + TEST(base::DeleteFileX(tmpPath), ()); + }; + bmManager.PrepareFileForSharing(std::move(categories), checker); +} + +UNIT_CLASS_TEST(Runner, ExportSingleUnicode) +{ + string file = GetPlatform().TestsDataPathForFile("gpx_test_data/Üφήが1.gpx"); + BookmarkManager bmManager(BM_CALLBACKS); + bmManager.EnableTestMode(true); + BookmarkManager::KMLDataCollection kmlDataCollection; + kmlDataCollection.emplace_back(file, LoadKmlFile(file, KmlFileType::Gpx)); + bmManager.CreateCategories(std::move(kmlDataCollection)); + auto categories = bmManager.GetBmGroupsIdList(); + auto checker = [](BookmarkManager::SharingResult const & result) + { + auto kmz = result.m_sharingPath; + ZipFileReader::FileList files; + ZipFileReader::FilesList(kmz, files); + TEST_EQUAL(1, files.size(), ()); + TEST_EQUAL("OrganicMaps.kml", files.at(0).first, ()); + auto tmpPath = base::JoinPath(GetPlatform().TmpDir(), "tmp.xml"); + ZipFileReader::UnzipFile(kmz, files.at(0).first, tmpPath); + TEST(base::DeleteFileX(kmz), ()); + TEST(base::DeleteFileX(tmpPath), ()); + }; + bmManager.PrepareFileForSharing(std::move(categories), checker); +} + UNIT_CLASS_TEST(Runner, Bookmarks_BrokenFile) { string const fileName = GetPlatform().TestsDataPathForFile("broken_bookmarks.kmb.test");