From ea47d8ed1c7b2ce98ea9300f0e6d29799c1c3c02 Mon Sep 17 00:00:00 2001 From: Kamil Zuranski Date: Thu, 9 Apr 2026 11:59:58 +0200 Subject: [PATCH 1/7] testing env setup --- build.sh | 3 +- endpoint/CMakeLists.txt | 4 + endpoint/programs/search_bench/CMakeLists.txt | 12 + .../search_bench/common_words_500.txt | 500 ++++++++++++++++++ endpoint/programs/search_bench/main.cpp | 188 +++++++ endpoint/programs/search_bench/run.sh | 8 + 6 files changed, 714 insertions(+), 1 deletion(-) create mode 100644 endpoint/programs/search_bench/CMakeLists.txt create mode 100644 endpoint/programs/search_bench/common_words_500.txt create mode 100644 endpoint/programs/search_bench/main.cpp create mode 100755 endpoint/programs/search_bench/run.sh diff --git a/build.sh b/build.sh index bff92d3e..59169430 100755 --- a/build.sh +++ b/build.sh @@ -17,6 +17,7 @@ cmake .. -G "Unix Makefiles" -DCMAKE_TOOLCHAIN_FILE=$GENERATORS_DIR/conan_toolch -DPRIVMX_BUILD_ENDPOINT_INTERFACE=ON \ -DPRIVMX_ENABLE_TESTS=ON \ -DPRIVMX_ENABLE_TESTS_E2E=ON \ - -DPRIVMX_BUILD_DEBUG=OFF + -DPRIVMX_BUILD_DEBUG=OFF \ + -DPRIVMX_BUILD_SEARCH_BENCHMARK=ON cmake --build . -- -j20 source $GENERATORS_DIR/deactivate_conanbuild.sh diff --git a/endpoint/CMakeLists.txt b/endpoint/CMakeLists.txt index d0bce7e9..abd80e0d 100644 --- a/endpoint/CMakeLists.txt +++ b/endpoint/CMakeLists.txt @@ -35,6 +35,10 @@ if(PRIVMX_BUILD_BENCHMARK AND PRIVMX_BUILD_ENDPOINT_ENDPOINT) add_subdirectory(programs/benchmark) endif() +if(PRIVMX_BUILD_SEARCH_BENCHMARK AND PRIVMX_BUILD_ENDPOINT_ENDPOINT) + add_subdirectory(programs/search_bench ) +endif() + if(PRIVMX_BUILD_STREAM_TESTING AND PRIVMX_BUILD_ENDPOINT_ENDPOINT) add_subdirectory(programs/stream_testing) endif() \ No newline at end of file diff --git a/endpoint/programs/search_bench/CMakeLists.txt b/endpoint/programs/search_bench/CMakeLists.txt new file mode 100644 index 00000000..5955fbc5 --- /dev/null +++ b/endpoint/programs/search_bench/CMakeLists.txt @@ -0,0 +1,12 @@ +target_include_directories(privmxendpointcrypto PRIVATE ${INCLUDE_DIRS}) +target_include_directories(privmxendpointcore PRIVATE ${INCLUDE_DIRS}) +target_include_directories(privmxendpointevent PRIVATE ${INCLUDE_DIRS}) +target_include_directories(privmxendpointthread PRIVATE ${INCLUDE_DIRS}) +target_include_directories(privmxendpointstore PRIVATE ${INCLUDE_DIRS}) +target_include_directories(privmxendpointinbox PRIVATE ${INCLUDE_DIRS}) +target_include_directories(privmxendpointkvdb PRIVATE ${INCLUDE_DIRS}) +target_include_directories(privmxendpointsearch PRIVATE ${INCLUDE_DIRS}) + +# search_bench +add_executable(search_bench ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp) +target_link_libraries(search_bench privmx privmxendpointcore privmxendpointcrypto privmxendpointstore privmxendpointkvdb privmxendpointsearch) diff --git a/endpoint/programs/search_bench/common_words_500.txt b/endpoint/programs/search_bench/common_words_500.txt new file mode 100644 index 00000000..e2f59176 --- /dev/null +++ b/endpoint/programs/search_bench/common_words_500.txt @@ -0,0 +1,500 @@ +a +about +above +across +act +action +activity +add +after +again +against +age +ago +air +all +allow +almost +alone +along +already +also +always +among +an +and +animal +another +answer +any +appear +apply +are +area +around +arrive +art +as +ask +at +away +back +bad +ball +bank +base +be +beat +beautiful +because +become +bed +been +before +begin +behind +believe +best +better +between +big +bird +black +blue +body +book +both +box +boy +bring +brother +build +business +but +buy +by +call +camera +can +car +care +carry +case +cat +cause +center +certain +chair +change +check +child +children +city +class +clean +clear +close +cold +college +color +come +common +community +company +complete +computer +consider +continue +control +cook +copy +corner +could +country +course +cover +create +cross +cut +dark +data +day +decide +deep +develop +did +die +difference +different +direct +do +doctor +dog +door +down +draw +dream +drive +drop +during +each +early +earth +east +easy +eat +education +effect +effort +eight +either +end +enough +enter +environment +especially +even +evening +event +ever +every +example +experience +eye +face +fact +family +far +farm +fast +father +feel +few +field +figure +fill +final +find +fine +finish +fire +first +fish +five +floor +follow +food +for +force +form +four +free +friend +from +front +full +game +garden +general +get +girl +give +go +good +government +great +green +ground +group +grow +guess +had +half +hand +happen +hard +has +have +he +head +hear +help +her +here +high +him +his +history +hold +home +horse +hot +hour +house +how +however +human +hundred +idea +if +important +in +include +increase +information +interest +into +is +issue +it +its +job +join +just +keep +kind +king +knew +know +land +language +large +last +late +later +laugh +law +lead +learn +leave +left +less +let +letter +life +light +like +line +list +listen +little +live +local +long +look +lot +love +low +machine +made +main +make +man +many +map +market +may +me +mean +measure +meet +member +men +message +might +mile +mind +minute +miss +money +month +more +morning +most +mother +mountain +move +much +music +must +my +name +nation +near +need +never +new +next +night +no +north +not +note +nothing +notice +now +number +of +off +offer +office +often +old +on +once +one +only +open +operation +opportunity +or +order +other +our +out +over +own +page +paper +part +party +pass +past +pay +people +perhaps +person +picture +piece +place +plan +plant +play +point +policy +poor +position +possible +power +present +pressure +pretty +problem +process +produce +product +program +project +provide +public +put +question +quick +quite +race +radio +rain +raise +reach +read +ready +real +reason +receive +record +red +remain +remember +report +research +result +return +right +river +road +rock +room +run +same +save +say +school +science +sea +season +second +see +seem +self +send +sense +service +set +seven +several +she +ship +short +should +show +side +simple +since +single +sister +sit +six +size +skill +small +social +some +someone +something +sometimes +song +soon +sound +south +space +speak +special +spring +stand +star +start +state +stay +step +still +stop +story +street +strong +student +study +such +summer +support +sure +system +table +take +talk +task +teach +team +technology +tell +ten +test +than +that +the +their +them +then +there +these +they +thing diff --git a/endpoint/programs/search_bench/main.cpp b/endpoint/programs/search_bench/main.cpp new file mode 100644 index 00000000..c8f15571 --- /dev/null +++ b/endpoint/programs/search_bench/main.cpp @@ -0,0 +1,188 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +// #include +#include + +#include +#include +#include +#include +#include +#include + +using namespace std; +using namespace privmx::endpoint; +namespace fs = std::filesystem; + +static vector getParamsList(int argc, char* argv[]) { + vector args(argv + 1, argv + argc); + return args; +} + +static std::vector readTextFileLines(const std::string& filePath) { + std::ifstream file(filePath); + if (!file.is_open()) { + throw std::runtime_error("Unable to open file: " + filePath); + } + + std::vector lines; + std::string line; + while (std::getline(file, line)) { + if (!line.empty() && line.back() == '\r') { + line.pop_back(); + } + if (line.empty()) { + continue; + } + lines.push_back(line); + } + + return lines; +} + +static void processAllTxtFiles( + const std::string& directoryPath, + const std::function& processContent +) { + if (!fs::exists(directoryPath)) { + throw std::runtime_error("Directory does not exist: " + directoryPath); + } + if (!fs::is_directory(directoryPath)) { + throw std::runtime_error("Path is not a directory: " + directoryPath); + } + + for (const auto& entry : fs::directory_iterator(directoryPath)) { + if (!entry.is_regular_file() || entry.path().extension() != ".txt") { + continue; + } + + std::ifstream file(entry.path()); + if (!file.is_open()) { + throw std::runtime_error("Unable to open file: " + entry.path().string()); + } + + std::ostringstream content; + content << file.rdbuf(); + processContent(content.str()); + } +} + +static std::vector generateMessages(const std::vector& words, int num) { + if (words.empty()) { + throw std::runtime_error("Words list cannot be empty"); + } + if (num < 0) { + throw std::runtime_error("Number of messages cannot be negative"); + } + + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution messageLengthDist(5, 20); + std::uniform_int_distribution wordIndexDist(0, words.size() - 1); + + std::vector messages; + messages.reserve(num); + + for (int i = 0; i < num; ++i) { + const int messageLength = messageLengthDist(gen); + std::ostringstream message; + + for (int j = 0; j < messageLength; ++j) { + if (j > 0) { + message << ' '; + } + message << words[wordIndexDist(gen)]; + } + + messages.push_back(message.str()); + } + + return messages; +} + +int main(int argc, char** argv) { + auto params = getParamsList(argc, argv); + if(params.size() != 5) { + std::cout << "Invalid params. Required params are: 'PrivKey', 'SolutionId', 'BridgeUrl', 'ContextId', 'docs_dir_path'" << std::endl; + return -1; + } + std::string privKey = {params[0].begin(), params[0].end()}; + std::string solutionId = {params[1].begin(), params[1].end()}; + std::string bridgeUrl = {params[2].begin(), params[2].end()}; + std::string contextId = {params[3].begin(), params[3].end()}; + std::string docsDir = {params[4].begin(), params[4].end()}; + + try { + std::cout << "Reading words from: " << docsDir << std::endl; + auto words = readTextFileLines(docsDir + "/" + "common_words_500.txt"); + std::cout << "Words count: " << words.size() << std::endl; + + core::Connection connection = core::Connection::connect( + privKey, + solutionId, + bridgeUrl + ); + kvdb::KvdbApi kvdb_api = kvdb::KvdbApi::create(connection); + store::StoreApi store_api = store::StoreApi::create(connection); + search::SearchApi search_api = search::SearchApi::create(connection, store_api, kvdb_api); + + + auto contextUsersInfo = connection.listContextUsers(contextId, {0, 100, "asc"}); + std::vector usersWithPubKey = {}; + for(const auto& userInfo : contextUsersInfo.readItems) { + usersWithPubKey.push_back(userInfo.user); + } + + auto index {search_api.createSearchIndex(contextId, usersWithPubKey, usersWithPubKey, {}, {}, search::IndexMode::WITH_CONTENT) }; + auto indexHandle {search_api.openSearchIndex(index)}; + + std::cout << "Adding docs from: " << docsDir << " to the index..." << std::endl; + int id = 1; + + // processAllTxtFiles(docsDir + "/rfc", [&](std::string content) { + // if (id > 10) return; + // std::string name = "name_" + std::to_string(id++); + // std::cout << "Adding doc: " << name << std::endl; + // search_api.addDocument(indexHandle, name, content); + // }); + + auto randomMessages = generateMessages(words, 1000); + for (auto message : randomMessages) { + std::string name = "name_" + std::to_string(id++); + std::cout << "Adding message: " << name << std::endl; + search_api.addDocument(indexHandle, name, message); + } + auto result = search_api.searchDocuments(indexHandle, "is", {.skip = 0, .limit = 100, .sortOrder = "asc"}); + for (const auto& doc : result.readItems) { + std::cout << doc.name << ": " << doc.content << std::endl; + } + search_api.closeSearchIndex(indexHandle); + std::cout << "Done!" << std::endl; + + } catch (const core::Exception& e) { + cerr << e.getFull() << endl; + } catch (const privmx::utils::PrivmxException& e) { + cerr << e.what() << endl; + cerr << e.getData() << endl; + cerr << e.getCode() << endl; + } catch (const exception& e) { + cerr << e.what() << endl; + } catch (...) { + cerr << "Error" << endl; + } + + + return 0; +} diff --git a/endpoint/programs/search_bench/run.sh b/endpoint/programs/search_bench/run.sh new file mode 100755 index 00000000..a2033c49 --- /dev/null +++ b/endpoint/programs/search_bench/run.sh @@ -0,0 +1,8 @@ +#!/bin/bash +PRIV_KEY="L3ycXibEzJm9t9swoJ4KtSmJsenHmmgRnYY79Q2TqfJMwTGaWfA7" +CONTEXT_ID="edfcb422-6428-48c1-b52b-7be4b673a170" +SOLUTION_ID="ab7ac88f-f611-4cb2-b163-5308a05b67dc" +BRIDGE_URL="http://localhost:9111" +DOCS_DIR="/home/zurek/search_bench" +cd ../../../build/endpoint/programs/search_bench || exit +./search_bench $PRIV_KEY $SOLUTION_ID $BRIDGE_URL $CONTEXT_ID $DOCS_DIR \ No newline at end of file From 08c4094347bfb0004dbe3f3c7012b61f3da0df89 Mon Sep 17 00:00:00 2001 From: Kamil Zuranski Date: Thu, 9 Apr 2026 17:28:40 +0200 Subject: [PATCH 2/7] feat: journal files optimized use --- endpoint/programs/search_bench/main.cpp | 52 ++- .../privmx/endpoint/search/PrivmxFS.hpp | 26 ++ endpoint/search/src/PrivmxFS.cpp | 345 ++++++++++++++---- endpoint/search/src/SearchApiImpl.cpp | 2 + 4 files changed, 355 insertions(+), 70 deletions(-) diff --git a/endpoint/programs/search_bench/main.cpp b/endpoint/programs/search_bench/main.cpp index c8f15571..9b940c7a 100644 --- a/endpoint/programs/search_bench/main.cpp +++ b/endpoint/programs/search_bench/main.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -158,16 +159,49 @@ int main(int argc, char** argv) { // search_api.addDocument(indexHandle, name, content); // }); - auto randomMessages = generateMessages(words, 1000); - for (auto message : randomMessages) { - std::string name = "name_" + std::to_string(id++); - std::cout << "Adding message: " << name << std::endl; - search_api.addDocument(indexHandle, name, message); - } - auto result = search_api.searchDocuments(indexHandle, "is", {.skip = 0, .limit = 100, .sortOrder = "asc"}); - for (const auto& doc : result.readItems) { - std::cout << doc.name << ": " << doc.content << std::endl; + const int batchCount = 1; + const int messagesPerBatch = 10; + long long totalBatchAddDurationMs = 0; + const auto searchStart100 = std::chrono::steady_clock::now(); + for (int i = 0; i < batchCount; i++) { + auto randomMessages = generateMessages(words, messagesPerBatch); + const auto searchStart = std::chrono::steady_clock::now(); + for (auto message : randomMessages) { + std::string name = "name_" + std::to_string(id++); + std::cout << "Adding message: " << name << std::endl; + search_api.addDocument(indexHandle, name, message); + } + const auto searchEnd = std::chrono::steady_clock::now(); + const auto searchDurationMs = std::chrono::duration_cast(searchEnd - searchStart).count(); + totalBatchAddDurationMs += searchDurationMs; + const auto totalElapsedMs = std::chrono::duration_cast(searchEnd - searchStart100).count(); + std::cout << "Adding 10 messages - took: " << searchDurationMs << " ms" << std::endl; + std::cout << "Average add time per 10-message batch after batch " << (i + 1) << ": " + << (static_cast(totalBatchAddDurationMs) / (i + 1)) << " ms" << std::endl; + std::cout << "Average total operation time per 10-message batch after batch " << (i + 1) << ": " + << (static_cast(totalElapsedMs) / (i + 1)) << " ms" << std::endl; + } + const auto searchEnd100 = std::chrono::steady_clock::now(); + const auto searchDurationMs100 = std::chrono::duration_cast(searchEnd100 - searchStart100).count(); + std::cout << "Adding 100 messages - took: " << searchDurationMs100 << " ms" << std::endl; + std::cout << "Average add time per 10-message batch: " << (static_cast(totalBatchAddDurationMs) / batchCount) << " ms" << std::endl; + std::cout << "Average total operation time per 10-message batch: " << (static_cast(searchDurationMs100) / batchCount) << " ms" << std::endl; + + + // // get existing index + // auto existingIndexes = search_api.listSearchIndexes(contextId, {0, 1, "desc"}); + // auto existingIndex = existingIndexes.readItems[0]; + // auto indexHandle = search_api.openSearchIndex(existingIndex.indexId); + // + // const auto searchStart = std::chrono::steady_clock::now(); + // auto result = search_api.searchDocuments(indexHandle, "is", {.skip = 0, .limit = 100, .sortOrder = "asc"}); + // const auto searchEnd = std::chrono::steady_clock::now(); + // const auto searchDurationMs = std::chrono::duration_cast(searchEnd - searchStart).count(); + // std::cout << "Search query took: " << searchDurationMs << " ms" << std::endl; + // for (const auto& doc : result.readItems) { + // std::cout << doc.name << ": " << doc.content << std::endl; + // } search_api.closeSearchIndex(indexHandle); std::cout << "Done!" << std::endl; diff --git a/endpoint/search/include/privmx/endpoint/search/PrivmxFS.hpp b/endpoint/search/include/privmx/endpoint/search/PrivmxFS.hpp index e8caac34..4f32c717 100644 --- a/endpoint/search/include/privmx/endpoint/search/PrivmxFS.hpp +++ b/endpoint/search/include/privmx/endpoint/search/PrivmxFS.hpp @@ -12,8 +12,11 @@ limitations under the License. #define _PRIVMXLIB_ENDPOINT_SEARCH_PRIVMXFS_HPP_ #include +#include +#include #include #include +#include #include #include "privmx/endpoint/core/Connection.hpp" @@ -76,6 +79,20 @@ class PrivmxFile { public: PrivmxFile(std::shared_ptr session, const std::string& fileId, const std::string& path); + struct MemoryFileState + { + std::string data; + LockLevel lockLevel = LockLevel::NONE; + bool reservedLock = false; + }; + + PrivmxFile( + std::shared_ptr session, + const std::string& fileId, + const std::string& path, + bool memoryOnly, + std::shared_ptr memoryFileState + ); void open(); void close(); privmx::endpoint::core::Buffer read(int64_t size, int64_t offset); @@ -93,21 +110,30 @@ class PrivmxFile int64_t fh = -1; Writer writer; LockSession lockSession; + bool memoryOnly = false; + std::shared_ptr memoryFileState; }; class PrivmxFS { public: static std::shared_ptr create(std::shared_ptr session); + static std::string getDebugStats(); PrivmxFS(const std::shared_ptr& session); std::shared_ptr openFile(const std::string& path); bool access(const std::string& path); void deleteFile(const std::string& path); private: + bool isJournalPath(const std::string& path) const; + std::string getCachedFileId(const std::string& name); std::string getFileId(const std::string& name); std::shared_ptr _session; + mutable std::mutex _fileIdCacheMutex; + std::unordered_map _fileIdCache; + mutable std::mutex _memoryFileMutex; + std::unordered_map> _memoryFiles; }; class PrivmxExtFS diff --git a/endpoint/search/src/PrivmxFS.cpp b/endpoint/search/src/PrivmxFS.cpp index 9c33efa1..d2907a10 100644 --- a/endpoint/search/src/PrivmxFS.cpp +++ b/endpoint/search/src/PrivmxFS.cpp @@ -1,14 +1,68 @@ #include "privmx/endpoint/search/PrivmxFS.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "privmx/endpoint/core/ConvertedExceptions.hpp" +#include "privmx/endpoint/core/ExceptionConverter.hpp" #include "privmx/endpoint/search/DynamicTypes.hpp" #include "privmx/endpoint/search/SearchException.hpp" -#include "privmx/endpoint/core/ExceptionConverter.hpp" -#include "privmx/endpoint/core/ConvertedExceptions.hpp" -#include static const privmx::endpoint::core::Buffer META = privmx::endpoint::core::Buffer::from("{}"); using namespace privmx::endpoint::search; +namespace { + +struct MethodDebugStat { + std::uint64_t callCount = 0; + long long totalDurationMs = 0; +}; + +std::mutex g_methodDebugStatsMutex; +std::unordered_map g_methodDebugStats; + +void recordMethodDebugStat(const std::string& methodName, long long durationMs) { + std::lock_guard lock(g_methodDebugStatsMutex); + auto& stat = g_methodDebugStats[methodName]; + ++stat.callCount; + stat.totalDurationMs += durationMs; +} + +template +decltype(auto) measureMethodCall(const std::string& methodName, Func&& func) { + const auto start = std::chrono::steady_clock::now(); + try { + if constexpr (std::is_void_v>) { + std::forward(func)(); + const auto end = std::chrono::steady_clock::now(); + const auto durationMs = std::chrono::duration_cast(end - start).count(); + recordMethodDebugStat(methodName, durationMs); + return; + } else { + auto result = std::forward(func)(); + const auto end = std::chrono::steady_clock::now(); + const auto durationMs = std::chrono::duration_cast(end - start).count(); + recordMethodDebugStat(methodName, durationMs); + return result; + } + } catch (...) { + const auto end = std::chrono::steady_clock::now(); + const auto durationMs = std::chrono::duration_cast(end - start).count(); + recordMethodDebugStat(methodName, durationMs); + throw; + } +} + +} // namespace + std::shared_ptr SessionManager::_singleton; std::shared_ptr SessionManager::get() { @@ -66,96 +120,237 @@ int64_t Writer::size() { } PrivmxFile::PrivmxFile(std::shared_ptr session, const std::string& fileId, const std::string& path) - : session(session), fileId(fileId), path(path), lockSession(session->kvdbApi, session->kvdbId, path) {} + : PrivmxFile(session, fileId, path, false, nullptr) {} + +PrivmxFile::PrivmxFile( + std::shared_ptr session, + const std::string& fileId, + const std::string& path, + bool memoryOnly, + std::shared_ptr memoryFileState +) : session(session), + fileId(fileId), + path(path), + lockSession(session->kvdbApi, session->kvdbId, path), + memoryOnly(memoryOnly), + memoryFileState(memoryFileState) {} void PrivmxFile::open() { + std::cout << "PrivmxFile::open" << std::endl; + if (memoryOnly) { + return; + } LOG_TRACE("PrivmxFile::open - ", fileId) fh = session->storeApi.openFile(fileId); } void PrivmxFile::close() { - if (fh != -1) { - session->storeApi.closeFile(fh); - fh = -1; - } + std::cout << "PrivmxFile::close" << std::endl; + return measureMethodCall("closeFile", [&]() { + if (memoryOnly) { + return; + } + if (fh != -1) { + session->storeApi.closeFile(fh); + fh = -1; + } + }); } privmx::endpoint::core::Buffer PrivmxFile::read(int64_t size, int64_t offset) { - sync(); - session->storeApi.seekInFile(fh, offset); - auto res = session->storeApi.readFromFile(fh, size); - return res; + std::cout << "PrivmxFile::read" << std::endl; + return measureMethodCall("read", [&]() { + if (memoryOnly) { + if (offset < 0) { + return privmx::endpoint::core::Buffer::from("", 0); + } + if (static_cast(offset) >= memoryFileState->data.size()) { + return privmx::endpoint::core::Buffer::from("", 0); + } + const auto availableSize = memoryFileState->data.size() - static_cast(offset); + const auto readSize = std::min(static_cast(size), availableSize); + return privmx::endpoint::core::Buffer::from(memoryFileState->data.data() + offset, readSize); + } + sync(); + session->storeApi.seekInFile(fh, offset); + auto res = session->storeApi.readFromFile(fh, size); + return res; + }); } void PrivmxFile::write(const privmx::endpoint::core::Buffer& data, int64_t offset) { - int64_t of; - std::string res; - tie(of, res) = writer.write(offset, data.stdString()); - if (of != -1) { - session->storeApi.seekInFile(fh, of); - session->storeApi.writeToFile(fh, privmx::endpoint::core::Buffer::from(res)); - } + std::cout << "PrivmxFile::write" << std::endl; + measureMethodCall("write", [&]() { + if (memoryOnly) { + if (offset < 0) { + throw std::runtime_error("Invalid write offset"); + } + const auto writeOffset = static_cast(offset); + if (memoryFileState->data.size() < writeOffset) { + memoryFileState->data.resize(writeOffset, '\0'); + } + const auto writeSize = data.size(); + if (memoryFileState->data.size() < writeOffset + writeSize) { + memoryFileState->data.resize(writeOffset + writeSize, '\0'); + } + std::memcpy(memoryFileState->data.data() + writeOffset, data.data(), writeSize); + return; + } + int64_t of; + std::string res; + tie(of, res) = writer.write(offset, data.stdString()); + if (of != -1) { + session->storeApi.seekInFile(fh, of); + session->storeApi.writeToFile(fh, privmx::endpoint::core::Buffer::from(res)); + } + }); } void PrivmxFile::truncate(int64_t size) { - session->storeApi.seekInFile(fh, size); - session->storeApi.writeToFile(fh, privmx::endpoint::core::Buffer::from("", 0), true); + std::cout << "PrivmxFile::truncate" << std::endl; + measureMethodCall("truncate", [&]() { + if (memoryOnly) { + if (size < 0) { + throw std::runtime_error("Invalid truncate size"); + } + memoryFileState->data.resize(static_cast(size), '\0'); + return; + } + session->storeApi.seekInFile(fh, size); + session->storeApi.writeToFile(fh, privmx::endpoint::core::Buffer::from("", 0), true); + }); } void PrivmxFile::sync() { - int64_t of; - std::string res; - tie(of, res) = writer.write(-1, ""); - if (of != -1) { - session->storeApi.seekInFile(fh, of); - session->storeApi.writeToFile(fh, privmx::endpoint::core::Buffer::from(res)); - } + std::cout << "PrivmxFile::sync" << std::endl; + measureMethodCall("sync", [&]() { + if (memoryOnly) { + return; + } + int64_t of; + std::string res; + tie(of, res) = writer.write(-1, ""); + if (of != -1) { + session->storeApi.seekInFile(fh, of); + session->storeApi.writeToFile(fh, privmx::endpoint::core::Buffer::from(res)); + } + }); } int64_t PrivmxFile::getFileSize() { - auto fileInfo = session->storeApi.getFile(fileId); - if(fileInfo.statusCode != 0) { - throw MalformedInternalFileException(); - } - return fileInfo.size; + std::cout << "PrivmxFile::getFileSize" << std::endl; + return measureMethodCall("getFileSize", [&]() { + if (memoryOnly) { + return static_cast(memoryFileState->data.size()); + } + auto fileInfo = session->storeApi.getFile(fileId); + if(fileInfo.statusCode != 0) { + throw MalformedInternalFileException(); + } + return fileInfo.size; + }); } bool PrivmxFile::lock(LockLevel level) { - bool val = lockSession.lock(level); - if (val) { - session->storeApi.syncFile(fh); - } - return val; + std::cout << "PrivmxFile::lock" << std::endl; + const std::string lockStatName = memoryOnly ? "lock_journal" : "lock_db"; + return measureMethodCall(lockStatName, [&]() { + if (memoryOnly) { + memoryFileState->lockLevel = level; + memoryFileState->reservedLock = level >= LockLevel::RESERVED; + return true; + } + bool val = lockSession.lock(level); + if (val) { + session->storeApi.syncFile(fh); + } + return val; + }); } bool PrivmxFile::unlock(LockLevel level) { - return lockSession.unlock(level); + std::cout << "PrivmxFile::unlock" << std::endl; + return measureMethodCall("unlock", [&]() { + if (memoryOnly) { + memoryFileState->lockLevel = level; + memoryFileState->reservedLock = level >= LockLevel::RESERVED; + return true; + } + return lockSession.unlock(level); + }); } bool PrivmxFile::checkReservedLock() { + std::cout << "PrivmxFile::checkReservedLock" << std::endl; + if (memoryOnly) { + return memoryFileState->reservedLock; + } return lockSession.checkReservedLock(); } std::shared_ptr PrivmxFS::create( std::shared_ptr session ) { + std::cout << "PrivmxFS::create" << std::endl; std::shared_ptr res = std::make_shared(session); return res; } +std::string PrivmxFS::getDebugStats() { + std::cout << "PrivmxFS::getDebugStats" << std::endl; + std::lock_guard lock(g_methodDebugStatsMutex); + std::ostringstream result; + result << "PrivmxFS debug stats:"; + for (const auto& [methodName, stat] : g_methodDebugStats) { + const double averageDurationMs = stat.callCount == 0 ? 0.0 : static_cast(stat.totalDurationMs) / stat.callCount; + result << "\n" << methodName + << " calls=" << stat.callCount + << " totalMs=" << stat.totalDurationMs + << " avgMs=" << averageDurationMs; + } + return result.str(); +} + std::shared_ptr PrivmxFS::openFile(const std::string& path) { - std::string fileId = getFileId(path); - std::shared_ptr result = std::make_shared(_session, fileId, path); - result->open(); - return result; + std::cout << "PrivmxFS::openFile" << std::endl; + return measureMethodCall("openFile", [&]() { + std::cout << "Opening file:" << path << std::endl; + if (isJournalPath(path)) { + std::lock_guard lock(_memoryFileMutex); + auto& memoryFileState = _memoryFiles[path]; + if (!memoryFileState) { + memoryFileState = std::make_shared(); + } + std::shared_ptr result = std::make_shared(_session, "", path, true, memoryFileState); + result->open(); + return result; + } + std::string fileId = getCachedFileId(path); + std::shared_ptr result = std::make_shared(_session, fileId, path); + result->open(); + return result; + }); } bool PrivmxFS::access(const std::string& path) { - LOG_TRACE("PrivmxFS::access - ", path, " | kvdbId: ",_session->kvdbId) - return _session->kvdbApi.hasEntry(_session->kvdbId, path); + std::cout << "PrivmxFS::access" << std::endl; + return measureMethodCall("access", [&]() { + if (isJournalPath(path)) { + std::lock_guard lock(_memoryFileMutex); + return _memoryFiles.find(path) != _memoryFiles.end(); + } + LOG_TRACE("PrivmxFS::access - ", path, " | kvdbId: ",_session->kvdbId) + return _session->kvdbApi.hasEntry(_session->kvdbId, path); + }); } void PrivmxFS::deleteFile(const std::string& path) { + std::cout << "PrivmxFS::deleteFile: " << path << std::endl; + if (isJournalPath(path)) { + std::lock_guard lock(_memoryFileMutex); + _memoryFiles.erase(path); + return; + } LOG_TRACE("PrivmxFS::deleteFile - ", path, " | kvdbId: ",_session->kvdbId) privmx::endpoint::kvdb::KvdbEntry kvdbEntry = _session->kvdbApi.getEntry(_session->kvdbId, path); std::string fileId = ""; @@ -165,27 +360,55 @@ void PrivmxFS::deleteFile(const std::string& path) { _session->kvdbApi.deleteEntry(_session->kvdbId, path); _session->storeApi.deleteFile(fileId); LockSession::destroyLock(_session->kvdbApi, _session->kvdbId, path); + std::lock_guard lock(_fileIdCacheMutex); + _fileIdCache.erase(path); } PrivmxFS::PrivmxFS( const std::shared_ptr& session -) : _session(session) {} +) : _session(session) { + std::cout << "PrivmxFS::PrivmxFS" << std::endl; +} -std::string PrivmxFS::getFileId(const std::string& name) { - LOG_TRACE("PrivmxFS::getFileId - ", name, " | kvdbId: ",_session->kvdbId) - try { - privmx::endpoint::kvdb::KvdbEntry kvdbEntry = _session->kvdbApi.getEntry(_session->kvdbId, name); - if(kvdbEntry.statusCode != 0) { - throw MalformedInternalFileIdException(); - } - std::string fileId = kvdbEntry.data.stdString(); - return fileId; - } catch (const privmx::endpoint::server::KvdbEntryDoesNotExistException& e) { - LOG_DEBUG("PrivmxFS::getFileId file not found, creating new file - ", name) - int64_t fh = _session->storeApi.createFile(_session->storeId, META, META, 0, true); - std::string fileId = _session->storeApi.closeFile(fh); - _session->kvdbApi.setEntry(_session->kvdbId, name, META, META, privmx::endpoint::core::Buffer::from(fileId)); - return fileId; +bool PrivmxFS::isJournalPath(const std::string& path) const { + return path.size() >= 8 && path.compare(path.size() - 8, 8, "-journal") == 0; +} + +std::string PrivmxFS::getCachedFileId(const std::string& name) { + std::cout << "PrivmxFS::getCachedFileId" << std::endl; + { + std::lock_guard lock(_fileIdCacheMutex); + auto it = _fileIdCache.find(name); + if (it != _fileIdCache.end()) { + return it->second; + } } + + std::string fileId = getFileId(name); + + std::lock_guard lock(_fileIdCacheMutex); + _fileIdCache[name] = fileId; + return fileId; +} + +std::string PrivmxFS::getFileId(const std::string& name) { + std::cout << "PrivmxFS::getFileId" << std::endl; + return measureMethodCall("getFileId", [&]() { + LOG_TRACE("PrivmxFS::getFileId - ", name, " | kvdbId: ",_session->kvdbId) + try { + privmx::endpoint::kvdb::KvdbEntry kvdbEntry = _session->kvdbApi.getEntry(_session->kvdbId, name); + if(kvdbEntry.statusCode != 0) { + throw MalformedInternalFileIdException(); + } + std::string fileId = kvdbEntry.data.stdString(); + return fileId; + } catch (const privmx::endpoint::server::KvdbEntryDoesNotExistException& e) { + LOG_DEBUG("PrivmxFS::getFileId file not found, creating new file - ", name) + int64_t fh = _session->storeApi.createFile(_session->storeId, META, META, 0, true); + std::string fileId = _session->storeApi.closeFile(fh); + _session->kvdbApi.setEntry(_session->kvdbId, name, META, META, privmx::endpoint::core::Buffer::from(fileId)); + return fileId; + } + }); } std::shared_ptr PrivmxExtFS::openFile(const std::string& path) { diff --git a/endpoint/search/src/SearchApiImpl.cpp b/endpoint/search/src/SearchApiImpl.cpp index 615b42c7..7ddba398 100644 --- a/endpoint/search/src/SearchApiImpl.cpp +++ b/endpoint/search/src/SearchApiImpl.cpp @@ -20,6 +20,7 @@ limitations under the License. #include "privmx/endpoint/store/StoreApiImpl.hpp" #include "privmx/endpoint/kvdb/KvdbApiImpl.hpp" +#include using namespace privmx::endpoint; using namespace privmx::endpoint::search; @@ -106,6 +107,7 @@ int64_t SearchApiImpl::openSearchIndex(const std::string& indexId) { void SearchApiImpl::closeSearchIndex(const int64_t indexHandle) { auto fts = _fts.get(indexHandle); fts->close(); + std::cout << PrivmxFS::getDebugStats() << std::endl; _fts.remove(indexHandle); } From d95d325b51d5dbd88bd568d2294458491fdfaf90 Mon Sep 17 00:00:00 2001 From: Kamil Zuranski Date: Thu, 9 Apr 2026 21:35:01 +0200 Subject: [PATCH 3/7] fix: addDocument() - reduce operations on backend --- .../privmx/endpoint/search/FullTextSearch.hpp | 3 +- .../privmx/endpoint/search/PrivmxFS.hpp | 2 + endpoint/search/src/FullTextSearch.cpp | 37 +++-- endpoint/search/src/PrivmxFS.cpp | 148 +++++++++++++++--- endpoint/search/src/SearchApiImpl.cpp | 2 +- endpoint/store/src/ServerApi.cpp | 2 + 6 files changed, 157 insertions(+), 37 deletions(-) diff --git a/endpoint/search/include/privmx/endpoint/search/FullTextSearch.hpp b/endpoint/search/include/privmx/endpoint/search/FullTextSearch.hpp index 274047d6..819d53eb 100644 --- a/endpoint/search/include/privmx/endpoint/search/FullTextSearch.hpp +++ b/endpoint/search/include/privmx/endpoint/search/FullTextSearch.hpp @@ -28,7 +28,7 @@ class FullTextSearch { public: static std::shared_ptr openDb(const std::string& filename, const IndexMode mode); - FullTextSearch(std::shared_ptr db, const IndexMode mode); + FullTextSearch(std::shared_ptr db, std::string filename, const IndexMode mode); int64_t addDocument(const std::string& name, const std::string& content); Document getDocument(const int64_t documentId); core::PagingList listDocuments(const core::PagingQuery& pagingQuery); @@ -44,6 +44,7 @@ class FullTextSearch int64_t getCountOfAll(); std::shared_ptr _db; + std::string _filename; IndexMode _mode; }; diff --git a/endpoint/search/include/privmx/endpoint/search/PrivmxFS.hpp b/endpoint/search/include/privmx/endpoint/search/PrivmxFS.hpp index 4f32c717..69bd5c56 100644 --- a/endpoint/search/include/privmx/endpoint/search/PrivmxFS.hpp +++ b/endpoint/search/include/privmx/endpoint/search/PrivmxFS.hpp @@ -119,6 +119,8 @@ class PrivmxFS public: static std::shared_ptr create(std::shared_ptr session); static std::string getDebugStats(); + static void beginDbOperation(const std::string& fullPath); + static void endDbOperation(const std::string& fullPath); PrivmxFS(const std::shared_ptr& session); std::shared_ptr openFile(const std::string& path); bool access(const std::string& path); diff --git a/endpoint/search/src/FullTextSearch.cpp b/endpoint/search/src/FullTextSearch.cpp index 258474bf..469560d1 100644 --- a/endpoint/search/src/FullTextSearch.cpp +++ b/endpoint/search/src/FullTextSearch.cpp @@ -12,6 +12,7 @@ limitations under the License. #include #include "privmx/endpoint/search/FullTextSearch.hpp" +#include "privmx/endpoint/search/PrivmxFS.hpp" #include "privmx/endpoint/search/PrivmxSqliteVFS.hpp" #include "privmx/endpoint/search/SearchException.hpp" #include @@ -42,30 +43,38 @@ std::shared_ptr FullTextSearch::openDb(const std::string& filena throw DatabaseAttachException(sqlite3_errmsg(db)); } - return std::make_shared(db2, mode); + return std::make_shared(db2, filename, mode); } -FullTextSearch::FullTextSearch(std::shared_ptr db, const IndexMode mode) : _db(std::move(db)), _mode(mode) {} +FullTextSearch::FullTextSearch(std::shared_ptr db, std::string filename, const IndexMode mode) + : _db(std::move(db)), _filename(std::move(filename)), _mode(mode) {} int64_t FullTextSearch::addDocument(const std::string& name, const std::string& content) { + PrivmxFS::beginDbOperation(_filename); const char* insertSql = "INSERT INTO pmx.documents (name, content) VALUES (?, ?);"; sqlite3_stmt* stmt; - if (sqlite3_prepare_v2(_db.get(), insertSql, -1, &stmt, nullptr) != SQLITE_OK) { - throw InsertPrepareException(sqlite3_errmsg(_db.get())); - } + try { + if (sqlite3_prepare_v2(_db.get(), insertSql, -1, &stmt, nullptr) != SQLITE_OK) { + throw InsertPrepareException(sqlite3_errmsg(_db.get())); + } - sqlite3_bind_text(stmt, 1, name.c_str(), -1, SQLITE_TRANSIENT); - sqlite3_bind_text(stmt, 2, content.c_str(), -1, SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 1, name.c_str(), -1, SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 2, content.c_str(), -1, SQLITE_TRANSIENT); + + int status = sqlite3_step(stmt); + if (status != SQLITE_DONE) { + sqlite3_finalize(stmt); + throw InsertExecuteException(sqlite3_errmsg(_db.get())); + } - int status = sqlite3_step(stmt); - if (status != SQLITE_DONE) { sqlite3_finalize(stmt); - throw InsertExecuteException(sqlite3_errmsg(_db.get())); + auto rowId = sqlite3_last_insert_rowid(_db.get()); + PrivmxFS::endDbOperation(_filename); + return rowId; + } catch (...) { + PrivmxFS::endDbOperation(_filename); + throw; } - - sqlite3_finalize(stmt); - - return sqlite3_last_insert_rowid(_db.get()); } Document FullTextSearch::getDocument(const int64_t documentId) { diff --git a/endpoint/search/src/PrivmxFS.cpp b/endpoint/search/src/PrivmxFS.cpp index d2907a10..eb72d4da 100644 --- a/endpoint/search/src/PrivmxFS.cpp +++ b/endpoint/search/src/PrivmxFS.cpp @@ -26,8 +26,21 @@ struct MethodDebugStat { long long totalDurationMs = 0; }; +struct ScopedPathInfo { + std::string sessionId; + std::string path; +}; + +struct ScopedDbLockState { + LockSession lockSession; + std::uint64_t refCount = 0; + bool locked = false; +}; + std::mutex g_methodDebugStatsMutex; std::unordered_map g_methodDebugStats; +std::mutex g_scopedDbLockMutex; +std::unordered_map> g_scopedDbLocks; void recordMethodDebugStat(const std::string& methodName, long long durationMs) { std::lock_guard lock(g_methodDebugStatsMutex); @@ -61,6 +74,29 @@ decltype(auto) measureMethodCall(const std::string& methodName, Func&& func) { } } +std::string makeScopedDbLockKey(const std::string& sessionId, const std::string& path) { + return sessionId + ":" + path; +} + +ScopedPathInfo parseScopedPath(const std::string& fullPath) { + Poco::Path path; + path.parse(fullPath); + if (path.depth() >= 2 && path[0] == "pmx") { + ScopedPathInfo result{.sessionId = path[1], .path = std::string()}; + path.popFrontDirectory(); + path.popFrontDirectory(); + result.path = path.toString(); + return result; + } + throw std::runtime_error("Invalid PrivmxFS scoped path: " + fullPath); +} + +bool isScopedDbLockActive(const std::shared_ptr& session, const std::string& path) { + std::lock_guard lock(g_scopedDbLockMutex); + auto it = g_scopedDbLocks.find(makeScopedDbLockKey(session->id, path)); + return it != g_scopedDbLocks.end() && it->second->locked; +} + } // namespace std::shared_ptr SessionManager::_singleton; @@ -136,7 +172,6 @@ PrivmxFile::PrivmxFile( memoryFileState(memoryFileState) {} void PrivmxFile::open() { - std::cout << "PrivmxFile::open" << std::endl; if (memoryOnly) { return; } @@ -145,7 +180,6 @@ void PrivmxFile::open() { } void PrivmxFile::close() { - std::cout << "PrivmxFile::close" << std::endl; return measureMethodCall("closeFile", [&]() { if (memoryOnly) { return; @@ -158,7 +192,6 @@ void PrivmxFile::close() { } privmx::endpoint::core::Buffer PrivmxFile::read(int64_t size, int64_t offset) { - std::cout << "PrivmxFile::read" << std::endl; return measureMethodCall("read", [&]() { if (memoryOnly) { if (offset < 0) { @@ -179,7 +212,6 @@ privmx::endpoint::core::Buffer PrivmxFile::read(int64_t size, int64_t offset) { } void PrivmxFile::write(const privmx::endpoint::core::Buffer& data, int64_t offset) { - std::cout << "PrivmxFile::write" << std::endl; measureMethodCall("write", [&]() { if (memoryOnly) { if (offset < 0) { @@ -207,7 +239,6 @@ void PrivmxFile::write(const privmx::endpoint::core::Buffer& data, int64_t offse } void PrivmxFile::truncate(int64_t size) { - std::cout << "PrivmxFile::truncate" << std::endl; measureMethodCall("truncate", [&]() { if (memoryOnly) { if (size < 0) { @@ -222,7 +253,6 @@ void PrivmxFile::truncate(int64_t size) { } void PrivmxFile::sync() { - std::cout << "PrivmxFile::sync" << std::endl; measureMethodCall("sync", [&]() { if (memoryOnly) { return; @@ -238,7 +268,6 @@ void PrivmxFile::sync() { } int64_t PrivmxFile::getFileSize() { - std::cout << "PrivmxFile::getFileSize" << std::endl; return measureMethodCall("getFileSize", [&]() { if (memoryOnly) { return static_cast(memoryFileState->data.size()); @@ -252,7 +281,6 @@ int64_t PrivmxFile::getFileSize() { } bool PrivmxFile::lock(LockLevel level) { - std::cout << "PrivmxFile::lock" << std::endl; const std::string lockStatName = memoryOnly ? "lock_journal" : "lock_db"; return measureMethodCall(lockStatName, [&]() { if (memoryOnly) { @@ -260,6 +288,9 @@ bool PrivmxFile::lock(LockLevel level) { memoryFileState->reservedLock = level >= LockLevel::RESERVED; return true; } + if (isScopedDbLockActive(session, path)) { + return true; + } bool val = lockSession.lock(level); if (val) { session->storeApi.syncFile(fh); @@ -269,35 +300,37 @@ bool PrivmxFile::lock(LockLevel level) { } bool PrivmxFile::unlock(LockLevel level) { - std::cout << "PrivmxFile::unlock" << std::endl; return measureMethodCall("unlock", [&]() { if (memoryOnly) { memoryFileState->lockLevel = level; memoryFileState->reservedLock = level >= LockLevel::RESERVED; return true; } + if (isScopedDbLockActive(session, path)) { + return true; + } return lockSession.unlock(level); }); } bool PrivmxFile::checkReservedLock() { - std::cout << "PrivmxFile::checkReservedLock" << std::endl; if (memoryOnly) { return memoryFileState->reservedLock; } + if (isScopedDbLockActive(session, path)) { + return true; + } return lockSession.checkReservedLock(); } std::shared_ptr PrivmxFS::create( std::shared_ptr session ) { - std::cout << "PrivmxFS::create" << std::endl; std::shared_ptr res = std::make_shared(session); return res; } std::string PrivmxFS::getDebugStats() { - std::cout << "PrivmxFS::getDebugStats" << std::endl; std::lock_guard lock(g_methodDebugStatsMutex); std::ostringstream result; result << "PrivmxFS debug stats:"; @@ -311,10 +344,89 @@ std::string PrivmxFS::getDebugStats() { return result.str(); } +void PrivmxFS::beginDbOperation(const std::string& fullPath) { + const auto parsed = parseScopedPath(fullPath); + if (parsed.path.size() >= 8 && parsed.path.compare(parsed.path.size() - 8, 8, "-journal") == 0) { + return; + } + + const auto session = SessionManager::get()->getSession(parsed.sessionId); + const auto key = makeScopedDbLockKey(parsed.sessionId, parsed.path); + + std::shared_ptr state; + bool shouldAcquire = false; + { + std::lock_guard lock(g_scopedDbLockMutex); + auto& entry = g_scopedDbLocks[key]; + if (!entry) { + entry = std::make_shared(ScopedDbLockState{ + .lockSession = LockSession(session->kvdbApi, session->kvdbId, parsed.path), + .refCount = 0, + .locked = false + }); + } + state = entry; + shouldAcquire = state->refCount == 0; + ++state->refCount; + } + + if (!shouldAcquire) { + return; + } + + try { + if (!state->lockSession.lock(LockLevel::EXCLUSIVE)) { + throw std::runtime_error("Unable to acquire scoped db lock"); + } + std::lock_guard lock(g_scopedDbLockMutex); + state->locked = true; + } catch (...) { + std::lock_guard lock(g_scopedDbLockMutex); + auto it = g_scopedDbLocks.find(key); + if (it != g_scopedDbLocks.end()) { + if (it->second->refCount > 0) { + --it->second->refCount; + } + if (it->second->refCount == 0) { + g_scopedDbLocks.erase(it); + } + } + throw; + } +} + +void PrivmxFS::endDbOperation(const std::string& fullPath) { + const auto parsed = parseScopedPath(fullPath); + if (parsed.path.size() >= 8 && parsed.path.compare(parsed.path.size() - 8, 8, "-journal") == 0) { + return; + } + + const auto key = makeScopedDbLockKey(parsed.sessionId, parsed.path); + std::shared_ptr state; + bool shouldRelease = false; + { + std::lock_guard lock(g_scopedDbLockMutex); + auto it = g_scopedDbLocks.find(key); + if (it == g_scopedDbLocks.end()) { + return; + } + state = it->second; + if (state->refCount > 0) { + --state->refCount; + } + shouldRelease = state->refCount == 0 && state->locked; + if (state->refCount == 0) { + g_scopedDbLocks.erase(it); + } + } + + if (shouldRelease) { + state->lockSession.unlock(LockLevel::NONE); + } +} + std::shared_ptr PrivmxFS::openFile(const std::string& path) { - std::cout << "PrivmxFS::openFile" << std::endl; return measureMethodCall("openFile", [&]() { - std::cout << "Opening file:" << path << std::endl; if (isJournalPath(path)) { std::lock_guard lock(_memoryFileMutex); auto& memoryFileState = _memoryFiles[path]; @@ -333,7 +445,6 @@ std::shared_ptr PrivmxFS::openFile(const std::string& path) { } bool PrivmxFS::access(const std::string& path) { - std::cout << "PrivmxFS::access" << std::endl; return measureMethodCall("access", [&]() { if (isJournalPath(path)) { std::lock_guard lock(_memoryFileMutex); @@ -345,7 +456,6 @@ bool PrivmxFS::access(const std::string& path) { } void PrivmxFS::deleteFile(const std::string& path) { - std::cout << "PrivmxFS::deleteFile: " << path << std::endl; if (isJournalPath(path)) { std::lock_guard lock(_memoryFileMutex); _memoryFiles.erase(path); @@ -365,16 +475,13 @@ void PrivmxFS::deleteFile(const std::string& path) { } PrivmxFS::PrivmxFS( const std::shared_ptr& session -) : _session(session) { - std::cout << "PrivmxFS::PrivmxFS" << std::endl; -} +) : _session(session) {} bool PrivmxFS::isJournalPath(const std::string& path) const { return path.size() >= 8 && path.compare(path.size() - 8, 8, "-journal") == 0; } std::string PrivmxFS::getCachedFileId(const std::string& name) { - std::cout << "PrivmxFS::getCachedFileId" << std::endl; { std::lock_guard lock(_fileIdCacheMutex); auto it = _fileIdCache.find(name); @@ -391,7 +498,6 @@ std::string PrivmxFS::getCachedFileId(const std::string& name) { } std::string PrivmxFS::getFileId(const std::string& name) { - std::cout << "PrivmxFS::getFileId" << std::endl; return measureMethodCall("getFileId", [&]() { LOG_TRACE("PrivmxFS::getFileId - ", name, " | kvdbId: ",_session->kvdbId) try { diff --git a/endpoint/search/src/SearchApiImpl.cpp b/endpoint/search/src/SearchApiImpl.cpp index 7ddba398..15efa0bf 100644 --- a/endpoint/search/src/SearchApiImpl.cpp +++ b/endpoint/search/src/SearchApiImpl.cpp @@ -98,7 +98,7 @@ core::PagingList SearchApiImpl::listSearchIndexes(const std::string int64_t SearchApiImpl::openSearchIndex(const std::string& indexId) { auto data = getIndexData(indexId); auto session = SessionManager::get()->addSession(_connection, _storeApi, _kvdbApi, indexId, data.storeId()); - std::string filename = "/pmx/" + session->id + "/index.db"; + std::string filename = "/pmx/" + session->id + "/index_db"; auto fts = FullTextSearch::openDb(filename, (IndexMode)data.mode()); fts->ensureTableCreated(); return _fts.add(fts); diff --git a/endpoint/store/src/ServerApi.cpp b/endpoint/store/src/ServerApi.cpp index d9a18a0e..7ccb3a5f 100644 --- a/endpoint/store/src/ServerApi.cpp +++ b/endpoint/store/src/ServerApi.cpp @@ -52,10 +52,12 @@ server::StoreFileCreateResult ServerApi::storeFileCreate(const server::StoreFile } server::StoreFileReadResult ServerApi::storeFileRead(const server::StoreFileReadModel& model) { + std::cout << "storeFileRead" << std::endl; return request("storeFileRead", model); } void ServerApi::storeFileWrite(const server::StoreFileWriteModel& model) { + std::cout << "storeFileWrite" << std::endl; request("storeFileWrite", model); } From 2746bc366a83640c91c7ac832c76216e374b2003 Mon Sep 17 00:00:00 2001 From: Kamil Zuranski Date: Thu, 9 Apr 2026 22:27:03 +0200 Subject: [PATCH 4/7] feat: update in test program --- endpoint/programs/search_bench/main.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/endpoint/programs/search_bench/main.cpp b/endpoint/programs/search_bench/main.cpp index 9b940c7a..541e599d 100644 --- a/endpoint/programs/search_bench/main.cpp +++ b/endpoint/programs/search_bench/main.cpp @@ -159,8 +159,8 @@ int main(int argc, char** argv) { // search_api.addDocument(indexHandle, name, content); // }); - const int batchCount = 1; - const int messagesPerBatch = 10; + const int batchCount = 10; + const int messagesPerBatch = 100; long long totalBatchAddDurationMs = 0; const auto searchStart100 = std::chrono::steady_clock::now(); for (int i = 0; i < batchCount; i++) { @@ -168,25 +168,25 @@ int main(int argc, char** argv) { const auto searchStart = std::chrono::steady_clock::now(); for (auto message : randomMessages) { std::string name = "name_" + std::to_string(id++); - std::cout << "Adding message: " << name << std::endl; + // std::cout << "Adding message: " << name << std::endl; search_api.addDocument(indexHandle, name, message); } const auto searchEnd = std::chrono::steady_clock::now(); const auto searchDurationMs = std::chrono::duration_cast(searchEnd - searchStart).count(); totalBatchAddDurationMs += searchDurationMs; const auto totalElapsedMs = std::chrono::duration_cast(searchEnd - searchStart100).count(); - std::cout << "Adding 10 messages - took: " << searchDurationMs << " ms" << std::endl; - std::cout << "Average add time per 10-message batch after batch " << (i + 1) << ": " + std::cout << "Adding " << messagesPerBatch << " messages - took: " << searchDurationMs << " ms" << std::endl; + std::cout << "Average add time per "<< messagesPerBatch<<"-message batch after batch " << (i + 1) << ": " << (static_cast(totalBatchAddDurationMs) / (i + 1)) << " ms" << std::endl; - std::cout << "Average total operation time per 10-message batch after batch " << (i + 1) << ": " + std::cout << "Average total operation time per "<(totalElapsedMs) / (i + 1)) << " ms" << std::endl; } const auto searchEnd100 = std::chrono::steady_clock::now(); const auto searchDurationMs100 = std::chrono::duration_cast(searchEnd100 - searchStart100).count(); - std::cout << "Adding 100 messages - took: " << searchDurationMs100 << " ms" << std::endl; - std::cout << "Average add time per 10-message batch: " << (static_cast(totalBatchAddDurationMs) / batchCount) << " ms" << std::endl; - std::cout << "Average total operation time per 10-message batch: " << (static_cast(searchDurationMs100) / batchCount) << " ms" << std::endl; + std::cout << "Adding "<<(messagesPerBatch * batchCount) <<" messages - took: " << searchDurationMs100 << " ms" << std::endl; + std::cout << "Average add time per "<(totalBatchAddDurationMs) / batchCount) << " ms" << std::endl; + std::cout << "Average total operation time per "<(searchDurationMs100) / batchCount) << " ms" << std::endl; // // get existing index From 3c0b9f9ed9966afbb231f705d189beb75d8d7b2d Mon Sep 17 00:00:00 2001 From: Kamil Zuranski Date: Fri, 10 Apr 2026 00:21:42 +0200 Subject: [PATCH 5/7] feat: added addDocuments method for adding multiple documents in batchs --- endpoint/programs/search_bench/main.cpp | 107 +++++++++++------- .../privmx/endpoint/search/FullTextSearch.hpp | 1 + .../privmx/endpoint/search/SearchApiImpl.hpp | 1 + .../endpoint/search/VarDeserializer.hpp | 3 + .../privmx/endpoint/search/VarSerializer.hpp | 3 + .../varinterface/SearchApiVarInterface.hpp | 12 +- .../privmx/endpoint/search/SearchApi.hpp | 9 ++ .../privmx/endpoint/search/Types.hpp | 18 ++- endpoint/search/src/FullTextSearch.cpp | 63 +++++++++-- endpoint/search/src/PrivmxFS.cpp | 1 + endpoint/search/src/SearchApi.cpp | 10 ++ endpoint/search/src/SearchApiImpl.cpp | 5 + endpoint/search/src/VarDeserializer.cpp | 7 ++ endpoint/search/src/VarSerializer.cpp | 11 ++ .../varinterface/SearchApiVarInterface.cpp | 9 ++ 15 files changed, 203 insertions(+), 57 deletions(-) diff --git a/endpoint/programs/search_bench/main.cpp b/endpoint/programs/search_bench/main.cpp index 541e599d..0dfb39e8 100644 --- a/endpoint/programs/search_bench/main.cpp +++ b/endpoint/programs/search_bench/main.cpp @@ -146,8 +146,8 @@ int main(int argc, char** argv) { usersWithPubKey.push_back(userInfo.user); } - auto index {search_api.createSearchIndex(contextId, usersWithPubKey, usersWithPubKey, {}, {}, search::IndexMode::WITH_CONTENT) }; - auto indexHandle {search_api.openSearchIndex(index)}; + // auto index {search_api.createSearchIndex(contextId, usersWithPubKey, usersWithPubKey, {}, {}, search::IndexMode::WITH_CONTENT) }; + // auto indexHandle {search_api.openSearchIndex(index)}; std::cout << "Adding docs from: " << docsDir << " to the index..." << std::endl; int id = 1; @@ -159,50 +159,75 @@ int main(int argc, char** argv) { // search_api.addDocument(indexHandle, name, content); // }); - const int batchCount = 10; + const int batchCount = 1; const int messagesPerBatch = 100; long long totalBatchAddDurationMs = 0; - const auto searchStart100 = std::chrono::steady_clock::now(); - for (int i = 0; i < batchCount; i++) { - auto randomMessages = generateMessages(words, messagesPerBatch); - const auto searchStart = std::chrono::steady_clock::now(); - for (auto message : randomMessages) { - std::string name = "name_" + std::to_string(id++); - // std::cout << "Adding message: " << name << std::endl; - search_api.addDocument(indexHandle, name, message); - } - const auto searchEnd = std::chrono::steady_clock::now(); - const auto searchDurationMs = std::chrono::duration_cast(searchEnd - searchStart).count(); - totalBatchAddDurationMs += searchDurationMs; - const auto totalElapsedMs = std::chrono::duration_cast(searchEnd - searchStart100).count(); - std::cout << "Adding " << messagesPerBatch << " messages - took: " << searchDurationMs << " ms" << std::endl; - std::cout << "Average add time per "<< messagesPerBatch<<"-message batch after batch " << (i + 1) << ": " - << (static_cast(totalBatchAddDurationMs) / (i + 1)) << " ms" << std::endl; - std::cout << "Average total operation time per "<(totalElapsedMs) / (i + 1)) << " ms" << std::endl; - - } - const auto searchEnd100 = std::chrono::steady_clock::now(); - const auto searchDurationMs100 = std::chrono::duration_cast(searchEnd100 - searchStart100).count(); - std::cout << "Adding "<<(messagesPerBatch * batchCount) <<" messages - took: " << searchDurationMs100 << " ms" << std::endl; - std::cout << "Average add time per "<(totalBatchAddDurationMs) / batchCount) << " ms" << std::endl; - std::cout << "Average total operation time per "<(searchDurationMs100) / batchCount) << " ms" << std::endl; - - - // // get existing index - // auto existingIndexes = search_api.listSearchIndexes(contextId, {0, 1, "desc"}); - // auto existingIndex = existingIndexes.readItems[0]; - // auto indexHandle = search_api.openSearchIndex(existingIndex.indexId); + // const auto searchStart100 = std::chrono::steady_clock::now(); + // for (int i = 0; i < batchCount; i++) { + // auto randomMessages = generateMessages(words, messagesPerBatch); + // const auto searchStart = std::chrono::steady_clock::now(); + // for (auto message : randomMessages) { + // std::string name = "name_" + std::to_string(id++); + // std::cout << "Adding message: " << name << std::endl; + // search_api.addDocument(indexHandle, name, message); + // } + // const auto searchEnd = std::chrono::steady_clock::now(); + // const auto searchDurationMs = std::chrono::duration_cast(searchEnd - searchStart).count(); + // totalBatchAddDurationMs += searchDurationMs; + // const auto totalElapsedMs = std::chrono::duration_cast(searchEnd - searchStart100).count(); + // std::cout << "Adding " << messagesPerBatch << " messages - took: " << searchDurationMs << " ms" << std::endl; + // std::cout << "Average add time per "<< messagesPerBatch<<"-message batch after batch " << (i + 1) << ": " + // << (static_cast(totalBatchAddDurationMs) / (i + 1)) << " ms" << std::endl; + // std::cout << "Average total operation time per "<(totalElapsedMs) / (i + 1)) << " ms" << std::endl; // - // const auto searchStart = std::chrono::steady_clock::now(); - // auto result = search_api.searchDocuments(indexHandle, "is", {.skip = 0, .limit = 100, .sortOrder = "asc"}); - // const auto searchEnd = std::chrono::steady_clock::now(); - // const auto searchDurationMs = std::chrono::duration_cast(searchEnd - searchStart).count(); - // std::cout << "Search query took: " << searchDurationMs << " ms" << std::endl; - // for (const auto& doc : result.readItems) { + // } + // const auto searchEnd100 = std::chrono::steady_clock::now(); + // const auto searchDurationMs100 = std::chrono::duration_cast(searchEnd100 - searchStart100).count(); + // std::cout << "Adding "<<(messagesPerBatch * batchCount) <<" messages - took: " << searchDurationMs100 << " ms" << std::endl; + // std::cout << "Average add time per "<(totalBatchAddDurationMs) / batchCount) << " ms" << std::endl; + // std::cout << "Average total operation time per "<(searchDurationMs100) / batchCount) << " ms" << std::endl; + + + + + // auto randomMessagesBatch = generateMessages(words, messagesPerBatch); + // std::vector documentsToAdd {}; + // int nameId = 1; + // for (auto doc : randomMessagesBatch) { + // std::string name = "name_" + std::to_string(nameId++); + // documentsToAdd.push_back({name, doc}); + // } + // const auto batchAddStart = std::chrono::steady_clock::now(); + // search_api.addDocuments(indexHandle, documentsToAdd); + // + // const auto batchAddEnd = std::chrono::steady_clock::now(); + // const auto batchAddDurationMs = std::chrono::duration_cast(batchAddEnd - batchAddStart).count(); + // + // std::cout << "Adding " << messagesPerBatch << " messages (as batch) - took: " << batchAddDurationMs << " ms" << std::endl; + + // auto added = search_api.listDocuments(indexHandle, {0, 100, "desc"}); + + // std::cout << "Added docs:" << std::endl; + // for (auto doc : added.readItems) { // std::cout << doc.name << ": " << doc.content << std::endl; // } - search_api.closeSearchIndex(indexHandle); + // std::cout << "Added docs count: " << added.readItems.size() << std::endl; + + // get existing index + auto existingIndexes = search_api.listSearchIndexes(contextId, {0, 1, "desc"}); + auto existingIndex = existingIndexes.readItems[0]; + auto indexHandle2 = search_api.openSearchIndex(existingIndex.indexId); + + const auto searchStart = std::chrono::steady_clock::now(); + auto result = search_api.searchDocuments(indexHandle2, "is", {.skip = 0, .limit = 100, .sortOrder = "asc"}); + const auto searchEnd = std::chrono::steady_clock::now(); + const auto searchDurationMs = std::chrono::duration_cast(searchEnd - searchStart).count(); + std::cout << "Search query took: " << searchDurationMs << " ms" << std::endl; + for (const auto& doc : result.readItems) { + std::cout << doc.name << ": " << doc.content << std::endl; + } + search_api.closeSearchIndex(indexHandle2); std::cout << "Done!" << std::endl; } catch (const core::Exception& e) { diff --git a/endpoint/search/include/privmx/endpoint/search/FullTextSearch.hpp b/endpoint/search/include/privmx/endpoint/search/FullTextSearch.hpp index 819d53eb..f82f5f58 100644 --- a/endpoint/search/include/privmx/endpoint/search/FullTextSearch.hpp +++ b/endpoint/search/include/privmx/endpoint/search/FullTextSearch.hpp @@ -30,6 +30,7 @@ class FullTextSearch static std::shared_ptr openDb(const std::string& filename, const IndexMode mode); FullTextSearch(std::shared_ptr db, std::string filename, const IndexMode mode); int64_t addDocument(const std::string& name, const std::string& content); + std::vector addDocuments(const std::vector& documents); Document getDocument(const int64_t documentId); core::PagingList listDocuments(const core::PagingQuery& pagingQuery); void updateDocument(const Document& document); diff --git a/endpoint/search/include/privmx/endpoint/search/SearchApiImpl.hpp b/endpoint/search/include/privmx/endpoint/search/SearchApiImpl.hpp index 1fb8ecc2..157c367f 100644 --- a/endpoint/search/include/privmx/endpoint/search/SearchApiImpl.hpp +++ b/endpoint/search/include/privmx/endpoint/search/SearchApiImpl.hpp @@ -89,6 +89,7 @@ class SearchApiImpl : public privmx::utils::ManualManagedClass void closeSearchIndex(const int64_t indexHandle); int64_t addDocument(const int64_t indexHandle, const std::string& name, const std::string& content); + std::vector addDocuments(const int64_t indexHandle, const std::vector& documents); void updateDocument(const int64_t indexHandle, const Document& document); diff --git a/endpoint/search/include/privmx/endpoint/search/VarDeserializer.hpp b/endpoint/search/include/privmx/endpoint/search/VarDeserializer.hpp index b3a725e2..a84dbd20 100644 --- a/endpoint/search/include/privmx/endpoint/search/VarDeserializer.hpp +++ b/endpoint/search/include/privmx/endpoint/search/VarDeserializer.hpp @@ -25,6 +25,9 @@ namespace core { template<> search::IndexMode VarDeserializer::deserialize(const Poco::Dynamic::Var& val, const std::string& name); +template<> +search::NewDocument VarDeserializer::deserialize(const Poco::Dynamic::Var& val, const std::string& name); + template<> search::Document VarDeserializer::deserialize(const Poco::Dynamic::Var& val, const std::string& name); diff --git a/endpoint/search/include/privmx/endpoint/search/VarSerializer.hpp b/endpoint/search/include/privmx/endpoint/search/VarSerializer.hpp index 4a946018..50a8a640 100644 --- a/endpoint/search/include/privmx/endpoint/search/VarSerializer.hpp +++ b/endpoint/search/include/privmx/endpoint/search/VarSerializer.hpp @@ -35,6 +35,9 @@ Poco::Dynamic::Var VarSerializer::serialize(const search::Ind template<> Poco::Dynamic::Var VarSerializer::serialize(const search::SearchIndex& val); +template<> +Poco::Dynamic::Var VarSerializer::serialize(const search::NewDocument& val); + template<> Poco::Dynamic::Var VarSerializer::serialize(const search::Document& val); diff --git a/endpoint/search/include/privmx/endpoint/search/varinterface/SearchApiVarInterface.hpp b/endpoint/search/include/privmx/endpoint/search/varinterface/SearchApiVarInterface.hpp index f284b8db..334c8234 100644 --- a/endpoint/search/include/privmx/endpoint/search/varinterface/SearchApiVarInterface.hpp +++ b/endpoint/search/include/privmx/endpoint/search/varinterface/SearchApiVarInterface.hpp @@ -34,11 +34,12 @@ class SearchApiVarInterface { OpenSearchIndex = 6, CloseSearchIndex = 7, AddDocument = 8, - UpdateDocument = 9, - DeleteDocument = 10, - GetDocument = 11, - ListDocuments = 12, - SearchDocuments = 13, + AddDocuments = 9, + UpdateDocument = 10, + DeleteDocument = 11, + GetDocument = 12, + ListDocuments = 13, + SearchDocuments = 14, }; SearchApiVarInterface(core::Connection connection, store::StoreApi storeApi, kvdb::KvdbApi kvdbApi, const core::VarSerializer& serializer) @@ -53,6 +54,7 @@ class SearchApiVarInterface { Poco::Dynamic::Var openSearchIndex(const Poco::Dynamic::Var& args); Poco::Dynamic::Var closeSearchIndex(const Poco::Dynamic::Var& args); Poco::Dynamic::Var addDocument(const Poco::Dynamic::Var& args); + Poco::Dynamic::Var addDocuments(const Poco::Dynamic::Var& args); Poco::Dynamic::Var updateDocument(const Poco::Dynamic::Var& args); Poco::Dynamic::Var deleteDocument(const Poco::Dynamic::Var& args); Poco::Dynamic::Var getDocument(const Poco::Dynamic::Var& args); diff --git a/endpoint/search/include_pub/privmx/endpoint/search/SearchApi.hpp b/endpoint/search/include_pub/privmx/endpoint/search/SearchApi.hpp index 9a42109f..c0cedf86 100644 --- a/endpoint/search/include_pub/privmx/endpoint/search/SearchApi.hpp +++ b/endpoint/search/include_pub/privmx/endpoint/search/SearchApi.hpp @@ -137,6 +137,15 @@ class SearchApi : public privmx::endpoint::core::ExtendedPointer */ int64_t addDocument(const int64_t indexHandle, const std::string& name, const std::string& content); + /** + * Adds multiple new documents to the Search Index in a single batch. + * + * @param indexHandle Handle of the Index to add the documents to + * @param documents Documents to add + * @return IDs of the newly added documents in input order + */ + std::vector addDocuments(const int64_t indexHandle, const std::vector& documents); + /** * Updates an existing document in the Search Index. * diff --git a/endpoint/search/include_pub/privmx/endpoint/search/Types.hpp b/endpoint/search/include_pub/privmx/endpoint/search/Types.hpp index 504a2316..1d3818fe 100644 --- a/endpoint/search/include_pub/privmx/endpoint/search/Types.hpp +++ b/endpoint/search/include_pub/privmx/endpoint/search/Types.hpp @@ -130,7 +130,23 @@ struct SearchIndex }; /** - * A structure representing a document for indexing. + * A structure representing a new document for indexing. + */ +struct NewDocument +{ + /** + * Document name + */ + std::string name; + + /** + * Document content + */ + std::string content; +}; + +/** + * A structure representing a document stored in the index. */ struct Document { diff --git a/endpoint/search/src/FullTextSearch.cpp b/endpoint/search/src/FullTextSearch.cpp index 469560d1..c4433b2c 100644 --- a/endpoint/search/src/FullTextSearch.cpp +++ b/endpoint/search/src/FullTextSearch.cpp @@ -50,28 +50,71 @@ FullTextSearch::FullTextSearch(std::shared_ptr db, std::string filename : _db(std::move(db)), _filename(std::move(filename)), _mode(mode) {} int64_t FullTextSearch::addDocument(const std::string& name, const std::string& content) { + NewDocument document; + document.name = name; + document.content = content; + auto result = addDocuments({document}); + return result.front(); +} + +std::vector FullTextSearch::addDocuments(const std::vector& documents) { + if (documents.empty()) { + return {}; + } + PrivmxFS::beginDbOperation(_filename); + const char* beginSql = "BEGIN IMMEDIATE;"; + const char* commitSql = "COMMIT;"; + const char* rollbackSql = "ROLLBACK;"; const char* insertSql = "INSERT INTO pmx.documents (name, content) VALUES (?, ?);"; - sqlite3_stmt* stmt; + sqlite3_stmt* stmt = nullptr; + bool transactionStarted = false; + std::vector rowIds; + rowIds.reserve(documents.size()); try { + if (sqlite3_exec(_db.get(), beginSql, nullptr, nullptr, nullptr) != SQLITE_OK) { + throw InsertExecuteException(sqlite3_errmsg(_db.get())); + } + transactionStarted = true; + if (sqlite3_prepare_v2(_db.get(), insertSql, -1, &stmt, nullptr) != SQLITE_OK) { throw InsertPrepareException(sqlite3_errmsg(_db.get())); } - sqlite3_bind_text(stmt, 1, name.c_str(), -1, SQLITE_TRANSIENT); - sqlite3_bind_text(stmt, 2, content.c_str(), -1, SQLITE_TRANSIENT); - - int status = sqlite3_step(stmt); - if (status != SQLITE_DONE) { - sqlite3_finalize(stmt); - throw InsertExecuteException(sqlite3_errmsg(_db.get())); + for (const auto& document : documents) { + sqlite3_bind_text(stmt, 1, document.name.c_str(), -1, SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 2, document.content.c_str(), -1, SQLITE_TRANSIENT); + + int status = sqlite3_step(stmt); + if (status != SQLITE_DONE) { + sqlite3_finalize(stmt); + stmt = nullptr; + throw InsertExecuteException(sqlite3_errmsg(_db.get())); + } + + rowIds.push_back(sqlite3_last_insert_rowid(_db.get())); + if (sqlite3_reset(stmt) != SQLITE_OK) { + sqlite3_finalize(stmt); + stmt = nullptr; + throw InsertExecuteException(sqlite3_errmsg(_db.get())); + } + sqlite3_clear_bindings(stmt); } sqlite3_finalize(stmt); - auto rowId = sqlite3_last_insert_rowid(_db.get()); + stmt = nullptr; + if (sqlite3_exec(_db.get(), commitSql, nullptr, nullptr, nullptr) != SQLITE_OK) { + throw InsertExecuteException(sqlite3_errmsg(_db.get())); + } PrivmxFS::endDbOperation(_filename); - return rowId; + return rowIds; } catch (...) { + if (stmt != nullptr) { + sqlite3_finalize(stmt); + } + if (transactionStarted) { + sqlite3_exec(_db.get(), rollbackSql, nullptr, nullptr, nullptr); + } PrivmxFS::endDbOperation(_filename); throw; } diff --git a/endpoint/search/src/PrivmxFS.cpp b/endpoint/search/src/PrivmxFS.cpp index eb72d4da..18b25594 100644 --- a/endpoint/search/src/PrivmxFS.cpp +++ b/endpoint/search/src/PrivmxFS.cpp @@ -51,6 +51,7 @@ void recordMethodDebugStat(const std::string& methodName, long long durationMs) template decltype(auto) measureMethodCall(const std::string& methodName, Func&& func) { + std::cout << "PrivmxFS::" << methodName << std::endl; const auto start = std::chrono::steady_clock::now(); try { if constexpr (std::is_void_v>) { diff --git a/endpoint/search/src/SearchApi.cpp b/endpoint/search/src/SearchApi.cpp index 8acad909..738467b6 100644 --- a/endpoint/search/src/SearchApi.cpp +++ b/endpoint/search/src/SearchApi.cpp @@ -156,6 +156,16 @@ int64_t SearchApi::addDocument(const int64_t indexHandle, const std::string& nam } } +std::vector SearchApi::addDocuments(const int64_t indexHandle, const std::vector& documents) { + auto impl = getImpl(); + try { + return impl->addDocuments(indexHandle, documents); + } catch (const privmx::utils::PrivmxException& e) { + core::ExceptionConverter::rethrowAsCoreException(e); + throw core::Exception("ExceptionConverter rethrow error"); + } +} + void SearchApi::updateDocument(const int64_t indexHandle, const Document& document) { auto impl = getImpl(); try { diff --git a/endpoint/search/src/SearchApiImpl.cpp b/endpoint/search/src/SearchApiImpl.cpp index 15efa0bf..2f524e72 100644 --- a/endpoint/search/src/SearchApiImpl.cpp +++ b/endpoint/search/src/SearchApiImpl.cpp @@ -116,6 +116,11 @@ int64_t SearchApiImpl::addDocument(const int64_t indexHandle, const std::string& return fts->addDocument(name, content); } +std::vector SearchApiImpl::addDocuments(const int64_t indexHandle, const std::vector& documents) { + auto fts = _fts.get(indexHandle); + return fts->addDocuments(documents); +} + void SearchApiImpl::updateDocument(const int64_t indexHandle, const Document& document) { auto fts = _fts.get(indexHandle); fts->updateDocument(document); diff --git a/endpoint/search/src/VarDeserializer.cpp b/endpoint/search/src/VarDeserializer.cpp index ecb1ca3c..b7c2c2b8 100644 --- a/endpoint/search/src/VarDeserializer.cpp +++ b/endpoint/search/src/VarDeserializer.cpp @@ -36,3 +36,10 @@ search::Document VarDeserializer::deserialize(const Poco::Dyna .name = deserialize(obj->get("name"), name + ".name"), .content = deserialize(obj->get("content"), name + ".content")}; } + +template<> +search::NewDocument VarDeserializer::deserialize(const Poco::Dynamic::Var& val, const std::string& name) { + Poco::JSON::Object::Ptr obj = val.extract(); + return {.name = deserialize(obj->get("name"), name + ".name"), + .content = deserialize(obj->get("content"), name + ".content")}; +} diff --git a/endpoint/search/src/VarSerializer.cpp b/endpoint/search/src/VarSerializer.cpp index c0288a3d..a6619197 100644 --- a/endpoint/search/src/VarSerializer.cpp +++ b/endpoint/search/src/VarSerializer.cpp @@ -82,3 +82,14 @@ Poco::Dynamic::Var VarSerializer::serialize(const search::Docu obj->set("content", serialize(val.content)); return obj; } + +template<> +Poco::Dynamic::Var VarSerializer::serialize(const search::NewDocument& val) { + Poco::JSON::Object::Ptr obj = new Poco::JSON::Object(); + if (_options.addType) { + obj->set("__type", "search$NewDocument"); + } + obj->set("name", serialize(val.name)); + obj->set("content", serialize(val.content)); + return obj; +} diff --git a/endpoint/search/src/varinterface/SearchApiVarInterface.cpp b/endpoint/search/src/varinterface/SearchApiVarInterface.cpp index fafdeb89..927fc316 100644 --- a/endpoint/search/src/varinterface/SearchApiVarInterface.cpp +++ b/endpoint/search/src/varinterface/SearchApiVarInterface.cpp @@ -29,6 +29,7 @@ std::map(argsArr->get(0), "indexHandle"); + auto documents = _deserializer.deserializeVector(argsArr->get(1), "documents"); + auto result = _searchApi.addDocuments(indexHandle, documents); + return _serializer.serialize(result); +} + Poco::Dynamic::Var SearchApiVarInterface::updateDocument(const Poco::Dynamic::Var& args) { auto argsArr = core::VarInterfaceUtil::validateAndExtractArray(args, 2); auto indexHandle = _deserializer.deserialize(argsArr->get(0), "indexHandle"); From f86b47fcf4a27e25f0a0dfdca0e6f7be14c25afc Mon Sep 17 00:00:00 2001 From: Kamil Zuranski Date: Sun, 12 Apr 2026 21:57:35 +0200 Subject: [PATCH 6/7] feat: add test on real data --- endpoint/programs/search_bench/main.cpp | 57 ++++++++++++++----------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/endpoint/programs/search_bench/main.cpp b/endpoint/programs/search_bench/main.cpp index 0dfb39e8..03e8ea02 100644 --- a/endpoint/programs/search_bench/main.cpp +++ b/endpoint/programs/search_bench/main.cpp @@ -22,6 +22,7 @@ #include #include #include +#include using namespace std; using namespace privmx::endpoint; @@ -65,7 +66,7 @@ static void processAllTxtFiles( } for (const auto& entry : fs::directory_iterator(directoryPath)) { - if (!entry.is_regular_file() || entry.path().extension() != ".txt") { + if (!entry.is_regular_file()) { continue; } @@ -113,6 +114,14 @@ static std::vector generateMessages(const std::vector& return messages; } +static std::string extractMessageDataFromJson(const std::string& json) { + auto jsonObject = privmx::utils::Utils::parseJsonObject(json); + if (!jsonObject->has("data")) { + throw std::runtime_error("JSON does not contain 'data' field"); + } + return jsonObject->getValue("data"); +} + int main(int argc, char** argv) { auto params = getParamsList(argc, argv); if(params.size() != 5) { @@ -146,30 +155,30 @@ int main(int argc, char** argv) { usersWithPubKey.push_back(userInfo.user); } - // auto index {search_api.createSearchIndex(contextId, usersWithPubKey, usersWithPubKey, {}, {}, search::IndexMode::WITH_CONTENT) }; - // auto indexHandle {search_api.openSearchIndex(index)}; + auto index {search_api.createSearchIndex(contextId, usersWithPubKey, usersWithPubKey, {}, {}, search::IndexMode::WITH_CONTENT) }; + auto indexHandle {search_api.openSearchIndex(index)}; std::cout << "Adding docs from: " << docsDir << " to the index..." << std::endl; int id = 1; - // processAllTxtFiles(docsDir + "/rfc", [&](std::string content) { - // if (id > 10) return; - // std::string name = "name_" + std::to_string(id++); - // std::cout << "Adding doc: " << name << std::endl; - // search_api.addDocument(indexHandle, name, content); - // }); + std::vector documentsToAdd {}; + processAllTxtFiles(docsDir + "/msgs", [&](std::string content) { + if (id > 100) return; + std::string name = "name_" + std::to_string(id++); + documentsToAdd.push_back({name, extractMessageDataFromJson(content)}); + }); + std::cout << "Adding " << documentsToAdd.size() << " documents..." << std::endl; const int batchCount = 1; const int messagesPerBatch = 100; long long totalBatchAddDurationMs = 0; - // const auto searchStart100 = std::chrono::steady_clock::now(); + const auto searchStart100 = std::chrono::steady_clock::now(); // for (int i = 0; i < batchCount; i++) { - // auto randomMessages = generateMessages(words, messagesPerBatch); + // // auto randomMessages = generateMessages(words, messagesPerBatch); // const auto searchStart = std::chrono::steady_clock::now(); - // for (auto message : randomMessages) { - // std::string name = "name_" + std::to_string(id++); - // std::cout << "Adding message: " << name << std::endl; - // search_api.addDocument(indexHandle, name, message); + // for (auto message : documentsToAdd) { + // id++; + // search_api.addDocument(indexHandle, message.name, message.content); // } // const auto searchEnd = std::chrono::steady_clock::now(); // const auto searchDurationMs = std::chrono::duration_cast(searchEnd - searchStart).count(); @@ -198,21 +207,22 @@ int main(int argc, char** argv) { // std::string name = "name_" + std::to_string(nameId++); // documentsToAdd.push_back({name, doc}); // } - // const auto batchAddStart = std::chrono::steady_clock::now(); - // search_api.addDocuments(indexHandle, documentsToAdd); - // - // const auto batchAddEnd = std::chrono::steady_clock::now(); - // const auto batchAddDurationMs = std::chrono::duration_cast(batchAddEnd - batchAddStart).count(); - // - // std::cout << "Adding " << messagesPerBatch << " messages (as batch) - took: " << batchAddDurationMs << " ms" << std::endl; + const auto batchAddStart = std::chrono::steady_clock::now(); + search_api.addDocuments(indexHandle, documentsToAdd); + + const auto batchAddEnd = std::chrono::steady_clock::now(); + const auto batchAddDurationMs = std::chrono::duration_cast(batchAddEnd - batchAddStart).count(); + + std::cout << "Adding " << messagesPerBatch << " messages (as batch) - took: " << batchAddDurationMs << " ms" << std::endl; - // auto added = search_api.listDocuments(indexHandle, {0, 100, "desc"}); + auto added = search_api.listDocuments(indexHandle, {0, 100, "desc"}); // std::cout << "Added docs:" << std::endl; // for (auto doc : added.readItems) { // std::cout << doc.name << ": " << doc.content << std::endl; // } // std::cout << "Added docs count: " << added.readItems.size() << std::endl; + search_api.closeSearchIndex(indexHandle); // get existing index auto existingIndexes = search_api.listSearchIndexes(contextId, {0, 1, "desc"}); @@ -241,7 +251,6 @@ int main(int argc, char** argv) { } catch (...) { cerr << "Error" << endl; } - return 0; } From ad88ad4bed308b4404036f288cd3bd7b0dd11b21 Mon Sep 17 00:00:00 2001 From: Kamil Zuranski Date: Sun, 12 Apr 2026 22:20:27 +0200 Subject: [PATCH 7/7] refactor: code cleanup --- .../privmx/endpoint/search/PrivmxFS.hpp | 1 - endpoint/search/src/PrivmxFS.cpp | 316 +++++++----------- endpoint/search/src/SearchApiImpl.cpp | 1 - 3 files changed, 120 insertions(+), 198 deletions(-) diff --git a/endpoint/search/include/privmx/endpoint/search/PrivmxFS.hpp b/endpoint/search/include/privmx/endpoint/search/PrivmxFS.hpp index 69bd5c56..2ebc58b3 100644 --- a/endpoint/search/include/privmx/endpoint/search/PrivmxFS.hpp +++ b/endpoint/search/include/privmx/endpoint/search/PrivmxFS.hpp @@ -118,7 +118,6 @@ class PrivmxFS { public: static std::shared_ptr create(std::shared_ptr session); - static std::string getDebugStats(); static void beginDbOperation(const std::string& fullPath); static void endDbOperation(const std::string& fullPath); PrivmxFS(const std::shared_ptr& session); diff --git a/endpoint/search/src/PrivmxFS.cpp b/endpoint/search/src/PrivmxFS.cpp index 18b25594..c8b66685 100644 --- a/endpoint/search/src/PrivmxFS.cpp +++ b/endpoint/search/src/PrivmxFS.cpp @@ -21,11 +21,6 @@ using namespace privmx::endpoint::search; namespace { -struct MethodDebugStat { - std::uint64_t callCount = 0; - long long totalDurationMs = 0; -}; - struct ScopedPathInfo { std::string sessionId; std::string path; @@ -37,44 +32,9 @@ struct ScopedDbLockState { bool locked = false; }; -std::mutex g_methodDebugStatsMutex; -std::unordered_map g_methodDebugStats; std::mutex g_scopedDbLockMutex; std::unordered_map> g_scopedDbLocks; -void recordMethodDebugStat(const std::string& methodName, long long durationMs) { - std::lock_guard lock(g_methodDebugStatsMutex); - auto& stat = g_methodDebugStats[methodName]; - ++stat.callCount; - stat.totalDurationMs += durationMs; -} - -template -decltype(auto) measureMethodCall(const std::string& methodName, Func&& func) { - std::cout << "PrivmxFS::" << methodName << std::endl; - const auto start = std::chrono::steady_clock::now(); - try { - if constexpr (std::is_void_v>) { - std::forward(func)(); - const auto end = std::chrono::steady_clock::now(); - const auto durationMs = std::chrono::duration_cast(end - start).count(); - recordMethodDebugStat(methodName, durationMs); - return; - } else { - auto result = std::forward(func)(); - const auto end = std::chrono::steady_clock::now(); - const auto durationMs = std::chrono::duration_cast(end - start).count(); - recordMethodDebugStat(methodName, durationMs); - return result; - } - } catch (...) { - const auto end = std::chrono::steady_clock::now(); - const auto durationMs = std::chrono::duration_cast(end - start).count(); - recordMethodDebugStat(methodName, durationMs); - throw; - } -} - std::string makeScopedDbLockKey(const std::string& sessionId, const std::string& path) { return sessionId + ":" + path; } @@ -181,137 +141,121 @@ void PrivmxFile::open() { } void PrivmxFile::close() { - return measureMethodCall("closeFile", [&]() { - if (memoryOnly) { - return; - } - if (fh != -1) { - session->storeApi.closeFile(fh); - fh = -1; - } - }); + if (memoryOnly) { + return; + } + if (fh != -1) { + session->storeApi.closeFile(fh); + fh = -1; + } } privmx::endpoint::core::Buffer PrivmxFile::read(int64_t size, int64_t offset) { - return measureMethodCall("read", [&]() { - if (memoryOnly) { - if (offset < 0) { - return privmx::endpoint::core::Buffer::from("", 0); - } - if (static_cast(offset) >= memoryFileState->data.size()) { - return privmx::endpoint::core::Buffer::from("", 0); - } - const auto availableSize = memoryFileState->data.size() - static_cast(offset); - const auto readSize = std::min(static_cast(size), availableSize); - return privmx::endpoint::core::Buffer::from(memoryFileState->data.data() + offset, readSize); + if (memoryOnly) { + if (offset < 0) { + return privmx::endpoint::core::Buffer::from("", 0); } - sync(); - session->storeApi.seekInFile(fh, offset); - auto res = session->storeApi.readFromFile(fh, size); - return res; - }); + if (static_cast(offset) >= memoryFileState->data.size()) { + return privmx::endpoint::core::Buffer::from("", 0); + } + const auto availableSize = memoryFileState->data.size() - static_cast(offset); + const auto readSize = std::min(static_cast(size), availableSize); + return privmx::endpoint::core::Buffer::from(memoryFileState->data.data() + offset, readSize); + } + sync(); + session->storeApi.seekInFile(fh, offset); + auto res = session->storeApi.readFromFile(fh, size); + return res; } void PrivmxFile::write(const privmx::endpoint::core::Buffer& data, int64_t offset) { - measureMethodCall("write", [&]() { - if (memoryOnly) { - if (offset < 0) { - throw std::runtime_error("Invalid write offset"); - } - const auto writeOffset = static_cast(offset); - if (memoryFileState->data.size() < writeOffset) { - memoryFileState->data.resize(writeOffset, '\0'); - } - const auto writeSize = data.size(); - if (memoryFileState->data.size() < writeOffset + writeSize) { - memoryFileState->data.resize(writeOffset + writeSize, '\0'); - } - std::memcpy(memoryFileState->data.data() + writeOffset, data.data(), writeSize); - return; + if (memoryOnly) { + if (offset < 0) { + throw std::runtime_error("Invalid write offset"); } - int64_t of; - std::string res; - tie(of, res) = writer.write(offset, data.stdString()); - if (of != -1) { - session->storeApi.seekInFile(fh, of); - session->storeApi.writeToFile(fh, privmx::endpoint::core::Buffer::from(res)); + const auto writeOffset = static_cast(offset); + if (memoryFileState->data.size() < writeOffset) { + memoryFileState->data.resize(writeOffset, '\0'); } - }); + const auto writeSize = data.size(); + if (memoryFileState->data.size() < writeOffset + writeSize) { + memoryFileState->data.resize(writeOffset + writeSize, '\0'); + } + std::memcpy(memoryFileState->data.data() + writeOffset, data.data(), writeSize); + return; + } + int64_t of; + std::string res; + tie(of, res) = writer.write(offset, data.stdString()); + if (of != -1) { + session->storeApi.seekInFile(fh, of); + session->storeApi.writeToFile(fh, privmx::endpoint::core::Buffer::from(res)); + } } void PrivmxFile::truncate(int64_t size) { - measureMethodCall("truncate", [&]() { - if (memoryOnly) { - if (size < 0) { - throw std::runtime_error("Invalid truncate size"); - } - memoryFileState->data.resize(static_cast(size), '\0'); - return; + if (memoryOnly) { + if (size < 0) { + throw std::runtime_error("Invalid truncate size"); } - session->storeApi.seekInFile(fh, size); - session->storeApi.writeToFile(fh, privmx::endpoint::core::Buffer::from("", 0), true); - }); + memoryFileState->data.resize(static_cast(size), '\0'); + return; + } + session->storeApi.seekInFile(fh, size); + session->storeApi.writeToFile(fh, privmx::endpoint::core::Buffer::from("", 0), true); } void PrivmxFile::sync() { - measureMethodCall("sync", [&]() { - if (memoryOnly) { - return; - } - int64_t of; - std::string res; - tie(of, res) = writer.write(-1, ""); - if (of != -1) { - session->storeApi.seekInFile(fh, of); - session->storeApi.writeToFile(fh, privmx::endpoint::core::Buffer::from(res)); - } - }); + if (memoryOnly) { + return; + } + int64_t of; + std::string res; + tie(of, res) = writer.write(-1, ""); + if (of != -1) { + session->storeApi.seekInFile(fh, of); + session->storeApi.writeToFile(fh, privmx::endpoint::core::Buffer::from(res)); + } } int64_t PrivmxFile::getFileSize() { - return measureMethodCall("getFileSize", [&]() { - if (memoryOnly) { - return static_cast(memoryFileState->data.size()); - } - auto fileInfo = session->storeApi.getFile(fileId); - if(fileInfo.statusCode != 0) { - throw MalformedInternalFileException(); - } - return fileInfo.size; - }); + if (memoryOnly) { + return static_cast(memoryFileState->data.size()); + } + auto fileInfo = session->storeApi.getFile(fileId); + if(fileInfo.statusCode != 0) { + throw MalformedInternalFileException(); + } + return fileInfo.size; } bool PrivmxFile::lock(LockLevel level) { const std::string lockStatName = memoryOnly ? "lock_journal" : "lock_db"; - return measureMethodCall(lockStatName, [&]() { - if (memoryOnly) { - memoryFileState->lockLevel = level; - memoryFileState->reservedLock = level >= LockLevel::RESERVED; - return true; - } - if (isScopedDbLockActive(session, path)) { - return true; - } - bool val = lockSession.lock(level); - if (val) { - session->storeApi.syncFile(fh); - } - return val; - }); + if (memoryOnly) { + memoryFileState->lockLevel = level; + memoryFileState->reservedLock = level >= LockLevel::RESERVED; + return true; + } + if (isScopedDbLockActive(session, path)) { + return true; + } + bool val = lockSession.lock(level); + if (val) { + session->storeApi.syncFile(fh); + } + return val; } bool PrivmxFile::unlock(LockLevel level) { - return measureMethodCall("unlock", [&]() { - if (memoryOnly) { - memoryFileState->lockLevel = level; - memoryFileState->reservedLock = level >= LockLevel::RESERVED; - return true; - } - if (isScopedDbLockActive(session, path)) { - return true; - } - return lockSession.unlock(level); - }); + if (memoryOnly) { + memoryFileState->lockLevel = level; + memoryFileState->reservedLock = level >= LockLevel::RESERVED; + return true; + } + if (isScopedDbLockActive(session, path)) { + return true; + } + return lockSession.unlock(level); } bool PrivmxFile::checkReservedLock() { @@ -331,20 +275,6 @@ std::shared_ptr PrivmxFS::create( return res; } -std::string PrivmxFS::getDebugStats() { - std::lock_guard lock(g_methodDebugStatsMutex); - std::ostringstream result; - result << "PrivmxFS debug stats:"; - for (const auto& [methodName, stat] : g_methodDebugStats) { - const double averageDurationMs = stat.callCount == 0 ? 0.0 : static_cast(stat.totalDurationMs) / stat.callCount; - result << "\n" << methodName - << " calls=" << stat.callCount - << " totalMs=" << stat.totalDurationMs - << " avgMs=" << averageDurationMs; - } - return result.str(); -} - void PrivmxFS::beginDbOperation(const std::string& fullPath) { const auto parsed = parseScopedPath(fullPath); if (parsed.path.size() >= 8 && parsed.path.compare(parsed.path.size() - 8, 8, "-journal") == 0) { @@ -427,33 +357,29 @@ void PrivmxFS::endDbOperation(const std::string& fullPath) { } std::shared_ptr PrivmxFS::openFile(const std::string& path) { - return measureMethodCall("openFile", [&]() { - if (isJournalPath(path)) { - std::lock_guard lock(_memoryFileMutex); - auto& memoryFileState = _memoryFiles[path]; - if (!memoryFileState) { - memoryFileState = std::make_shared(); - } - std::shared_ptr result = std::make_shared(_session, "", path, true, memoryFileState); - result->open(); - return result; + if (isJournalPath(path)) { + std::lock_guard lock(_memoryFileMutex); + auto& memoryFileState = _memoryFiles[path]; + if (!memoryFileState) { + memoryFileState = std::make_shared(); } - std::string fileId = getCachedFileId(path); - std::shared_ptr result = std::make_shared(_session, fileId, path); + std::shared_ptr result = std::make_shared(_session, "", path, true, memoryFileState); result->open(); return result; - }); + } + std::string fileId = getCachedFileId(path); + std::shared_ptr result = std::make_shared(_session, fileId, path); + result->open(); + return result; } bool PrivmxFS::access(const std::string& path) { - return measureMethodCall("access", [&]() { - if (isJournalPath(path)) { - std::lock_guard lock(_memoryFileMutex); - return _memoryFiles.find(path) != _memoryFiles.end(); - } - LOG_TRACE("PrivmxFS::access - ", path, " | kvdbId: ",_session->kvdbId) - return _session->kvdbApi.hasEntry(_session->kvdbId, path); - }); + if (isJournalPath(path)) { + std::lock_guard lock(_memoryFileMutex); + return _memoryFiles.find(path) != _memoryFiles.end(); + } + LOG_TRACE("PrivmxFS::access - ", path, " | kvdbId: ",_session->kvdbId) + return _session->kvdbApi.hasEntry(_session->kvdbId, path); } void PrivmxFS::deleteFile(const std::string& path) { @@ -499,23 +425,21 @@ std::string PrivmxFS::getCachedFileId(const std::string& name) { } std::string PrivmxFS::getFileId(const std::string& name) { - return measureMethodCall("getFileId", [&]() { - LOG_TRACE("PrivmxFS::getFileId - ", name, " | kvdbId: ",_session->kvdbId) - try { - privmx::endpoint::kvdb::KvdbEntry kvdbEntry = _session->kvdbApi.getEntry(_session->kvdbId, name); - if(kvdbEntry.statusCode != 0) { - throw MalformedInternalFileIdException(); - } - std::string fileId = kvdbEntry.data.stdString(); - return fileId; - } catch (const privmx::endpoint::server::KvdbEntryDoesNotExistException& e) { - LOG_DEBUG("PrivmxFS::getFileId file not found, creating new file - ", name) - int64_t fh = _session->storeApi.createFile(_session->storeId, META, META, 0, true); - std::string fileId = _session->storeApi.closeFile(fh); - _session->kvdbApi.setEntry(_session->kvdbId, name, META, META, privmx::endpoint::core::Buffer::from(fileId)); - return fileId; + LOG_TRACE("PrivmxFS::getFileId - ", name, " | kvdbId: ",_session->kvdbId) + try { + privmx::endpoint::kvdb::KvdbEntry kvdbEntry = _session->kvdbApi.getEntry(_session->kvdbId, name); + if(kvdbEntry.statusCode != 0) { + throw MalformedInternalFileIdException(); } - }); + std::string fileId = kvdbEntry.data.stdString(); + return fileId; + } catch (const privmx::endpoint::server::KvdbEntryDoesNotExistException& e) { + LOG_DEBUG("PrivmxFS::getFileId file not found, creating new file - ", name) + int64_t fh = _session->storeApi.createFile(_session->storeId, META, META, 0, true); + std::string fileId = _session->storeApi.closeFile(fh); + _session->kvdbApi.setEntry(_session->kvdbId, name, META, META, privmx::endpoint::core::Buffer::from(fileId)); + return fileId; + } } std::shared_ptr PrivmxExtFS::openFile(const std::string& path) { diff --git a/endpoint/search/src/SearchApiImpl.cpp b/endpoint/search/src/SearchApiImpl.cpp index 2f524e72..7d8ab563 100644 --- a/endpoint/search/src/SearchApiImpl.cpp +++ b/endpoint/search/src/SearchApiImpl.cpp @@ -107,7 +107,6 @@ int64_t SearchApiImpl::openSearchIndex(const std::string& indexId) { void SearchApiImpl::closeSearchIndex(const int64_t indexHandle) { auto fts = _fts.get(indexHandle); fts->close(); - std::cout << PrivmxFS::getDebugStats() << std::endl; _fts.remove(indexHandle); }