From 935afacce3ca5e177a270e6296ed9f48f65dc8a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Wed, 24 Oct 2018 21:11:13 +0100 Subject: [PATCH 01/56] add plugin chain_to_mongo_plugin --- .gitmodules | 2 +- libraries/chainbase | 2 +- plugins/chain_to_mongo_plugin/CMakeLists.txt | 7 ++++ .../chain_to_mongo_plugin.cpp | 33 +++++++++++++++++ .../chain_to_mongo_plugin.hpp | 35 +++++++++++++++++++ 5 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 plugins/chain_to_mongo_plugin/CMakeLists.txt create mode 100644 plugins/chain_to_mongo_plugin/chain_to_mongo_plugin.cpp create mode 100644 plugins/chain_to_mongo_plugin/include/eosio/chain_to_mongo_plugin/chain_to_mongo_plugin.hpp diff --git a/.gitmodules b/.gitmodules index 7d0f8a37f7b..17149e7e362 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "libraries/chainbase"] path = libraries/chainbase - url = https://github.com/eosio/chainbase + url = https://github.com/mmcs85/chainbase ignore = dirty [submodule "libraries/appbase"] path = libraries/appbase diff --git a/libraries/chainbase b/libraries/chainbase index 8ca96ad6b18..eb8230d50b6 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit 8ca96ad6b18709d65a7d1f67f8893978f25babcf +Subproject commit eb8230d50b6735f3a7881579c0500254c6e3d112 diff --git a/plugins/chain_to_mongo_plugin/CMakeLists.txt b/plugins/chain_to_mongo_plugin/CMakeLists.txt new file mode 100644 index 00000000000..22b3c1af3a5 --- /dev/null +++ b/plugins/chain_to_mongo_plugin/CMakeLists.txt @@ -0,0 +1,7 @@ +file(GLOB HEADERS "include/eosio/chain_to_mongo_plugin/*.hpp") +add_library( chain_to_mongo_plugin + chain_to_mongo_plugin.cpp + ${HEADERS} ) + +target_link_libraries( chain_to_mongo_plugin http_plugin chain_plugin ) +target_include_directories( chain_to_mongo_plugin PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) diff --git a/plugins/chain_to_mongo_plugin/chain_to_mongo_plugin.cpp b/plugins/chain_to_mongo_plugin/chain_to_mongo_plugin.cpp new file mode 100644 index 00000000000..5c72061d807 --- /dev/null +++ b/plugins/chain_to_mongo_plugin/chain_to_mongo_plugin.cpp @@ -0,0 +1,33 @@ +/** + * @file + * @copyright defined in eos/LICENSE.txt + */ +#include +#include +#include + +namespace eosio { + +static appbase::abstract_plugin& _chain_to_mongo_plugin = app().register_plugin(); + +using namespace eosio; + +typedef typename get_index_type::type index_type; + +void chain_to_mongo_plugin::plugin_startup() { + const chainbase::database& db = app().get_plugin().chain().db(); + + db.get_mutable_index().applied_emplace.connect( [&]( const value_type& t ) { + elog("CHAIN_TO_MONGO emplace: ${t}", ("t", t.to_string())); + }); + + db.get_mutable_index().applied_modify.connect( [&]( const value_type& t ) { + elog("CHAIN_TO_MONGO modify: ${t}", ("t", t.to_string())); + }); + + db.get_mutable_index().applied_remove.connect( [&]( const value_type& t ) { + elog("CHAIN_TO_MONGO remove: ${t}", ("t", t.to_string())); + }); +} + +} diff --git a/plugins/chain_to_mongo_plugin/include/eosio/chain_to_mongo_plugin/chain_to_mongo_plugin.hpp b/plugins/chain_to_mongo_plugin/include/eosio/chain_to_mongo_plugin/chain_to_mongo_plugin.hpp new file mode 100644 index 00000000000..ec4501456c9 --- /dev/null +++ b/plugins/chain_to_mongo_plugin/include/eosio/chain_to_mongo_plugin/chain_to_mongo_plugin.hpp @@ -0,0 +1,35 @@ +/** + * @file + * @copyright defined in eos/LICENSE.txt + */ +#pragma once + +#include +#include + +#include + +namespace eosio { + +using namespace appbase; + +class chain_to_mongo_plugin : public plugin { +public: + APPBASE_PLUGIN_REQUIRES((http_plugin) (chain_plugin)) + + chain_to_mongo_plugin() = default; + chain_to_mongo_plugin(const chain_to_mongo_plugin&) = delete; + chain_to_mongo_plugin(chain_to_mongo_plugin&&) = delete; + chain_to_mongo_plugin& operator=(const chain_to_mongo_plugin&) = delete; + chain_to_mongo_plugin& operator=(chain_to_mongo_plugin&&) = delete; + virtual ~chain_to_mongo_plugin() override = default; + + virtual void set_program_options(options_description& cli, options_description& cfg) override {} + void plugin_initialize(const variables_map& vm) {} + void plugin_startup(); + void plugin_shutdown() {} + +private: +}; + +} From b691bb07ac20a0f08b9b17851c7e7be44c6e0554 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Fri, 26 Oct 2018 01:56:04 +0100 Subject: [PATCH 02/56] change plugin name --- plugins/statetrack_plugin/CMakeLists.txt | 7 +++ .../statetrack_plugin/statetrack_plugin.hpp | 35 +++++++++++++ .../statetrack_plugin/statetrack_plugin.cpp | 52 +++++++++++++++++++ 3 files changed, 94 insertions(+) create mode 100644 plugins/statetrack_plugin/CMakeLists.txt create mode 100644 plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp create mode 100644 plugins/statetrack_plugin/statetrack_plugin.cpp diff --git a/plugins/statetrack_plugin/CMakeLists.txt b/plugins/statetrack_plugin/CMakeLists.txt new file mode 100644 index 00000000000..fc25f1d6542 --- /dev/null +++ b/plugins/statetrack_plugin/CMakeLists.txt @@ -0,0 +1,7 @@ +file(GLOB HEADERS "include/eosio/statetrack_plugin/*.hpp") +add_library( statetrack_plugin + statetrack_plugin.cpp + ${HEADERS} ) + +target_link_libraries( statetrack_plugin http_plugin chain_plugin ) +target_include_directories( statetrack_plugin PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) diff --git a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp new file mode 100644 index 00000000000..704c7ce3554 --- /dev/null +++ b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp @@ -0,0 +1,35 @@ +/** + * @file + * @copyright defined in eos/LICENSE.txt + */ +#pragma once + +#include +#include + +#include + +namespace eosio { + +using namespace appbase; + +class statetrack_plugin : public plugin { +public: + APPBASE_PLUGIN_REQUIRES((http_plugin) (chain_plugin)) + + statetrack_plugin() = default; + statetrack_plugin(const statetrack_plugin&) = delete; + statetrack_plugin(statetrack_plugin&&) = delete; + statetrack_plugin& operator=(const statetrack_plugin&) = delete; + statetrack_plugin& operator=(statetrack_plugin&&) = delete; + virtual ~statetrack_plugin() override = default; + + virtual void set_program_options(options_description& cli, options_description& cfg) override {} + void plugin_initialize(const variables_map& vm) {} + void plugin_startup(); + void plugin_shutdown() {} + +private: +}; + +} diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp new file mode 100644 index 00000000000..e5cda855a51 --- /dev/null +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -0,0 +1,52 @@ +/** + * @file + * @copyright defined in eos/LICENSE.txt + */ +#include +#include +#include +#include + +namespace eosio { + +static appbase::abstract_plugin& _statetrack_plugin = app().register_plugin(); + +using namespace eosio; + + +typedef typename chainbase::get_index_type::type kv_index_type; +typedef typename chainbase::get_index_type::type ti_index_type; + +void statetrack_plugin::plugin_startup() { + const chainbase::database& db = app().get_plugin().chain().db(); + + auto& ti_index = db.get_index(); + + ti_index.connect_emplace( [&](const auto& tio ) { + elog("STATETRACK table_id_object emplace: ${tio}", ("tio", tio.id)); + }); + + ti_index.connect_modify( [&](const auto& tio ) { + elog("STATETRACK table_id_object modify: ${tio}", ("tio", tio.id)); + }); + + ti_index.connect_remove( [&](const auto& tio ) { + elog("STATETRACK table_id_object remove: ${tio}", ("tio", tio.id)); + }); + + auto& kv_index = db.get_index(); + + kv_index.connect_emplace( [&](const auto& kvo ) { + elog("STATETRACK kv_index_type emplace: ${kvo}", ("kvo", kvo.id)); + }); + + kv_index.connect_modify( [&](const auto& kvo ) { + elog("STATETRACK kv_index_type modify: ${kvo}", ("kvo", kvo.id)); + }); + + kv_index.connect_remove( [&](const auto& kvo ) { + elog("STATETRACK kv_index_type remove: ${kvo}", ("kvo", kvo.id)); + }); +} + +} From 176d73ccd0135fb44ffec0ba25eb5192c876d439 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Fri, 26 Oct 2018 15:56:06 +0100 Subject: [PATCH 03/56] Update statetrack_plugin.cpp --- .../statetrack_plugin/statetrack_plugin.cpp | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index e5cda855a51..ef84efa7eac 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -22,30 +22,30 @@ void statetrack_plugin::plugin_startup() { auto& ti_index = db.get_index(); - ti_index.connect_emplace( [&](const auto& tio ) { - elog("STATETRACK table_id_object emplace: ${tio}", ("tio", tio.id)); + ti_index.connect_emplace( [&](decltype(v) v ) { + elog("STATETRACK table_id_object emplace: ${v}", ("v", v.id)); }); - ti_index.connect_modify( [&](const auto& tio ) { - elog("STATETRACK table_id_object modify: ${tio}", ("tio", tio.id)); + ti_index.connect_modify( [&](decltype(v) v ) { + elog("STATETRACK table_id_object modify: ${v}", ("v", tio.id)); }); - ti_index.connect_remove( [&](const auto& tio ) { - elog("STATETRACK table_id_object remove: ${tio}", ("tio", tio.id)); + ti_index.connect_remove( [&](decltype(v) v ) { + elog("STATETRACK table_id_object remove: ${v}", ("v", tio.id)); }); auto& kv_index = db.get_index(); - kv_index.connect_emplace( [&](const auto& kvo ) { - elog("STATETRACK kv_index_type emplace: ${kvo}", ("kvo", kvo.id)); + kv_index.connect_emplace( [&](decltype(v) v ) { + elog("STATETRACK kv_index_type emplace: ${v}", ("v", kvo.id)); }); - kv_index.connect_modify( [&](const auto& kvo ) { - elog("STATETRACK kv_index_type modify: ${kvo}", ("kvo", kvo.id)); + kv_index.connect_modify( [&](decltype(v) v ) { + elog("STATETRACK kv_index_type modify: ${v}", ("v", kvo.id)); }); - kv_index.connect_remove( [&](const auto& kvo ) { - elog("STATETRACK kv_index_type remove: ${kvo}", ("kvo", kvo.id)); + kv_index.connect_remove( [&](decltype(v) v ) { + elog("STATETRACK kv_index_type remove: ${v}", ("v", kvo.id)); }); } From 800b5ae8846cb1e37f2d81ad0a157fe823c322f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Fri, 26 Oct 2018 15:59:44 +0100 Subject: [PATCH 04/56] Update statetrack_plugin.cpp --- .../statetrack_plugin/statetrack_plugin.cpp | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index ef84efa7eac..6d56579d2a8 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -22,30 +22,30 @@ void statetrack_plugin::plugin_startup() { auto& ti_index = db.get_index(); - ti_index.connect_emplace( [&](decltype(v) v ) { - elog("STATETRACK table_id_object emplace: ${v}", ("v", v.id)); + ti_index.connect_emplace( [&](decltype(tio) tio ) { + elog("STATETRACK table_id_object emplace: ${tio}", ("tio", tio.id)); }); - ti_index.connect_modify( [&](decltype(v) v ) { - elog("STATETRACK table_id_object modify: ${v}", ("v", tio.id)); + ti_index.connect_modify( [&](decltype(tio) tio ) { + elog("STATETRACK table_id_object modify: ${tio}", ("tio", tio.id)); }); - ti_index.connect_remove( [&](decltype(v) v ) { - elog("STATETRACK table_id_object remove: ${v}", ("v", tio.id)); + ti_index.connect_remove( [&](decltype(tio) tio ) { + elog("STATETRACK table_id_object remove: ${tio}", ("tio", tio.id)); }); auto& kv_index = db.get_index(); - kv_index.connect_emplace( [&](decltype(v) v ) { - elog("STATETRACK kv_index_type emplace: ${v}", ("v", kvo.id)); + kv_index.connect_emplace( [&](decltype(kvo) kvo ) { + elog("STATETRACK kv_index_type emplace: ${kvo}", ("kvo", kvo.id)); }); - kv_index.connect_modify( [&](decltype(v) v ) { - elog("STATETRACK kv_index_type modify: ${v}", ("v", kvo.id)); + kv_index.connect_modify( [&](decltype(kvo) kvo ) { + elog("STATETRACK kv_index_type modify: ${kvo}", ("kvo", kvo.id)); }); - kv_index.connect_remove( [&](decltype(v) v ) { - elog("STATETRACK kv_index_type remove: ${v}", ("v", kvo.id)); + kv_index.connect_remove( [&](decltype(kvo) kvo ) { + elog("STATETRACK kv_index_type remove: ${kvo}", ("kvo", kvo.id)); }); } From 71fd6c4e8b7f31d7eb2ce71103051ca215d93467 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Fri, 26 Oct 2018 19:36:46 +0100 Subject: [PATCH 05/56] fixes --- libraries/chainbase | 2 +- plugins/CMakeLists.txt | 1 + plugins/chain_to_mongo_plugin/CMakeLists.txt | 7 ---- .../chain_to_mongo_plugin.cpp | 33 ----------------- .../chain_to_mongo_plugin.hpp | 35 ------------------- 5 files changed, 2 insertions(+), 76 deletions(-) delete mode 100644 plugins/chain_to_mongo_plugin/CMakeLists.txt delete mode 100644 plugins/chain_to_mongo_plugin/chain_to_mongo_plugin.cpp delete mode 100644 plugins/chain_to_mongo_plugin/include/eosio/chain_to_mongo_plugin/chain_to_mongo_plugin.hpp diff --git a/libraries/chainbase b/libraries/chainbase index eb8230d50b6..d6978b9e7c7 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit eb8230d50b6735f3a7881579c0500254c6e3d112 +Subproject commit d6978b9e7c7f11381a5fe35c8ba99e579dc5a8dd diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 9b0b17b9d0a..23c1cc13eb7 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -7,6 +7,7 @@ add_subdirectory(chain_plugin) add_subdirectory(chain_api_plugin) add_subdirectory(producer_plugin) add_subdirectory(producer_api_plugin) +add_subdirectory(statetrack_plugin) add_subdirectory(history_plugin) add_subdirectory(history_api_plugin) diff --git a/plugins/chain_to_mongo_plugin/CMakeLists.txt b/plugins/chain_to_mongo_plugin/CMakeLists.txt deleted file mode 100644 index 22b3c1af3a5..00000000000 --- a/plugins/chain_to_mongo_plugin/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -file(GLOB HEADERS "include/eosio/chain_to_mongo_plugin/*.hpp") -add_library( chain_to_mongo_plugin - chain_to_mongo_plugin.cpp - ${HEADERS} ) - -target_link_libraries( chain_to_mongo_plugin http_plugin chain_plugin ) -target_include_directories( chain_to_mongo_plugin PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) diff --git a/plugins/chain_to_mongo_plugin/chain_to_mongo_plugin.cpp b/plugins/chain_to_mongo_plugin/chain_to_mongo_plugin.cpp deleted file mode 100644 index 5c72061d807..00000000000 --- a/plugins/chain_to_mongo_plugin/chain_to_mongo_plugin.cpp +++ /dev/null @@ -1,33 +0,0 @@ -/** - * @file - * @copyright defined in eos/LICENSE.txt - */ -#include -#include -#include - -namespace eosio { - -static appbase::abstract_plugin& _chain_to_mongo_plugin = app().register_plugin(); - -using namespace eosio; - -typedef typename get_index_type::type index_type; - -void chain_to_mongo_plugin::plugin_startup() { - const chainbase::database& db = app().get_plugin().chain().db(); - - db.get_mutable_index().applied_emplace.connect( [&]( const value_type& t ) { - elog("CHAIN_TO_MONGO emplace: ${t}", ("t", t.to_string())); - }); - - db.get_mutable_index().applied_modify.connect( [&]( const value_type& t ) { - elog("CHAIN_TO_MONGO modify: ${t}", ("t", t.to_string())); - }); - - db.get_mutable_index().applied_remove.connect( [&]( const value_type& t ) { - elog("CHAIN_TO_MONGO remove: ${t}", ("t", t.to_string())); - }); -} - -} diff --git a/plugins/chain_to_mongo_plugin/include/eosio/chain_to_mongo_plugin/chain_to_mongo_plugin.hpp b/plugins/chain_to_mongo_plugin/include/eosio/chain_to_mongo_plugin/chain_to_mongo_plugin.hpp deleted file mode 100644 index ec4501456c9..00000000000 --- a/plugins/chain_to_mongo_plugin/include/eosio/chain_to_mongo_plugin/chain_to_mongo_plugin.hpp +++ /dev/null @@ -1,35 +0,0 @@ -/** - * @file - * @copyright defined in eos/LICENSE.txt - */ -#pragma once - -#include -#include - -#include - -namespace eosio { - -using namespace appbase; - -class chain_to_mongo_plugin : public plugin { -public: - APPBASE_PLUGIN_REQUIRES((http_plugin) (chain_plugin)) - - chain_to_mongo_plugin() = default; - chain_to_mongo_plugin(const chain_to_mongo_plugin&) = delete; - chain_to_mongo_plugin(chain_to_mongo_plugin&&) = delete; - chain_to_mongo_plugin& operator=(const chain_to_mongo_plugin&) = delete; - chain_to_mongo_plugin& operator=(chain_to_mongo_plugin&&) = delete; - virtual ~chain_to_mongo_plugin() override = default; - - virtual void set_program_options(options_description& cli, options_description& cfg) override {} - void plugin_initialize(const variables_map& vm) {} - void plugin_startup(); - void plugin_shutdown() {} - -private: -}; - -} From cbdee7e19c86d7136057a4a98318ef25b4732199 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Sat, 27 Oct 2018 02:06:45 +0100 Subject: [PATCH 06/56] change signals usage to function callback --- .../statetrack_plugin/statetrack_plugin.hpp | 1 + .../statetrack_plugin/statetrack_plugin.cpp | 57 ++++++++++++------- 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp index 704c7ce3554..6c7e3dcb061 100644 --- a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp +++ b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp @@ -30,6 +30,7 @@ class statetrack_plugin : public plugin { void plugin_shutdown() {} private: + std::unique_ptr my; }; } diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index 6d56579d2a8..ba4653c2b78 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -2,6 +2,7 @@ * @file * @copyright defined in eos/LICENSE.txt */ + #include #include #include @@ -17,36 +18,50 @@ using namespace eosio; typedef typename chainbase::get_index_type::type kv_index_type; typedef typename chainbase::get_index_type::type ti_index_type; +typedef boost::signals2::signal st1; +typedef st1::slot_type sst1; + + +class statetrack_plugin_impl { + public: + void ti_callback(const chain::table_id_object& tio); + +}; + +void statetrack_plugin_impl::ti_callback(const chain::table_id_object& tio) { + elog("STATETRACK table_id_object emplace: ${tio}", ("tio", tio.id)); +} + void statetrack_plugin::plugin_startup() { - const chainbase::database& db = app().get_plugin().chain().db(); + const chainbase::database& db = app().get_plugin().chain().db(); - auto& ti_index = db.get_index(); + const auto& ti_index = db.get_index(); - ti_index.connect_emplace( [&](decltype(tio) tio ) { - elog("STATETRACK table_id_object emplace: ${tio}", ("tio", tio.id)); - }); + ti_index.applied_emplace = [&](const chain::table_id_object& tio) { + elog("STATETRACK table_id_object modify: ${tio}", ("tio", tio.id)); + }; - ti_index.connect_modify( [&](decltype(tio) tio ) { - elog("STATETRACK table_id_object modify: ${tio}", ("tio", tio.id)); - }); +// ti_index.connect_modify( [&](decltype(tio) tio ) { +// elog("STATETRACK table_id_object modify: ${tio}", ("tio", tio.id)); +// }); - ti_index.connect_remove( [&](decltype(tio) tio ) { - elog("STATETRACK table_id_object remove: ${tio}", ("tio", tio.id)); - }); +// ti_index.connect_remove( [&](decltype(tio) tio ) { +// elog("STATETRACK table_id_object remove: ${tio}", ("tio", tio.id)); +// }); - auto& kv_index = db.get_index(); +// auto& kv_index = db.get_index(); - kv_index.connect_emplace( [&](decltype(kvo) kvo ) { - elog("STATETRACK kv_index_type emplace: ${kvo}", ("kvo", kvo.id)); - }); +// kv_index.connect_emplace( [&](decltype(kvo) kvo ) { +// elog("STATETRACK kv_index_type emplace: ${kvo}", ("kvo", kvo.id)); +// }); - kv_index.connect_modify( [&](decltype(kvo) kvo ) { - elog("STATETRACK kv_index_type modify: ${kvo}", ("kvo", kvo.id)); - }); +// kv_index.connect_modify( [&](decltype(kvo) kvo ) { +// elog("STATETRACK kv_index_type modify: ${kvo}", ("kvo", kvo.id)); +// }); - kv_index.connect_remove( [&](decltype(kvo) kvo ) { - elog("STATETRACK kv_index_type remove: ${kvo}", ("kvo", kvo.id)); - }); +// kv_index.connect_remove( [&](decltype(kvo) kvo ) { +// elog("STATETRACK kv_index_type remove: ${kvo}", ("kvo", kvo.id)); +// }); } } From ff889dc9cde17ea6da095dc8ffb5c39815895a2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Sat, 27 Oct 2018 14:32:09 +0100 Subject: [PATCH 07/56] Update statetrack_plugin.cpp --- .../statetrack_plugin/statetrack_plugin.cpp | 66 ++++++++----------- 1 file changed, 29 insertions(+), 37 deletions(-) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index ba4653c2b78..cb500aca4c0 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -14,54 +14,46 @@ static appbase::abstract_plugin& _statetrack_plugin = app().register_plugin::type kv_index_type; -typedef typename chainbase::get_index_type::type ti_index_type; - -typedef boost::signals2::signal st1; -typedef st1::slot_type sst1; - class statetrack_plugin_impl { public: - void ti_callback(const chain::table_id_object& tio); - + abi_def get_kvo_abi( const chainbase::database& db, const chain::key_value_object& tio ); + }; -void statetrack_plugin_impl::ti_callback(const chain::table_id_object& tio) { - elog("STATETRACK table_id_object emplace: ${tio}", ("tio", tio.id)); -} +abi_def statetrack_plugin_impl::get_kvo_abi( const chainbase::database& db, const chain::key_value_object& kvo ) { + const auto &d = db.db(); + const chain::table_id_object *tio = d.find(kvo.t_id); + EOS_ASSERT(tio != nullptr, chain::account_query_exception, "Fail to retrieve table_id_object for ${t_id}", ("t_id", kvo.t_id) ); + + const account_object *co = d.find(tio->code); + EOS_ASSERT(co != nullptr, chain::account_query_exception, "Fail to retrieve account for ${account}", ("account", account) ); + + abi_def abi; + abi_serializer::to_abi(co->abi, abi); + return abi; +} + void statetrack_plugin::plugin_startup() { const chainbase::database& db = app().get_plugin().chain().db(); - - const auto& ti_index = db.get_index(); - - ti_index.applied_emplace = [&](const chain::table_id_object& tio) { - elog("STATETRACK table_id_object modify: ${tio}", ("tio", tio.id)); + + auto& kv_index = db.get_index(); + + kv_index.applied_emplace = [&](const chain::key_value_object& kvo) { + elog("STATETRACK key_value_object emplace: ${kvo}", ("kvo", kvo.id)); + + }; - -// ti_index.connect_modify( [&](decltype(tio) tio ) { -// elog("STATETRACK table_id_object modify: ${tio}", ("tio", tio.id)); -// }); - -// ti_index.connect_remove( [&](decltype(tio) tio ) { -// elog("STATETRACK table_id_object remove: ${tio}", ("tio", tio.id)); -// }); - -// auto& kv_index = db.get_index(); -// kv_index.connect_emplace( [&](decltype(kvo) kvo ) { -// elog("STATETRACK kv_index_type emplace: ${kvo}", ("kvo", kvo.id)); -// }); - -// kv_index.connect_modify( [&](decltype(kvo) kvo ) { -// elog("STATETRACK kv_index_type modify: ${kvo}", ("kvo", kvo.id)); -// }); - -// kv_index.connect_remove( [&](decltype(kvo) kvo ) { -// elog("STATETRACK kv_index_type remove: ${kvo}", ("kvo", kvo.id)); -// }); + kv_index.applied_modify = [&](const chain::key_value_object& kvo) { + elog("STATETRACK key_value_object modify: ${kvo}", ("kvo", kvo.id)); + }; + + kv_index.applied_remove = [&](const chain::key_value_object& kvo) { + elog("STATETRACK key_value_object remove: ${kvo}", ("kvo", kvo.id)); + }; } } From e496e4b27c55389314284ac609cfbd4367840268 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Sat, 27 Oct 2018 22:16:53 +0100 Subject: [PATCH 08/56] update chainbase --- libraries/chainbase | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chainbase b/libraries/chainbase index d6978b9e7c7..f88cc3c71cc 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit d6978b9e7c7f11381a5fe35c8ba99e579dc5a8dd +Subproject commit f88cc3c71cc807de98f296dcc26cef64add316a5 From 407b4c63be7239cb0c461cb1d7a0b3f6c6b626c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Sun, 28 Oct 2018 01:40:55 +0000 Subject: [PATCH 09/56] covert kvo to json --- libraries/chainbase | 2 +- plugins/statetrack_plugin/CMakeLists.txt | 4 +- .../statetrack_plugin/statetrack_plugin.hpp | 35 ++++- .../statetrack_plugin/statetrack_plugin.cpp | 146 ++++++++++++++---- programs/nodeos/CMakeLists.txt | 1 + 5 files changed, 145 insertions(+), 43 deletions(-) diff --git a/libraries/chainbase b/libraries/chainbase index f88cc3c71cc..816d36675f4 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit f88cc3c71cc807de98f296dcc26cef64add316a5 +Subproject commit 816d36675f4ac93e58b66f1e7dca8d4d710ab3a6 diff --git a/plugins/statetrack_plugin/CMakeLists.txt b/plugins/statetrack_plugin/CMakeLists.txt index fc25f1d6542..bd16f122828 100644 --- a/plugins/statetrack_plugin/CMakeLists.txt +++ b/plugins/statetrack_plugin/CMakeLists.txt @@ -3,5 +3,5 @@ add_library( statetrack_plugin statetrack_plugin.cpp ${HEADERS} ) -target_link_libraries( statetrack_plugin http_plugin chain_plugin ) -target_include_directories( statetrack_plugin PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) +target_link_libraries( statetrack_plugin chain_plugin appbase fc ) +target_include_directories( statetrack_plugin PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) \ No newline at end of file diff --git a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp index 6c7e3dcb061..7f0d9ddf05e 100644 --- a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp +++ b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp @@ -4,7 +4,9 @@ */ #pragma once -#include +#include +#include +#include #include #include @@ -13,24 +15,43 @@ namespace eosio { using namespace appbase; +enum op_type_enum { + CREATE, + MODIFY, + REMOVE +}; + +struct db_op_row { + uint64_t id; + op_type_enum op_type; + account_name code; + chain::scope_name scope; + chain::table_name table; + account_name payer; + fc::variant value; +}; + class statetrack_plugin : public plugin { public: - APPBASE_PLUGIN_REQUIRES((http_plugin) (chain_plugin)) + APPBASE_PLUGIN_REQUIRES((chain_plugin)) - statetrack_plugin() = default; + statetrack_plugin(); statetrack_plugin(const statetrack_plugin&) = delete; statetrack_plugin(statetrack_plugin&&) = delete; statetrack_plugin& operator=(const statetrack_plugin&) = delete; statetrack_plugin& operator=(statetrack_plugin&&) = delete; virtual ~statetrack_plugin() override = default; - virtual void set_program_options(options_description& cli, options_description& cfg) override {} - void plugin_initialize(const variables_map& vm) {} + virtual void set_program_options(options_description& cli, options_description& cfg) override; + void plugin_initialize(const variables_map& options); void plugin_startup(); - void plugin_shutdown() {} + void plugin_shutdown(); private: - std::unique_ptr my; + std::unique_ptr my; }; } + +FC_REFLECT_ENUM( eosio::op_type_enum, (CREATE)(MODIFY)(REMOVE) ) +FC_REFLECT( eosio::db_op_row, (id)(op_type)(code)(scope)(table)(payer)(value) ) \ No newline at end of file diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index cb500aca4c0..e30dff440b1 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -3,9 +3,6 @@ * @copyright defined in eos/LICENSE.txt */ -#include -#include -#include #include namespace eosio { @@ -14,46 +11,129 @@ static appbase::abstract_plugin& _statetrack_plugin = app().register_plugin::type kv_index_type; class statetrack_plugin_impl { public: - abi_def get_kvo_abi( const chainbase::database& db, const chain::key_value_object& tio ); - + const chain::table_id_object& get_kvo_tio(const chainbase::database& db, const chain::key_value_object& kvo); + const chain::account_object& get_tio_co(const chainbase::database& db, const chain::table_id_object& tio); + const abi_def& get_co_abi(const chain::account_object& co ); + fc::variant get_kvo_row(const chainbase::database& db, const chain::table_id_object& tio, const chain::key_value_object& kvo, bool json = true); + db_op_row get_db_op_row(const chainbase::database& db, const chain::key_value_object& kvo, op_type_enum op_type, bool json = true); + + static void copy_inline_row(const chain::key_value_object& obj, vector& data) { + data.resize( obj.value.size() ); + memcpy( data.data(), obj.value.data(), obj.value.size() ); + } + private: + fc::microseconds abi_serializer_max_time = fc::microseconds(1000 * 1000); + bool shorten_abi_errors = true; }; -abi_def statetrack_plugin_impl::get_kvo_abi( const chainbase::database& db, const chain::key_value_object& kvo ) { - const auto &d = db.db(); +// statetrack plugin implementation - const chain::table_id_object *tio = d.find(kvo.t_id); - EOS_ASSERT(tio != nullptr, chain::account_query_exception, "Fail to retrieve table_id_object for ${t_id}", ("t_id", kvo.t_id) ); - - const account_object *co = d.find(tio->code); - EOS_ASSERT(co != nullptr, chain::account_query_exception, "Fail to retrieve account for ${account}", ("account", account) ); - - abi_def abi; - abi_serializer::to_abi(co->abi, abi); - return abi; +const chain::table_id_object& statetrack_plugin_impl::get_kvo_tio(const chainbase::database& db, const chain::key_value_object& kvo) { + const chain::table_id_object *tio = db.find(kvo.t_id); + EOS_ASSERT(tio != nullptr, chain::contract_table_query_exception, "Fail to retrieve table for ${t_id}", ("t_id", kvo.t_id) ); + return *tio; } - + +const chain::account_object& statetrack_plugin_impl::get_tio_co(const chainbase::database& db, const chain::table_id_object& tio) { + const chain::account_object *co = db.find(tio.code); + EOS_ASSERT(co != nullptr, chain::account_query_exception, "Fail to retrieve account for ${code}", ("code", tio.code) ); + return *co; +} + +const abi_def& statetrack_plugin_impl::get_co_abi( const chain::account_object& co ) { + abi_def* abi = new abi_def(); + abi_serializer::to_abi(co.abi, *abi); + return *abi; +} + +fc::variant statetrack_plugin_impl::get_kvo_row(const chainbase::database& db, const chain::table_id_object& tio, const chain::key_value_object& kvo, bool json) { + vector data; + copy_inline_row(kvo, data); + + if (json) { + const chain::account_object& co = get_tio_co(db, tio); + const abi_def& abi = get_co_abi(co); + + abi_serializer abis; + abis.set_abi(abi, abi_serializer_max_time); + + return abis.binary_to_variant( abis.get_table_type(tio.table), data, abi_serializer_max_time, shorten_abi_errors ); + } else { + return fc::variant(data); + } +} + +db_op_row statetrack_plugin_impl::get_db_op_row(const chainbase::database& db, const chain::key_value_object& kvo, op_type_enum op_type, bool json) { + db_op_row row; + + const chain::table_id_object& tio = get_kvo_tio(db, kvo); + + row.id = kvo.primary_key; + row.op_type = op_type; + row.code = tio.code; + row.scope = tio.scope; + row.table = tio.table; + + if(op_type == op_type_enum::CREATE || + op_type == op_type_enum::MODIFY) { + row.value = get_kvo_row(db, tio, kvo, json); + } + + return row; +} + +// Plugin implementation + +statetrack_plugin::statetrack_plugin():my(new statetrack_plugin_impl()){} + +void statetrack_plugin::set_program_options(options_description&, options_description& cfg) { + cfg.add_options() + ("option-name", bpo::value()->default_value("default value"), + "Option Description") + ; +} + +void statetrack_plugin::plugin_initialize(const variables_map& options) { + ilog("initializing statetrack plugin"); + try { + if( options.count( "option-name" )) { + // Handle the option + } + } + FC_LOG_AND_RETHROW() +} + void statetrack_plugin::plugin_startup() { - const chainbase::database& db = app().get_plugin().chain().db(); - - auto& kv_index = db.get_index(); + try { + const chainbase::database& db = app().get_plugin().chain().db(); - kv_index.applied_emplace = [&](const chain::key_value_object& kvo) { - elog("STATETRACK key_value_object emplace: ${kvo}", ("kvo", kvo.id)); - - - }; - - kv_index.applied_modify = [&](const chain::key_value_object& kvo) { - elog("STATETRACK key_value_object modify: ${kvo}", ("kvo", kvo.id)); - }; - - kv_index.applied_remove = [&](const chain::key_value_object& kvo) { - elog("STATETRACK key_value_object remove: ${kvo}", ("kvo", kvo.id)); - }; + auto& kv_index = db.get_index(); + + kv_index.applied_emplace = [&](const chain::key_value_object& kvo) { + fc::string data = fc::json::to_string(my->get_db_op_row(db, kvo, op_type_enum::CREATE)); + ilog("STATETRACK key_value_object emplace: ${data}", ("data", data)); + }; + + kv_index.applied_modify = [&](const chain::key_value_object& kvo) { + fc::string data = fc::json::to_string(my->get_db_op_row(db, kvo, op_type_enum::MODIFY)); + ilog("STATETRACK key_value_object modify: ${data}", ("data", data)); + }; + + kv_index.applied_remove = [&](const chain::key_value_object& kvo) { + fc::string data = fc::json::to_string(my->get_db_op_row(db, kvo, op_type_enum::REMOVE)); + ilog("STATETRACK key_value_object remove: ${data}", ("data", data)); + }; + } + FC_LOG_AND_RETHROW() +} + +void statetrack_plugin::plugin_shutdown() { + ilog("statetrack plugin shutdown"); } } diff --git a/programs/nodeos/CMakeLists.txt b/programs/nodeos/CMakeLists.txt index 9e1481c23c3..e525a5b3caa 100644 --- a/programs/nodeos/CMakeLists.txt +++ b/programs/nodeos/CMakeLists.txt @@ -60,6 +60,7 @@ target_link_libraries( ${NODE_EXECUTABLE_NAME} PRIVATE -Wl,${whole_archive_flag} txn_test_gen_plugin -Wl,${no_whole_archive_flag} PRIVATE -Wl,${whole_archive_flag} db_size_api_plugin -Wl,${no_whole_archive_flag} PRIVATE -Wl,${whole_archive_flag} producer_api_plugin -Wl,${no_whole_archive_flag} + PRIVATE -Wl,${whole_archive_flag} statetrack_plugin -Wl,${no_whole_archive_flag} PRIVATE -Wl,${whole_archive_flag} test_control_plugin -Wl,${no_whole_archive_flag} PRIVATE -Wl,${whole_archive_flag} test_control_api_plugin -Wl,${no_whole_archive_flag} PRIVATE -Wl,${build_id_flag} From b49720fc65cfc81cd727104b8d2e1879c1acbaf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Sun, 28 Oct 2018 10:35:35 +0000 Subject: [PATCH 10/56] fixes to scope --- .../statetrack_plugin/statetrack_plugin.hpp | 6 ++++-- .../statetrack_plugin/statetrack_plugin.cpp | 21 +++++++++++++++++-- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp index 7f0d9ddf05e..a78a41bfc44 100644 --- a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp +++ b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp @@ -15,6 +15,8 @@ namespace eosio { using namespace appbase; +typedef chain::key_value_object::id_type kvo_id_type; + enum op_type_enum { CREATE, MODIFY, @@ -22,10 +24,10 @@ enum op_type_enum { }; struct db_op_row { - uint64_t id; + kvo_id_type id; op_type_enum op_type; account_name code; - chain::scope_name scope; + fc::string scope; chain::table_name table; account_name payer; fc::variant value; diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index e30dff440b1..be8a0af4eb6 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -73,10 +73,27 @@ db_op_row statetrack_plugin_impl::get_db_op_row(const chainbase::database& db, c const chain::table_id_object& tio = get_kvo_tio(db, kvo); - row.id = kvo.primary_key; + row.id = kvo.id; row.op_type = op_type; row.code = tio.code; - row.scope = tio.scope; + + std::string scope = tio.scope.to_string(); + + if(scope.length() > 0 && scope[0] == '.') { + vector v; + uint64_t sym = tio.scope; + for( int i = 0; i < 7; ++i ) { + char c = (char)(sym & 0xff); + if( !c ) break; + v.emplace_back(c); + sym >>= 8; + } + row.scope = std::string(v.begin(),v.end()); + } + else { + row.scope = scope; + } + row.table = tio.table; if(op_type == op_type_enum::CREATE || From 4cf47729322470c2e600c757b6fde31100e27218 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Mon, 29 Oct 2018 11:36:07 +0000 Subject: [PATCH 11/56] Update statetrack_plugin.cpp --- .../statetrack_plugin/statetrack_plugin.cpp | 49 ++++++++++++++----- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index be8a0af4eb6..fbe2231713c 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -20,12 +20,13 @@ class statetrack_plugin_impl { const chain::account_object& get_tio_co(const chainbase::database& db, const chain::table_id_object& tio); const abi_def& get_co_abi(const chain::account_object& co ); fc::variant get_kvo_row(const chainbase::database& db, const chain::table_id_object& tio, const chain::key_value_object& kvo, bool json = true); - db_op_row get_db_op_row(const chainbase::database& db, const chain::key_value_object& kvo, op_type_enum op_type, bool json = true); + db_op get_db_op(const chainbase::database& db, const chain::key_value_object& kvo, op_type_enum op_type, bool json = true); + db_rev get_db_rev(const int64_t revision, op_type_enum op_type); static void copy_inline_row(const chain::key_value_object& obj, vector& data) { data.resize( obj.value.size() ); memcpy( data.data(), obj.value.data(), obj.value.size() ); - } + } private: fc::microseconds abi_serializer_max_time = fc::microseconds(1000 * 1000); bool shorten_abi_errors = true; @@ -68,14 +69,14 @@ fc::variant statetrack_plugin_impl::get_kvo_row(const chainbase::database& db, c } } -db_op_row statetrack_plugin_impl::get_db_op_row(const chainbase::database& db, const chain::key_value_object& kvo, op_type_enum op_type, bool json) { - db_op_row row; +db_op statetrack_plugin_impl::get_db_op(const chainbase::database& db, const chain::key_value_object& kvo, op_type_enum op_type, bool json) { + db_op op; const chain::table_id_object& tio = get_kvo_tio(db, kvo); - row.id = kvo.id; - row.op_type = op_type; - row.code = tio.code; + op.id = kvo.id; + op.op_type = op_type; + op.code = tio.code; std::string scope = tio.scope.to_string(); @@ -88,20 +89,27 @@ db_op_row statetrack_plugin_impl::get_db_op_row(const chainbase::database& db, c v.emplace_back(c); sym >>= 8; } - row.scope = std::string(v.begin(),v.end()); + op.scope = std::string(v.begin(),v.end()); } else { - row.scope = scope; + op.scope = scope; } - row.table = tio.table; + op.table = tio.table; if(op_type == op_type_enum::CREATE || op_type == op_type_enum::MODIFY) { - row.value = get_kvo_row(db, tio, kvo, json); + op.value = get_kvo_row(db, tio, kvo, json); } - return row; + return op; +} + +db_rev statetrack_plugin_impl::get_db_rev(const int64_t revision, op_type_enum op_type) { + db_rev rev; + rev.op_type = op_type; + rev.revision = revision; + return rev; } // Plugin implementation @@ -142,9 +150,24 @@ void statetrack_plugin::plugin_startup() { }; kv_index.applied_remove = [&](const chain::key_value_object& kvo) { - fc::string data = fc::json::to_string(my->get_db_op_row(db, kvo, op_type_enum::REMOVE)); + fc::string data = fc::json::to_string(my->get_db_op(db, kvo, op_type_enum::REMOVE)); ilog("STATETRACK key_value_object remove: ${data}", ("data", data)); }; + + kv_index.applied_undo = [&](const int64_t revision) { + fc::string data = fc::json::to_string(my->get_db_rev(revision, op_type_enum::UNDO)); + ilog("STATETRACK undo: ${data}", ("data", data)); + }; + + kv_index.applied_squash = [&](const int64_t revision) { + fc::string data = fc::json::to_string(my->get_db_rev(revision, op_type_enum::SQUASH)); + ilog("STATETRACK squash: ${data}", ("data", data)); + }; + + kv_index.applied_commit = [&](const int64_t revision) { + fc::string data = fc::json::to_string(my->get_db_rev(revision, op_type_enum::COMMIT)); + ilog("STATETRACK commit: ${data}", ("data", data)); + }; } FC_LOG_AND_RETHROW() } From 7ab5e1a0652327c7cac8773645e8f0417915c21f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Mon, 29 Oct 2018 11:37:44 +0000 Subject: [PATCH 12/56] Update statetrack_plugin.hpp --- .../eosio/statetrack_plugin/statetrack_plugin.hpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp index a78a41bfc44..9983dd1a14f 100644 --- a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp +++ b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp @@ -20,10 +20,13 @@ typedef chain::key_value_object::id_type kvo_id_type; enum op_type_enum { CREATE, MODIFY, - REMOVE + REMOVE, + UNDO, + SQUASH, + COMMIT }; -struct db_op_row { +struct db_op { kvo_id_type id; op_type_enum op_type; account_name code; @@ -32,6 +35,11 @@ struct db_op_row { account_name payer; fc::variant value; }; + +struct db_rev { + op_type_enum op_type; + int64_t revision; +}; class statetrack_plugin : public plugin { public: @@ -56,4 +64,5 @@ class statetrack_plugin : public plugin { } FC_REFLECT_ENUM( eosio::op_type_enum, (CREATE)(MODIFY)(REMOVE) ) -FC_REFLECT( eosio::db_op_row, (id)(op_type)(code)(scope)(table)(payer)(value) ) \ No newline at end of file +FC_REFLECT( eosio::db_op, (id)(op_type)(code)(scope)(table)(payer)(value) ) +FC_REFLECT( eosio::db_rev, (op_type)(revision) ) From abfb5d17f14c4cf644d97d1032e512607b3b0ccd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Mon, 29 Oct 2018 14:35:07 +0000 Subject: [PATCH 13/56] Update statetrack_plugin.cpp --- .../statetrack_plugin/statetrack_plugin.cpp | 140 +++++++++++------- 1 file changed, 89 insertions(+), 51 deletions(-) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index fbe2231713c..8b6c16a36a0 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -5,6 +5,11 @@ #include +namespace { + const char* SENDER_BIND = "zmq-sender-bind"; + const char* SENDER_BIND_DEFAULT = "tcp://127.0.0.1:5556"; +} + namespace eosio { static appbase::abstract_plugin& _statetrack_plugin = app().register_plugin(); @@ -16,6 +21,12 @@ typedef typename chainbase::get_index_type::type kv_ind class statetrack_plugin_impl { public: + statetrack_plugin_impl(): + context(1), + sender_socket(context, ZMQ_PUSH) {} + + void send_smq_msg(const string content); + const chain::table_id_object& get_kvo_tio(const chainbase::database& db, const chain::key_value_object& kvo); const chain::account_object& get_tio_co(const chainbase::database& db, const chain::table_id_object& tio); const abi_def& get_co_abi(const chain::account_object& co ); @@ -27,13 +38,34 @@ class statetrack_plugin_impl { data.resize( obj.value.size() ); memcpy( data.data(), obj.value.data(), obj.value.size() ); } + + static std::string scope_sym_to_string(uint64_t sym_code) { + vector v; + for( int i = 0; i < 7; ++i ) { + char c = (char)(sym_code & 0xff); + if( !c ) break; + v.emplace_back(c); + sym_code >>= 8; + } + return std::string(v.begin(),v.end()); + } private: - fc::microseconds abi_serializer_max_time = fc::microseconds(1000 * 1000); + zmq::context_t context; + zmq::socket_t sender_socket; + fc::string socket_bind_str; + fc::microseconds abi_serializer_max_time; bool shorten_abi_errors = true; }; // statetrack plugin implementation +void statetrack_plugin_impl::send_smq_msg(const fc::string content) { + zmq::message_t message(content.length()); + unsigned char* ptr = (unsigned char*) message.data(); + memcpy(ptr, content.c_str(), content.length()); + sender_socket.send(message); +} + const chain::table_id_object& statetrack_plugin_impl::get_kvo_tio(const chainbase::database& db, const chain::key_value_object& kvo) { const chain::table_id_object *tio = db.find(kvo.t_id); EOS_ASSERT(tio != nullptr, chain::contract_table_query_exception, "Fail to retrieve table for ${t_id}", ("t_id", kvo.t_id) ); @@ -81,15 +113,7 @@ db_op statetrack_plugin_impl::get_db_op(const chainbase::database& db, const cha std::string scope = tio.scope.to_string(); if(scope.length() > 0 && scope[0] == '.') { - vector v; - uint64_t sym = tio.scope; - for( int i = 0; i < 7; ++i ) { - char c = (char)(sym & 0xff); - if( !c ) break; - v.emplace_back(c); - sym >>= 8; - } - op.scope = std::string(v.begin(),v.end()); + op.scope = scope_sym_to_string(tio.scope); } else { op.scope = scope; @@ -118,62 +142,76 @@ statetrack_plugin::statetrack_plugin():my(new statetrack_plugin_impl()){} void statetrack_plugin::set_program_options(options_description&, options_description& cfg) { cfg.add_options() - ("option-name", bpo::value()->default_value("default value"), - "Option Description") - ; + (SENDER_BIND, bpo::value()->default_value(SENDER_BIND_DEFAULT), + "ZMQ Sender Socket binding") + ; } void statetrack_plugin::plugin_initialize(const variables_map& options) { ilog("initializing statetrack plugin"); + try { - if( options.count( "option-name" )) { - // Handle the option + my->socket_bind_str = options.at(SENDER_BIND).as(); + if (my->socket_bind_str.empty()) { + wlog("zmq-sender-bind not specified => eosio::statetrack_plugin disabled."); + return; } + + ilog("Binding to ZMQ PUSH socket ${u}", ("u", my->socket_bind_str)); + my->sender_socket.bind(my->socket_bind_str); + + chain_plugin& chain_plug = app().get_plugin(); + const chainbase::database& db = chain_plug.chain().db(); + + my->abi_serializer_max_time = chain_plug->get_abi_serializer_max_time(); + + ilog("Binding database events"); + + auto& kv_index = db.get_index(); + + kv_index.applied_emplace = [&](const chain::key_value_object& kvo) { + fc::string data = fc::json::to_string(my->get_db_op_row(db, kvo, op_type_enum::CREATE)); + ilog("STATETRACK key_value_object emplace: ${data}", ("data", data)); + }; + + kv_index.applied_modify = [&](const chain::key_value_object& kvo) { + fc::string data = fc::json::to_string(my->get_db_op_row(db, kvo, op_type_enum::MODIFY)); + ilog("STATETRACK key_value_object modify: ${data}", ("data", data)); + }; + + kv_index.applied_remove = [&](const chain::key_value_object& kvo) { + fc::string data = fc::json::to_string(my->get_db_op(db, kvo, op_type_enum::REMOVE)); + ilog("STATETRACK key_value_object remove: ${data}", ("data", data)); + }; + + kv_index.applied_undo = [&](const int64_t revision) { + fc::string data = fc::json::to_string(my->get_db_rev(revision, op_type_enum::UNDO)); + ilog("STATETRACK undo: ${data}", ("data", data)); + }; + + kv_index.applied_squash = [&](const int64_t revision) { + fc::string data = fc::json::to_string(my->get_db_rev(revision, op_type_enum::SQUASH)); + ilog("STATETRACK squash: ${data}", ("data", data)); + }; + + kv_index.applied_commit = [&](const int64_t revision) { + fc::string data = fc::json::to_string(my->get_db_rev(revision, op_type_enum::COMMIT)); + ilog("STATETRACK commit: ${data}", ("data", data)); + }; } FC_LOG_AND_RETHROW() } void statetrack_plugin::plugin_startup() { - try { - const chainbase::database& db = app().get_plugin().chain().db(); - - auto& kv_index = db.get_index(); - - kv_index.applied_emplace = [&](const chain::key_value_object& kvo) { - fc::string data = fc::json::to_string(my->get_db_op_row(db, kvo, op_type_enum::CREATE)); - ilog("STATETRACK key_value_object emplace: ${data}", ("data", data)); - }; - - kv_index.applied_modify = [&](const chain::key_value_object& kvo) { - fc::string data = fc::json::to_string(my->get_db_op_row(db, kvo, op_type_enum::MODIFY)); - ilog("STATETRACK key_value_object modify: ${data}", ("data", data)); - }; - - kv_index.applied_remove = [&](const chain::key_value_object& kvo) { - fc::string data = fc::json::to_string(my->get_db_op(db, kvo, op_type_enum::REMOVE)); - ilog("STATETRACK key_value_object remove: ${data}", ("data", data)); - }; - - kv_index.applied_undo = [&](const int64_t revision) { - fc::string data = fc::json::to_string(my->get_db_rev(revision, op_type_enum::UNDO)); - ilog("STATETRACK undo: ${data}", ("data", data)); - }; - - kv_index.applied_squash = [&](const int64_t revision) { - fc::string data = fc::json::to_string(my->get_db_rev(revision, op_type_enum::SQUASH)); - ilog("STATETRACK squash: ${data}", ("data", data)); - }; - - kv_index.applied_commit = [&](const int64_t revision) { - fc::string data = fc::json::to_string(my->get_db_rev(revision, op_type_enum::COMMIT)); - ilog("STATETRACK commit: ${data}", ("data", data)); - }; - } - FC_LOG_AND_RETHROW() + } void statetrack_plugin::plugin_shutdown() { ilog("statetrack plugin shutdown"); + if( ! my->socket_bind_str.empty() ) { + my->sender_socket.disconnect(my->socket_bind_str); + my->sender_socket.close(); + } } } From 4ff38be731cb4cbfee90a0ddc03cbd08b6d54c7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Mon, 29 Oct 2018 14:37:13 +0000 Subject: [PATCH 14/56] Update statetrack_plugin.cpp --- plugins/statetrack_plugin/statetrack_plugin.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index 8b6c16a36a0..71c8e49a555 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -3,6 +3,7 @@ * @copyright defined in eos/LICENSE.txt */ +#include #include namespace { From f6c501d94877c364e5a85c7d87d97975b1ff90db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Mon, 29 Oct 2018 14:46:36 +0000 Subject: [PATCH 15/56] Update statetrack_plugin.cpp --- plugins/statetrack_plugin/statetrack_plugin.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index 71c8e49a555..7afe2862e74 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -26,7 +26,7 @@ class statetrack_plugin_impl { context(1), sender_socket(context, ZMQ_PUSH) {} - void send_smq_msg(const string content); + void send_zmq_msg(const string content); const chain::table_id_object& get_kvo_tio(const chainbase::database& db, const chain::key_value_object& kvo); const chain::account_object& get_tio_co(const chainbase::database& db, const chain::table_id_object& tio); @@ -60,7 +60,7 @@ class statetrack_plugin_impl { // statetrack plugin implementation -void statetrack_plugin_impl::send_smq_msg(const fc::string content) { +void statetrack_plugin_impl::send_zmq_msg(const fc::string content) { zmq::message_t message(content.length()); unsigned char* ptr = (unsigned char*) message.data(); memcpy(ptr, content.c_str(), content.length()); @@ -173,31 +173,37 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { kv_index.applied_emplace = [&](const chain::key_value_object& kvo) { fc::string data = fc::json::to_string(my->get_db_op_row(db, kvo, op_type_enum::CREATE)); ilog("STATETRACK key_value_object emplace: ${data}", ("data", data)); + my->send_zmq_msg(data); }; kv_index.applied_modify = [&](const chain::key_value_object& kvo) { fc::string data = fc::json::to_string(my->get_db_op_row(db, kvo, op_type_enum::MODIFY)); ilog("STATETRACK key_value_object modify: ${data}", ("data", data)); + my->send_zmq_msg(data); }; kv_index.applied_remove = [&](const chain::key_value_object& kvo) { fc::string data = fc::json::to_string(my->get_db_op(db, kvo, op_type_enum::REMOVE)); ilog("STATETRACK key_value_object remove: ${data}", ("data", data)); + my->send_zmq_msg(data); }; kv_index.applied_undo = [&](const int64_t revision) { fc::string data = fc::json::to_string(my->get_db_rev(revision, op_type_enum::UNDO)); ilog("STATETRACK undo: ${data}", ("data", data)); + my->send_zmq_msg(data); }; kv_index.applied_squash = [&](const int64_t revision) { fc::string data = fc::json::to_string(my->get_db_rev(revision, op_type_enum::SQUASH)); ilog("STATETRACK squash: ${data}", ("data", data)); + my->send_zmq_msg(data); }; kv_index.applied_commit = [&](const int64_t revision) { fc::string data = fc::json::to_string(my->get_db_rev(revision, op_type_enum::COMMIT)); ilog("STATETRACK commit: ${data}", ("data", data)); + my->send_zmq_msg(data); }; } FC_LOG_AND_RETHROW() From 85dab8f289c5418cfd92fa20e11bba2d6e7a11ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Mon, 29 Oct 2018 17:17:15 +0000 Subject: [PATCH 16/56] Update statetrack_plugin.cpp --- plugins/statetrack_plugin/statetrack_plugin.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index 7afe2862e74..77a6377586c 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -7,7 +7,7 @@ #include namespace { - const char* SENDER_BIND = "zmq-sender-bind"; + const char* SENDER_BIND = "st-zmq-sender-bind"; const char* SENDER_BIND_DEFAULT = "tcp://127.0.0.1:5556"; } @@ -146,6 +146,14 @@ void statetrack_plugin::set_program_options(options_description&, options_descri (SENDER_BIND, bpo::value()->default_value(SENDER_BIND_DEFAULT), "ZMQ Sender Socket binding") ; + cfg.add_options() + ("st-filter-on,f", bpo::value>()->composing(), + "Track tables which match code:scope:table.") + ; + cfg.add_options() + ("st-filter-out,F", bpo::value>()->composing(), + "Do not track tables which match code:scope:table.") + ; } void statetrack_plugin::plugin_initialize(const variables_map& options) { From 032615dbdaf5f459b98b51d96f7af12e792fd580 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Mon, 29 Oct 2018 20:16:38 +0000 Subject: [PATCH 17/56] update chainbase --- libraries/chainbase | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chainbase b/libraries/chainbase index 816d36675f4..8529e29856c 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit 816d36675f4ac93e58b66f1e7dca8d4d710ab3a6 +Subproject commit 8529e29856c23aee76a4092a404781371cd5226e From f3a9a80834c92e69b9802b4c91c5403e89f337c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Mon, 29 Oct 2018 23:34:04 +0000 Subject: [PATCH 18/56] fixes --- libraries/chainbase | 2 +- plugins/statetrack_plugin/CMakeLists.txt | 23 +++- .../statetrack_plugin/statetrack_plugin.hpp | 14 +-- .../statetrack_plugin/statetrack_plugin.cpp | 104 ++++++++++-------- 4 files changed, 86 insertions(+), 57 deletions(-) diff --git a/libraries/chainbase b/libraries/chainbase index 8529e29856c..2e29ae44d1a 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit 8529e29856c23aee76a4092a404781371cd5226e +Subproject commit 2e29ae44d1a43dc5eba15f123b40678cf2b4a896 diff --git a/plugins/statetrack_plugin/CMakeLists.txt b/plugins/statetrack_plugin/CMakeLists.txt index bd16f122828..8bfdca2d36c 100644 --- a/plugins/statetrack_plugin/CMakeLists.txt +++ b/plugins/statetrack_plugin/CMakeLists.txt @@ -1,7 +1,28 @@ +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules") + +## load in pkg-config support +find_package(PkgConfig REQUIRED) +## use pkg-config to get hints for 0mq locations +pkg_check_modules(PC_ZeroMQ REQUIRED libzmq) + +## use the hint from above to find where 'zmq.hpp' is located +find_path(ZeroMQ_INCLUDE_DIR + NAMES zmq.hpp + PATHS ${PC_ZeroMQ_INCLUDE_DIRS} +) + +## use the hint from about to find the location of libzmq +find_library(ZeroMQ_LIBRARY + NAMES zmq + PATHS ${PC_ZeroMQ_LIBRARY_DIRS} + ) + +message(STATUS "[Additional Plugin] EOSIO ZeroMQ plugin enabled") + file(GLOB HEADERS "include/eosio/statetrack_plugin/*.hpp") add_library( statetrack_plugin statetrack_plugin.cpp ${HEADERS} ) -target_link_libraries( statetrack_plugin chain_plugin appbase fc ) +target_link_libraries( statetrack_plugin ${ZeroMQ_LIBRARY} chain_plugin appbase fc ) target_include_directories( statetrack_plugin PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) \ No newline at end of file diff --git a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp index 9983dd1a14f..94143ba07b7 100644 --- a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp +++ b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp @@ -18,12 +18,12 @@ using namespace appbase; typedef chain::key_value_object::id_type kvo_id_type; enum op_type_enum { - CREATE, - MODIFY, - REMOVE, - UNDO, - SQUASH, - COMMIT + CREATE = 0, + MODIFY = 1, + REMOVE = 2, + UNDO = 3, + SQUASH = 4, + COMMIT = 5 }; struct db_op { @@ -63,6 +63,6 @@ class statetrack_plugin : public plugin { } -FC_REFLECT_ENUM( eosio::op_type_enum, (CREATE)(MODIFY)(REMOVE) ) +FC_REFLECT_ENUM( eosio::op_type_enum, (CREATE)(MODIFY)(REMOVE)(UNDO)(SQUASH)(COMMIT) ) FC_REFLECT( eosio::db_op, (id)(op_type)(code)(scope)(table)(payer)(value) ) FC_REFLECT( eosio::db_rev, (op_type)(revision) ) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index 77a6377586c..acf18ee40b1 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -8,7 +8,7 @@ namespace { const char* SENDER_BIND = "st-zmq-sender-bind"; - const char* SENDER_BIND_DEFAULT = "tcp://127.0.0.1:5556"; + const char* SENDER_BIND_DEFAULT = "tcp://127.0.0.1:3000"; } namespace eosio { @@ -22,39 +22,46 @@ typedef typename chainbase::get_index_type::type kv_ind class statetrack_plugin_impl { public: - statetrack_plugin_impl(): - context(1), - sender_socket(context, ZMQ_PUSH) {} - - void send_zmq_msg(const string content); - - const chain::table_id_object& get_kvo_tio(const chainbase::database& db, const chain::key_value_object& kvo); - const chain::account_object& get_tio_co(const chainbase::database& db, const chain::table_id_object& tio); - const abi_def& get_co_abi(const chain::account_object& co ); - fc::variant get_kvo_row(const chainbase::database& db, const chain::table_id_object& tio, const chain::key_value_object& kvo, bool json = true); - db_op get_db_op(const chainbase::database& db, const chain::key_value_object& kvo, op_type_enum op_type, bool json = true); - db_rev get_db_rev(const int64_t revision, op_type_enum op_type); - - static void copy_inline_row(const chain::key_value_object& obj, vector& data) { - data.resize( obj.value.size() ); - memcpy( data.data(), obj.value.data(), obj.value.size() ); - } - - static std::string scope_sym_to_string(uint64_t sym_code) { - vector v; - for( int i = 0; i < 7; ++i ) { - char c = (char)(sym_code & 0xff); - if( !c ) break; - v.emplace_back(c); - sym_code >>= 8; + statetrack_plugin_impl() { + context = new zmq::context_t(1); + sender_socket = new zmq::socket_t(*context, ZMQ_PUSH); } - return std::string(v.begin(),v.end()); - } - private: - zmq::context_t context; - zmq::socket_t sender_socket; + + ~statetrack_plugin_impl() { + delete sender_socket; + delete context; + } + + void send_zmq_msg(const string content); + + const chain::table_id_object& get_kvo_tio(const chainbase::database& db, const chain::key_value_object& kvo); + const chain::account_object& get_tio_co(const chainbase::database& db, const chain::table_id_object& tio); + const abi_def& get_co_abi(const chain::account_object& co ); + fc::variant get_kvo_row(const chainbase::database& db, const chain::table_id_object& tio, const chain::key_value_object& kvo, bool json = true); + db_op get_db_op(const chainbase::database& db, const chain::key_value_object& kvo, op_type_enum op_type, bool json = true); + db_rev get_db_rev(const int64_t revision, op_type_enum op_type); + + static void copy_inline_row(const chain::key_value_object& obj, vector& data) { + data.resize( obj.value.size() ); + memcpy( data.data(), obj.value.data(), obj.value.size() ); + } + + static fc::string scope_sym_to_string(uint64_t sym_code) { + vector v; + for( int i = 0; i < 7; ++i ) { + char c = (char)(sym_code & 0xff); + if( !c ) break; + v.emplace_back(c); + sym_code >>= 8; + } + return fc::string(v.begin(),v.end()); + } + fc::string socket_bind_str; - fc::microseconds abi_serializer_max_time; + zmq::context_t* context; + zmq::socket_t* sender_socket; + fc::microseconds abi_serializer_max_time; + private: bool shorten_abi_errors = true; }; @@ -62,9 +69,8 @@ class statetrack_plugin_impl { void statetrack_plugin_impl::send_zmq_msg(const fc::string content) { zmq::message_t message(content.length()); - unsigned char* ptr = (unsigned char*) message.data(); - memcpy(ptr, content.c_str(), content.length()); - sender_socket.send(message); + memcpy(message.data(), content.c_str(), content.length()); + sender_socket->send(message); } const chain::table_id_object& statetrack_plugin_impl::get_kvo_tio(const chainbase::database& db, const chain::key_value_object& kvo) { @@ -111,7 +117,7 @@ db_op statetrack_plugin_impl::get_db_op(const chainbase::database& db, const cha op.op_type = op_type; op.code = tio.code; - std::string scope = tio.scope.to_string(); + fc::string scope = tio.scope.to_string(); if(scope.length() > 0 && scope[0] == '.') { op.scope = scope_sym_to_string(tio.scope); @@ -160,32 +166,34 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { ilog("initializing statetrack plugin"); try { - my->socket_bind_str = options.at(SENDER_BIND).as(); - if (my->socket_bind_str.empty()) { - wlog("zmq-sender-bind not specified => eosio::statetrack_plugin disabled."); - return; - } + my->socket_bind_str = options.at(SENDER_BIND).as(); + if (my->socket_bind_str.empty()) { + wlog("zmq-sender-bind not specified => eosio::statetrack_plugin disabled."); + return; + } - ilog("Binding to ZMQ PUSH socket ${u}", ("u", my->socket_bind_str)); - my->sender_socket.bind(my->socket_bind_str); + ilog("Bind to ZMQ PUSH socket ${u}", ("u", my->socket_bind_str)); + my->sender_socket->bind(my->socket_bind_str); + + ilog("Bind to ZMQ PUSH socket successful"); chain_plugin& chain_plug = app().get_plugin(); const chainbase::database& db = chain_plug.chain().db(); - my->abi_serializer_max_time = chain_plug->get_abi_serializer_max_time(); + my->abi_serializer_max_time = chain_plug.get_abi_serializer_max_time(); ilog("Binding database events"); auto& kv_index = db.get_index(); kv_index.applied_emplace = [&](const chain::key_value_object& kvo) { - fc::string data = fc::json::to_string(my->get_db_op_row(db, kvo, op_type_enum::CREATE)); + fc::string data = fc::json::to_string(my->get_db_op(db, kvo, op_type_enum::CREATE)); ilog("STATETRACK key_value_object emplace: ${data}", ("data", data)); my->send_zmq_msg(data); }; kv_index.applied_modify = [&](const chain::key_value_object& kvo) { - fc::string data = fc::json::to_string(my->get_db_op_row(db, kvo, op_type_enum::MODIFY)); + fc::string data = fc::json::to_string(my->get_db_op(db, kvo, op_type_enum::MODIFY)); ilog("STATETRACK key_value_object modify: ${data}", ("data", data)); my->send_zmq_msg(data); }; @@ -224,8 +232,8 @@ void statetrack_plugin::plugin_startup() { void statetrack_plugin::plugin_shutdown() { ilog("statetrack plugin shutdown"); if( ! my->socket_bind_str.empty() ) { - my->sender_socket.disconnect(my->socket_bind_str); - my->sender_socket.close(); + my->sender_socket->disconnect(my->socket_bind_str); + my->sender_socket->close(); } } From ce1329a0e1abe371c2c47798cd9d71e6ed45a2e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Tue, 30 Oct 2018 12:05:36 +0000 Subject: [PATCH 19/56] Update statetrack_plugin.cpp --- plugins/statetrack_plugin/statetrack_plugin.cpp | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index acf18ee40b1..0b1fa426244 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -22,16 +22,6 @@ typedef typename chainbase::get_index_type::type kv_ind class statetrack_plugin_impl { public: - statetrack_plugin_impl() { - context = new zmq::context_t(1); - sender_socket = new zmq::socket_t(*context, ZMQ_PUSH); - } - - ~statetrack_plugin_impl() { - delete sender_socket; - delete context; - } - void send_zmq_msg(const string content); const chain::table_id_object& get_kvo_tio(const chainbase::database& db, const chain::key_value_object& kvo); @@ -166,6 +156,9 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { ilog("initializing statetrack plugin"); try { + my->context = new zmq::context_t(1); + my->sender_socket = new zmq::socket_t(*my->context, ZMQ_PUSH); + my->socket_bind_str = options.at(SENDER_BIND).as(); if (my->socket_bind_str.empty()) { wlog("zmq-sender-bind not specified => eosio::statetrack_plugin disabled."); @@ -234,6 +227,8 @@ void statetrack_plugin::plugin_shutdown() { if( ! my->socket_bind_str.empty() ) { my->sender_socket->disconnect(my->socket_bind_str); my->sender_socket->close(); + delete my->sender_socket; + delete my->context; } } From 8145b9ebfc7bd3a20a4bea4faadfb6a1c170dcb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Tue, 30 Oct 2018 15:26:57 +0000 Subject: [PATCH 20/56] Update statetrack_plugin.cpp --- plugins/statetrack_plugin/statetrack_plugin.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index 0b1fa426244..ef9825d8b4c 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -203,11 +203,11 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { my->send_zmq_msg(data); }; - kv_index.applied_squash = [&](const int64_t revision) { + /*kv_index.applied_squash = [&](const int64_t revision) { fc::string data = fc::json::to_string(my->get_db_rev(revision, op_type_enum::SQUASH)); ilog("STATETRACK squash: ${data}", ("data", data)); my->send_zmq_msg(data); - }; + };*/ kv_index.applied_commit = [&](const int64_t revision) { fc::string data = fc::json::to_string(my->get_db_rev(revision, op_type_enum::COMMIT)); @@ -227,9 +227,9 @@ void statetrack_plugin::plugin_shutdown() { if( ! my->socket_bind_str.empty() ) { my->sender_socket->disconnect(my->socket_bind_str); my->sender_socket->close(); - delete my->sender_socket; - delete my->context; } + delete my->sender_socket; + delete my->context; } } From e4a300a8f1413e3e97326118970417440e689dfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Tue, 30 Oct 2018 15:28:33 +0000 Subject: [PATCH 21/56] Update statetrack_plugin.cpp --- .../statetrack_plugin/statetrack_plugin.cpp | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index ef9825d8b4c..d799bacde51 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -156,17 +156,17 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { ilog("initializing statetrack plugin"); try { - my->context = new zmq::context_t(1); - my->sender_socket = new zmq::socket_t(*my->context, ZMQ_PUSH); + my->socket_bind_str = options.at(SENDER_BIND).as(); + if (my->socket_bind_str.empty()) { + wlog("zmq-sender-bind not specified => eosio::statetrack_plugin disabled."); + return; + } - my->socket_bind_str = options.at(SENDER_BIND).as(); - if (my->socket_bind_str.empty()) { - wlog("zmq-sender-bind not specified => eosio::statetrack_plugin disabled."); - return; - } + my->context = new zmq::context_t(1); + my->sender_socket = new zmq::socket_t(*my->context, ZMQ_PUSH); ilog("Bind to ZMQ PUSH socket ${u}", ("u", my->socket_bind_str)); - my->sender_socket->bind(my->socket_bind_str); + my->sender_socket->connect(my->socket_bind_str); ilog("Bind to ZMQ PUSH socket successful"); @@ -227,9 +227,9 @@ void statetrack_plugin::plugin_shutdown() { if( ! my->socket_bind_str.empty() ) { my->sender_socket->disconnect(my->socket_bind_str); my->sender_socket->close(); + delete my->sender_socket; + delete my->context; } - delete my->sender_socket; - delete my->context; } } From 89b5727551bdb55b3b439e77046d3d76661ecdc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Tue, 30 Oct 2018 15:41:52 +0000 Subject: [PATCH 22/56] Create README.md --- plugins/statetrack_plugin/README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 plugins/statetrack_plugin/README.md diff --git a/plugins/statetrack_plugin/README.md b/plugins/statetrack_plugin/README.md new file mode 100644 index 00000000000..360d9ba6be2 --- /dev/null +++ b/plugins/statetrack_plugin/README.md @@ -0,0 +1,21 @@ +#statetrack_plugin + +This plugin adds hooks on chainbase operations emplace, modify, remove, undo, squash and commit and routes the objects and revision numbers to a ZMQ queue. + +Configuration +The following configuration statements in config.ini are recognized: + +plugin = eosio::statetrack_plugin -- enables the state track plugin + +st-zmq-sender-bind = ENDPOINT -- specifies the PUSH socket connect endpoint. Default value: tcp://127.0.0.1:3000. + +#adding library on Mac + +#adding library on Ubuntu + +```shell +apt-get install -y pkg-config libzmq5-dev +``` + +#Building + From 0c2f6b5ac1874d59f7bdd2a39ff8cd120b915dbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Tue, 30 Oct 2018 15:43:51 +0000 Subject: [PATCH 23/56] Update README.md --- plugins/statetrack_plugin/README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/plugins/statetrack_plugin/README.md b/plugins/statetrack_plugin/README.md index 360d9ba6be2..0b22709d18f 100644 --- a/plugins/statetrack_plugin/README.md +++ b/plugins/statetrack_plugin/README.md @@ -1,21 +1,22 @@ -#statetrack_plugin +# statetrack_plugin This plugin adds hooks on chainbase operations emplace, modify, remove, undo, squash and commit and routes the objects and revision numbers to a ZMQ queue. -Configuration +### Configuration + The following configuration statements in config.ini are recognized: plugin = eosio::statetrack_plugin -- enables the state track plugin st-zmq-sender-bind = ENDPOINT -- specifies the PUSH socket connect endpoint. Default value: tcp://127.0.0.1:3000. -#adding library on Mac +### adding library on Mac -#adding library on Ubuntu +### adding library on Ubuntu -```shell +``` apt-get install -y pkg-config libzmq5-dev ``` -#Building +### Building From e07a989fde7184fe434aa812c537593add7660f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Tue, 30 Oct 2018 16:49:12 +0000 Subject: [PATCH 24/56] Update README.md --- plugins/statetrack_plugin/README.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/plugins/statetrack_plugin/README.md b/plugins/statetrack_plugin/README.md index 0b22709d18f..364b15ae32c 100644 --- a/plugins/statetrack_plugin/README.md +++ b/plugins/statetrack_plugin/README.md @@ -10,9 +10,14 @@ plugin = eosio::statetrack_plugin -- enables the state track plugin st-zmq-sender-bind = ENDPOINT -- specifies the PUSH socket connect endpoint. Default value: tcp://127.0.0.1:3000. -### adding library on Mac +### adding ZMQ library on Mac -### adding library on Ubuntu +``` +brew tap jmuncaster/homebrew-header-only +brew install jmuncaster/header-only/cppzmq +``` + +### adding ZMQ library on Ubuntu ``` apt-get install -y pkg-config libzmq5-dev @@ -20,3 +25,7 @@ apt-get install -y pkg-config libzmq5-dev ### Building +``` +./eosio_build.sh +./eosio_install.sh +``` From 3fc83633257be4b717f03642b46da77aaa2d99aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Wed, 31 Oct 2018 17:16:39 +0000 Subject: [PATCH 25/56] Update statetrack_plugin.cpp --- .../statetrack_plugin/statetrack_plugin.cpp | 131 ++++++++++++++---- 1 file changed, 101 insertions(+), 30 deletions(-) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index d799bacde51..274e469d73e 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -14,15 +14,26 @@ namespace { namespace eosio { static appbase::abstract_plugin& _statetrack_plugin = app().register_plugin(); +typedef typename chainbase::get_index_type::type kv_index_type; + +struct filter_entry { + name code; + name scope; + name table; -using namespace eosio; - + std::tuple key() const { + return std::make_tuple(code, scope, table); + } -typedef typename chainbase::get_index_type::type kv_index_type; + friend bool operator<( const filter_entry& a, const filter_entry& b ) { + return a.key() < b.key(); + } + }; class statetrack_plugin_impl { public: void send_zmq_msg(const string content); + bool filter(const chain::table_id_object& tio); const chain::table_id_object& get_kvo_tio(const chainbase::database& db, const chain::key_value_object& kvo); const chain::account_object& get_tio_co(const chainbase::database& db, const chain::table_id_object& tio); @@ -37,20 +48,27 @@ class statetrack_plugin_impl { } static fc::string scope_sym_to_string(uint64_t sym_code) { - vector v; - for( int i = 0; i < 7; ++i ) { - char c = (char)(sym_code & 0xff); - if( !c ) break; - v.emplace_back(c); - sym_code >>= 8; + fc::string scope = sym_code.to_string(); + if(scope.length() > 0 && scope[0] == '.') { + vector v; + for( int i = 0; i < 7; ++i ) { + char c = (char)(sym_code & 0xff); + if( !c ) break; + v.emplace_back(c); + sym_code >>= 8; + } + return fc::string(v.begin(),v.end()); } - return fc::string(v.begin(),v.end()); + return scope; } fc::string socket_bind_str; zmq::context_t* context; zmq::socket_t* sender_socket; - fc::microseconds abi_serializer_max_time; + + std::vector filter_on; + std::vector filter_out; + fc::microseconds abi_serializer_max_time; private: bool shorten_abi_errors = true; }; @@ -62,6 +80,40 @@ void statetrack_plugin_impl::send_zmq_msg(const fc::string content) { memcpy(message.data(), content.c_str(), content.length()); sender_socket->send(message); } + +bool statetrack_plugin_impl::filter(const chain::table_id_object& tio) { + auto code = tio.code; + auto scope = scope_sym_to_string(tio.scope); + auto table = tio.table; + + if(filter_on.size() > 0) { + bool pass_on = false; + if (filter_on.find({ code, 0, 0 }) != filter_on.end()) { + pass_on = true; + } + if (filter_on.find({ code, scope, 0 }) != filter_on.end()) { + pass_on = true; + } + if (filter_on.find({ code, scope, table }) != filter_on.end()) { + pass_on = true; + } + if (!pass_on) { return false; } + } + + if(filter_out.size() > 0) { + if (filter_out.find({ code, 0, 0 }) != filter_out.end()) { + return false; + } + if (filter_out.find({ code, scope, 0 }) != filter_out.end()) { + return false; + } + if (filter_out.find({ code, scope, table }) != filter_out.end()) { + return false; + } + } + + return true; +} const chain::table_id_object& statetrack_plugin_impl::get_kvo_tio(const chainbase::database& db, const chain::key_value_object& kvo) { const chain::table_id_object *tio = db.find(kvo.t_id); @@ -106,16 +158,7 @@ db_op statetrack_plugin_impl::get_db_op(const chainbase::database& db, const cha op.id = kvo.id; op.op_type = op_type; op.code = tio.code; - - fc::string scope = tio.scope.to_string(); - - if(scope.length() > 0 && scope[0] == '.') { - op.scope = scope_sym_to_string(tio.scope); - } - else { - op.scope = scope; - } - + op.scope = scope_sym_to_string(tio.scope); op.table = tio.table; if(op_type == op_type_enum::CREATE || @@ -156,6 +199,28 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { ilog("initializing statetrack plugin"); try { + if( options.count( "st-filter-on" )) { + auto fo = options.at( "filter-on" ).as>(); + for( auto& s : fo ) { + std::vector v; + boost::split( v, s, boost::is_any_of( ":" )); + EOS_ASSERT( v.size() == 3, fc::invalid_arg_exception, "Invalid value ${s} for --filter-on", ("s", s)); + filter_entry fe{v[0], v[1], v[2]}; + my->filter_on.insert( fe ); + } + } + + if( options.count( "st-filter-out" )) { + auto fo = options.at( "filter-out" ).as>(); + for( auto& s : fo ) { + std::vector v; + boost::split( v, s, boost::is_any_of( ":" )); + EOS_ASSERT( v.size() == 3, fc::invalid_arg_exception, "Invalid value ${s} for --filter-out", ("s", s)); + filter_entry fe{v[0], v[1], v[2]}; + my->filter_out.insert( fe ); + } + } + my->socket_bind_str = options.at(SENDER_BIND).as(); if (my->socket_bind_str.empty()) { wlog("zmq-sender-bind not specified => eosio::statetrack_plugin disabled."); @@ -180,21 +245,27 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { auto& kv_index = db.get_index(); kv_index.applied_emplace = [&](const chain::key_value_object& kvo) { - fc::string data = fc::json::to_string(my->get_db_op(db, kvo, op_type_enum::CREATE)); - ilog("STATETRACK key_value_object emplace: ${data}", ("data", data)); - my->send_zmq_msg(data); + if(my->filter(my->get_kvo_tio(db, kvo))) { + fc::string data = fc::json::to_string(my->get_db_op(db, kvo, op_type_enum::CREATE)); + ilog("STATETRACK key_value_object emplace: ${data}", ("data", data)); + my->send_zmq_msg(data); + } }; kv_index.applied_modify = [&](const chain::key_value_object& kvo) { - fc::string data = fc::json::to_string(my->get_db_op(db, kvo, op_type_enum::MODIFY)); - ilog("STATETRACK key_value_object modify: ${data}", ("data", data)); - my->send_zmq_msg(data); + if(my->filter(my->get_kvo_tio(db, kvo))) { + fc::string data = fc::json::to_string(my->get_db_op(db, kvo, op_type_enum::MODIFY)); + ilog("STATETRACK key_value_object modify: ${data}", ("data", data)); + my->send_zmq_msg(data); + } }; kv_index.applied_remove = [&](const chain::key_value_object& kvo) { - fc::string data = fc::json::to_string(my->get_db_op(db, kvo, op_type_enum::REMOVE)); - ilog("STATETRACK key_value_object remove: ${data}", ("data", data)); - my->send_zmq_msg(data); + if(my->filter(my->get_kvo_tio(db, kvo))) { + fc::string data = fc::json::to_string(my->get_db_op(db, kvo, op_type_enum::REMOVE)); + ilog("STATETRACK key_value_object remove: ${data}", ("data", data)); + my->send_zmq_msg(data); + } }; kv_index.applied_undo = [&](const int64_t revision) { From 5ebaf2791989d3676c78664404db17b478958770 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Wed, 31 Oct 2018 17:37:22 +0000 Subject: [PATCH 26/56] Update statetrack_plugin.cpp --- .../statetrack_plugin/statetrack_plugin.cpp | 47 +++++++++---------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index 274e469d73e..66c6024fdf3 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -41,6 +41,9 @@ class statetrack_plugin_impl { fc::variant get_kvo_row(const chainbase::database& db, const chain::table_id_object& tio, const chain::key_value_object& kvo, bool json = true); db_op get_db_op(const chainbase::database& db, const chain::key_value_object& kvo, op_type_enum op_type, bool json = true); db_rev get_db_rev(const int64_t revision, op_type_enum op_type); + + void on_applied_op(const chainbase::database& db, const chain::key_value_object& kvo, op_type_enum op_type); + void on_applied_rev(const int64_t revision, op_type_enum op_type); static void copy_inline_row(const chain::key_value_object& obj, vector& data) { data.resize( obj.value.size() ); @@ -175,6 +178,20 @@ db_rev statetrack_plugin_impl::get_db_rev(const int64_t revision, op_type_enum o rev.revision = revision; return rev; } + +void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const chain::key_value_object& kvo, op_type_enum op_type) { + if(filter(get_kvo_tio(db, kvo))) { + fc::string data = fc::json::to_string(get_db_op(db, kvo, op_type)); + ilog("STATETRACK key_value_object {op_type}: ${data}", ("op_type", op_type), ("data", data)); + send_zmq_msg(data); + } +} + +void statetrack_plugin_impl::on_applied_rev(const int64_t revision, op_type_enum op_type) { + fc::string data = fc::json::to_string(get_db_rev(revision, op_type)); + ilog("STATETRACK {op_type}: ${data}", ("op_type", op_type), ("data", data)); + send_zmq_msg(data); +} // Plugin implementation @@ -245,45 +262,27 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { auto& kv_index = db.get_index(); kv_index.applied_emplace = [&](const chain::key_value_object& kvo) { - if(my->filter(my->get_kvo_tio(db, kvo))) { - fc::string data = fc::json::to_string(my->get_db_op(db, kvo, op_type_enum::CREATE)); - ilog("STATETRACK key_value_object emplace: ${data}", ("data", data)); - my->send_zmq_msg(data); - } + my->on_applied_op(db, kvo, op_type_enum::CREATE); }; kv_index.applied_modify = [&](const chain::key_value_object& kvo) { - if(my->filter(my->get_kvo_tio(db, kvo))) { - fc::string data = fc::json::to_string(my->get_db_op(db, kvo, op_type_enum::MODIFY)); - ilog("STATETRACK key_value_object modify: ${data}", ("data", data)); - my->send_zmq_msg(data); - } + my->on_applied_op(db, kvo, op_type_enum::MODIFY); }; kv_index.applied_remove = [&](const chain::key_value_object& kvo) { - if(my->filter(my->get_kvo_tio(db, kvo))) { - fc::string data = fc::json::to_string(my->get_db_op(db, kvo, op_type_enum::REMOVE)); - ilog("STATETRACK key_value_object remove: ${data}", ("data", data)); - my->send_zmq_msg(data); - } + my->on_applied_op(db, kvo, op_type_enum::REMOVE); }; kv_index.applied_undo = [&](const int64_t revision) { - fc::string data = fc::json::to_string(my->get_db_rev(revision, op_type_enum::UNDO)); - ilog("STATETRACK undo: ${data}", ("data", data)); - my->send_zmq_msg(data); + my->on_applied_rev(revision, op_type_enum::UNDO); }; /*kv_index.applied_squash = [&](const int64_t revision) { - fc::string data = fc::json::to_string(my->get_db_rev(revision, op_type_enum::SQUASH)); - ilog("STATETRACK squash: ${data}", ("data", data)); - my->send_zmq_msg(data); + my->on_applied_rev(revision, op_type_enum::SQUASH); };*/ kv_index.applied_commit = [&](const int64_t revision) { - fc::string data = fc::json::to_string(my->get_db_rev(revision, op_type_enum::COMMIT)); - ilog("STATETRACK commit: ${data}", ("data", data)); - my->send_zmq_msg(data); + my->on_applied_rev(revision, op_type_enum::COMMIT); }; } FC_LOG_AND_RETHROW() From 8a5869f383ca39013d572ee95a29a88ba1d6b1b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Wed, 31 Oct 2018 17:37:47 +0000 Subject: [PATCH 27/56] Update statetrack_plugin.cpp --- plugins/statetrack_plugin/statetrack_plugin.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index 66c6024fdf3..5af5b490107 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -217,7 +217,7 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { try { if( options.count( "st-filter-on" )) { - auto fo = options.at( "filter-on" ).as>(); + auto fo = options.at( "st-filter-on" ).as>(); for( auto& s : fo ) { std::vector v; boost::split( v, s, boost::is_any_of( ":" )); @@ -228,7 +228,7 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { } if( options.count( "st-filter-out" )) { - auto fo = options.at( "filter-out" ).as>(); + auto fo = options.at( "st-filter-out" ).as>(); for( auto& s : fo ) { std::vector v; boost::split( v, s, boost::is_any_of( ":" )); From 67e2054bed66e21f56369a679dc0019b430cc5d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Thu, 1 Nov 2018 01:47:24 +0000 Subject: [PATCH 28/56] fixes to undo --- libraries/chainbase | 2 +- .../statetrack_plugin/statetrack_plugin.hpp | 5 ++-- .../statetrack_plugin/statetrack_plugin.cpp | 24 +++++++++---------- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/libraries/chainbase b/libraries/chainbase index 2e29ae44d1a..2412f00982a 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit 2e29ae44d1a43dc5eba15f123b40678cf2b4a896 +Subproject commit 2412f00982af0227abfd35810f2131e24c53d459 diff --git a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp index 94143ba07b7..0a3aebfeb74 100644 --- a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp +++ b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp @@ -22,8 +22,7 @@ enum op_type_enum { MODIFY = 1, REMOVE = 2, UNDO = 3, - SQUASH = 4, - COMMIT = 5 + COMMIT = 4 }; struct db_op { @@ -63,6 +62,6 @@ class statetrack_plugin : public plugin { } -FC_REFLECT_ENUM( eosio::op_type_enum, (CREATE)(MODIFY)(REMOVE)(UNDO)(SQUASH)(COMMIT) ) +FC_REFLECT_ENUM( eosio::op_type_enum, (CREATE)(MODIFY)(REMOVE)(UNDO)(COMMIT) ) FC_REFLECT( eosio::db_op, (id)(op_type)(code)(scope)(table)(payer)(value) ) FC_REFLECT( eosio::db_rev, (op_type)(revision) ) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index 5af5b490107..051dd0f0edd 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -5,6 +5,7 @@ #include #include +#include namespace { const char* SENDER_BIND = "st-zmq-sender-bind"; @@ -50,15 +51,16 @@ class statetrack_plugin_impl { memcpy( data.data(), obj.value.data(), obj.value.size() ); } - static fc::string scope_sym_to_string(uint64_t sym_code) { + static fc::string scope_sym_to_string(chain::scope_name sym_code) { fc::string scope = sym_code.to_string(); if(scope.length() > 0 && scope[0] == '.') { + uint64_t scope_int = sym_code; vector v; for( int i = 0; i < 7; ++i ) { - char c = (char)(sym_code & 0xff); + char c = (char)(scope_int & 0xff); if( !c ) break; v.emplace_back(c); - sym_code >>= 8; + scope_int >>= 8; } return fc::string(v.begin(),v.end()); } @@ -69,8 +71,8 @@ class statetrack_plugin_impl { zmq::context_t* context; zmq::socket_t* sender_socket; - std::vector filter_on; - std::vector filter_out; + std::set filter_on; + std::set filter_out; fc::microseconds abi_serializer_max_time; private: bool shorten_abi_errors = true; @@ -182,14 +184,14 @@ db_rev statetrack_plugin_impl::get_db_rev(const int64_t revision, op_type_enum o void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const chain::key_value_object& kvo, op_type_enum op_type) { if(filter(get_kvo_tio(db, kvo))) { fc::string data = fc::json::to_string(get_db_op(db, kvo, op_type)); - ilog("STATETRACK key_value_object {op_type}: ${data}", ("op_type", op_type), ("data", data)); + ilog("STATETRACK key_value_object {op_type}: ${data}", ("op_type", op_type) ("data", data)); send_zmq_msg(data); } } void statetrack_plugin_impl::on_applied_rev(const int64_t revision, op_type_enum op_type) { fc::string data = fc::json::to_string(get_db_rev(revision, op_type)); - ilog("STATETRACK {op_type}: ${data}", ("op_type", op_type), ("data", data)); + ilog("STATETRACK {op_type}: ${data}", ("op_type", op_type) ("data", data)); send_zmq_msg(data); } @@ -248,7 +250,7 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { my->sender_socket = new zmq::socket_t(*my->context, ZMQ_PUSH); ilog("Bind to ZMQ PUSH socket ${u}", ("u", my->socket_bind_str)); - my->sender_socket->connect(my->socket_bind_str); + my->sender_socket->bind(my->socket_bind_str); ilog("Bind to ZMQ PUSH socket successful"); @@ -276,11 +278,7 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { kv_index.applied_undo = [&](const int64_t revision) { my->on_applied_rev(revision, op_type_enum::UNDO); }; - - /*kv_index.applied_squash = [&](const int64_t revision) { - my->on_applied_rev(revision, op_type_enum::SQUASH); - };*/ - + kv_index.applied_commit = [&](const int64_t revision) { my->on_applied_rev(revision, op_type_enum::COMMIT); }; From 27c777cffeae924c683ae0b7a9ca45ae17bef48e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Thu, 1 Nov 2018 15:20:13 +0000 Subject: [PATCH 29/56] work on account and permissions events --- .../statetrack_plugin/statetrack_plugin.hpp | 45 ++- .../statetrack_plugin/statetrack_plugin.cpp | 288 +++++++++++++++--- 2 files changed, 284 insertions(+), 49 deletions(-) diff --git a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp index 0a3aebfeb74..f43952e26bf 100644 --- a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp +++ b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp @@ -7,6 +7,9 @@ #include #include #include +#include +#include +#include #include #include @@ -15,23 +18,43 @@ namespace eosio { using namespace appbase; +typedef chain::account_object::id_type co_id_type; +typedef chain::permission_usage_object::id_type puo_id_type; +typedef chain::permission_object::id_type po_id_type; +typedef chain::permission_link_object::id_type plo_id_type; typedef chain::key_value_object::id_type kvo_id_type; + enum op_type_enum { - CREATE = 0, - MODIFY = 1, - REMOVE = 2, - UNDO = 3, - COMMIT = 4 + TABLE_REMOVE = 0, + ROW_CREATE = 1, + ROW_MODIFY = 2, + ROW_REMOVE = 3, + REV_UNDO = 4, + REV_COMMIT = 5, +}; + +struct db_account { + account_name name; + uint8_t vm_type = 0; + uint8_t vm_version = 0; + bool privileged = false; + + chain::time_point last_code_update; + chain::digest_type code_version; + chain::block_timestamp_type creation_date; }; -struct db_op { - kvo_id_type id; +struct db_table { op_type_enum op_type; account_name code; fc::string scope; chain::table_name table; account_name payer; +}; + +struct db_op : db_table { + uint64_t id; fc::variant value; }; @@ -62,6 +85,8 @@ class statetrack_plugin : public plugin { } -FC_REFLECT_ENUM( eosio::op_type_enum, (CREATE)(MODIFY)(REMOVE)(UNDO)(COMMIT) ) -FC_REFLECT( eosio::db_op, (id)(op_type)(code)(scope)(table)(payer)(value) ) -FC_REFLECT( eosio::db_rev, (op_type)(revision) ) +FC_REFLECT_ENUM( eosio::op_type_enum, (TABLE_REMOVE)(ROW_CREATE)(ROW_MODIFY)(ROW_REMOVE)(REV_UNDO)(REV_COMMIT) ) +FC_REFLECT( eosio::db_account, (name)(vm_type)(vm_version)(privileged)(last_code_update)(code_version)(creation_date) ) +FC_REFLECT( eosio::db_table, (op_type)(code)(scope)(table)(payer) ) +FC_REFLECT_DERIVED( eosio::db_op, (eosio::db_table), (id)(value) ) +FC_REFLECT( eosio::db_rev, (op_type)(revision) ) \ No newline at end of file diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index 051dd0f0edd..fc4ab23bfaf 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -15,7 +15,14 @@ namespace { namespace eosio { static appbase::abstract_plugin& _statetrack_plugin = app().register_plugin(); + +typedef typename chainbase::get_index_type::type co_index_type; +typedef typename chainbase::get_index_type::type po_index_type; +typedef typename chainbase::get_index_type::type plo_index_type; +typedef typename chainbase::get_index_type::type ti_index_type; typedef typename chainbase::get_index_type::type kv_index_type; + + struct filter_entry { name code; @@ -34,16 +41,27 @@ struct filter_entry { class statetrack_plugin_impl { public: void send_zmq_msg(const string content); - bool filter(const chain::table_id_object& tio); + bool filter(const chain::account_name& code, const fc::string& scope, const chain::table_name& table); - const chain::table_id_object& get_kvo_tio(const chainbase::database& db, const chain::key_value_object& kvo); + const chain::table_id_object* get_kvo_tio(const chainbase::database& db, const chain::key_value_object& kvo); const chain::account_object& get_tio_co(const chainbase::database& db, const chain::table_id_object& tio); const abi_def& get_co_abi(const chain::account_object& co ); fc::variant get_kvo_row(const chainbase::database& db, const chain::table_id_object& tio, const chain::key_value_object& kvo, bool json = true); - db_op get_db_op(const chainbase::database& db, const chain::key_value_object& kvo, op_type_enum op_type, bool json = true); + + db_op get_db_op(const chainbase::database& db, const chain::account_object& co, op_type_enum op_type); + db_op get_db_op(const chainbase::database& db, const chain::permission_object& po, op_type_enum op_type); + db_op get_db_op(const chainbase::database& db, const chain::permission_link_object& plo, op_type_enum op_type); + db_op get_db_op(const chainbase::database& db, const chain::table_id_object& tio, const chain::key_value_object& kvo, op_type_enum op_type, bool json = true); + db_rev get_db_rev(const int64_t revision, op_type_enum op_type); + void on_applied_table(const chainbase::database& db, const chain::table_id_object& tio, op_type_enum op_type); + + void on_applied_op(const chainbase::database& db, const chain::account_object& co, op_type_enum op_type); + void on_applied_op(const chainbase::database& db, const chain::permission_object& po, op_type_enum op_type); + void on_applied_op(const chainbase::database& db, const chain::permission_link_object& plo, op_type_enum op_type); void on_applied_op(const chainbase::database& db, const chain::key_value_object& kvo, op_type_enum op_type); + void on_applied_rev(const int64_t revision, op_type_enum op_type); static void copy_inline_row(const chain::key_value_object& obj, vector& data) { @@ -55,6 +73,9 @@ class statetrack_plugin_impl { fc::string scope = sym_code.to_string(); if(scope.length() > 0 && scope[0] == '.') { uint64_t scope_int = sym_code; + + ilog("TEST scope_int ${scope_int}", ("scope_int", scope_int)); + vector v; for( int i = 0; i < 7; ++i ) { char c = (char)(scope_int & 0xff); @@ -81,16 +102,14 @@ class statetrack_plugin_impl { // statetrack plugin implementation void statetrack_plugin_impl::send_zmq_msg(const fc::string content) { - zmq::message_t message(content.length()); - memcpy(message.data(), content.c_str(), content.length()); - sender_socket->send(message); + if(sender_socket != nullptr) { + zmq::message_t message(content.length()); + memcpy(message.data(), content.c_str(), content.length()); + sender_socket->send(message); + } } - -bool statetrack_plugin_impl::filter(const chain::table_id_object& tio) { - auto code = tio.code; - auto scope = scope_sym_to_string(tio.scope); - auto table = tio.table; - + +bool statetrack_plugin_impl::filter(const chain::account_name& code, const fc::string& scope, const chain::table_name& table) { if(filter_on.size() > 0) { bool pass_on = false; if (filter_on.find({ code, 0, 0 }) != filter_on.end()) { @@ -120,10 +139,8 @@ bool statetrack_plugin_impl::filter(const chain::table_id_object& tio) { return true; } -const chain::table_id_object& statetrack_plugin_impl::get_kvo_tio(const chainbase::database& db, const chain::key_value_object& kvo) { - const chain::table_id_object *tio = db.find(kvo.t_id); - EOS_ASSERT(tio != nullptr, chain::contract_table_query_exception, "Fail to retrieve table for ${t_id}", ("t_id", kvo.t_id) ); - return *tio; +const chain::table_id_object* statetrack_plugin_impl::get_kvo_tio(const chainbase::database& db, const chain::key_value_object& kvo) { + return db.find(kvo.t_id); } const chain::account_object& statetrack_plugin_impl::get_tio_co(const chainbase::database& db, const chain::table_id_object& tio) { @@ -155,19 +172,72 @@ fc::variant statetrack_plugin_impl::get_kvo_row(const chainbase::database& db, c } } -db_op statetrack_plugin_impl::get_db_op(const chainbase::database& db, const chain::key_value_object& kvo, op_type_enum op_type, bool json) { +db_op statetrack_plugin_impl::get_db_op(const chainbase::database& db, const chain::account_object& co, op_type_enum op_type) { + db_op op; + + name system = N(system); + + op.id = co.id._id; + op.op_type = op_type; + op.code = system; + op.scope = system.to_string(); + op.table = N(accounts); + + db_account account; + account.name = co.name; + account.vm_type = co.vm_type; + account.vm_version = co.vm_version; + account.privileged = co.privileged; + account.last_code_update = co.last_code_update; + account.code_version = co.code_version; + account.creation_date = co.creation_date; + + op.value = fc::variant(account); + + return op; +} + +db_op statetrack_plugin_impl::get_db_op(const chainbase::database& db, const chain::permission_object& po, op_type_enum op_type) { + db_op op; + + name system = N(system); + + op.id = po.id._id; + op.op_type = op_type; + op.code = system; + op.scope = system.to_string(); + op.table = N(permissions); + op.value = fc::variant(po); + + return op; +} + +db_op statetrack_plugin_impl::get_db_op(const chainbase::database& db, const chain::permission_link_object& plo, op_type_enum op_type) { db_op op; - const chain::table_id_object& tio = get_kvo_tio(db, kvo); + name system = N(system); + + op.id = plo.id._id; + op.op_type = op_type; + op.code = system; + op.scope = system.to_string(); + op.table = N(permission_links); + op.value = fc::variant(plo); + + return op; +} + +db_op statetrack_plugin_impl::get_db_op(const chainbase::database& db, const chain::table_id_object& tio, const chain::key_value_object& kvo, op_type_enum op_type, bool json) { + db_op op; - op.id = kvo.id; + op.id = kvo.primary_key; op.op_type = op_type; op.code = tio.code; op.scope = scope_sym_to_string(tio.scope); op.table = tio.table; - if(op_type == op_type_enum::CREATE || - op_type == op_type_enum::MODIFY) { + if(op_type == op_type_enum::ROW_CREATE || + op_type == op_type_enum::ROW_MODIFY) { op.value = get_kvo_row(db, tio, kvo, json); } @@ -175,23 +245,93 @@ db_op statetrack_plugin_impl::get_db_op(const chainbase::database& db, const cha } db_rev statetrack_plugin_impl::get_db_rev(const int64_t revision, op_type_enum op_type) { - db_rev rev; - rev.op_type = op_type; - rev.revision = revision; + db_rev rev = {op_type, revision}; return rev; } + +void statetrack_plugin_impl::on_applied_table(const chainbase::database& db, const chain::table_id_object& tio, op_type_enum op_type) { + auto code = tio.code; + auto scope = scope_sym_to_string(tio.scope); + auto table = tio.table; + + if(filter(code, scope, table)) { + db_table table; + + table.op_type = op_type; + table.code = tio.code; + table.scope = scope_sym_to_string(tio.scope); + table.table = tio.table; + + fc::string data = fc::json::to_string(table); + ilog("STATETRACK table_id_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); + send_zmq_msg(data); + } +} + +void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const chain::account_object& co, op_type_enum op_type) { + name system = N(system); + auto code = system; + auto scope = system.to_string(); + auto table = N(accounts); + + if(filter(code, scope, table)) { + fc::string data = fc::json::to_string(get_db_op(db, co, op_type)); + ilog("STATETRACK account_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); + send_zmq_msg(data); + } +} + +void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const chain::permission_object& po, op_type_enum op_type) { + name system = N(system); + auto code = system; + auto scope = system.to_string(); + auto table = N(permission); + + if(filter(code, scope, table)) { + fc::string data = fc::json::to_string(get_db_op(db, po, op_type)); + ilog("STATETRACK permission_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); + send_zmq_msg(data); + } +} + +void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const chain::permission_link_object& plo, op_type_enum op_type) { + name system = N(system); + auto code = system; + auto scope = system.to_string(); + auto table = N(permission_links); + + if(filter(code, scope, table)) { + fc::string data = fc::json::to_string(get_db_op(db, plo, op_type)); + ilog("STATETRACK permission_link_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); + send_zmq_msg(data); + } +} void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const chain::key_value_object& kvo, op_type_enum op_type) { - if(filter(get_kvo_tio(db, kvo))) { - fc::string data = fc::json::to_string(get_db_op(db, kvo, op_type)); - ilog("STATETRACK key_value_object {op_type}: ${data}", ("op_type", op_type) ("data", data)); + + const chain::table_id_object* tio_ptr = get_kvo_tio(db, kvo); + + if(tio_ptr == nullptr) + return; + + const chain::table_id_object& tio = *tio_ptr; + + auto code = tio.code; + auto scope = scope_sym_to_string(tio.scope); + auto table = tio.table; + + if(filter(code, scope, table)) { + fc::string data = fc::json::to_string(get_db_op(db, *tio_ptr, kvo, op_type)); + ilog("STATETRACK key_value_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); send_zmq_msg(data); } } + + void statetrack_plugin_impl::on_applied_rev(const int64_t revision, op_type_enum op_type) { fc::string data = fc::json::to_string(get_db_rev(revision, op_type)); - ilog("STATETRACK {op_type}: ${data}", ("op_type", op_type) ("data", data)); + ilog("STATETRACK ${op_type}: ${data}", ("op_type", op_type)("data", data)); send_zmq_msg(data); } @@ -260,28 +400,98 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { my->abi_serializer_max_time = chain_plug.get_abi_serializer_max_time(); ilog("Binding database events"); + + auto undo_lambda = [&](const int64_t revision) { + my->on_applied_rev(revision, op_type_enum::REV_UNDO); + }; + + auto commit_lambda = [&](const int64_t revision) { + my->on_applied_rev(revision, op_type_enum::REV_COMMIT); + }; + + // account_object events + + auto& co_inde = db.get_index(); + + co_inde.applied_emplace = [&](const chain::account_object& co) { + my->on_applied_op(db, co, op_type_enum::ROW_CREATE); + }; + + co_inde.applied_modify = [&](const chain::account_object& co) { + my->on_applied_op(db, co, op_type_enum::ROW_MODIFY); + }; + + co_inde.applied_remove = [&](const chain::account_object& co) { + my->on_applied_op(db, co, op_type_enum::ROW_REMOVE); + }; + + co_inde.applied_undo = undo_lambda; + co_inde.applied_commit = commit_lambda; + + // permisson_object events + + auto& po_inde = db.get_index(); + + po_inde.applied_emplace = [&](const chain::permission_object& po) { + my->on_applied_op(db, po, op_type_enum::ROW_CREATE); + }; + + po_inde.applied_modify = [&](const chain::permission_object& po) { + my->on_applied_op(db, po, op_type_enum::ROW_MODIFY); + }; + + po_inde.applied_remove = [&](const chain::permission_object& po) { + my->on_applied_op(db, po, op_type_enum::ROW_REMOVE); + }; + + po_inde.applied_undo = undo_lambda; + po_inde.applied_commit = commit_lambda; + + // permisson_link_object events + + auto& plo_inde = db.get_index(); + + plo_inde.applied_emplace = [&](const chain::permission_link_object& plo) { + my->on_applied_op(db, plo, op_type_enum::ROW_CREATE); + }; + + plo_inde.applied_modify = [&](const chain::permission_link_object& plo) { + my->on_applied_op(db, plo, op_type_enum::ROW_MODIFY); + }; + + plo_inde.applied_remove = [&](const chain::permission_link_object& plo) { + my->on_applied_op(db, plo, op_type_enum::ROW_REMOVE); + }; + + plo_inde.applied_undo = undo_lambda; + plo_inde.applied_commit = commit_lambda; + + // table_id_object events + + auto& ti_index = db.get_index(); + + ti_index.applied_remove = [&](const chain::table_id_object& tio) { + my->on_applied_table(db, tio, op_type_enum::TABLE_REMOVE); + }; + + // key_value_object events auto& kv_index = db.get_index(); - + kv_index.applied_emplace = [&](const chain::key_value_object& kvo) { - my->on_applied_op(db, kvo, op_type_enum::CREATE); + my->on_applied_op(db, kvo, op_type_enum::ROW_CREATE); }; kv_index.applied_modify = [&](const chain::key_value_object& kvo) { - my->on_applied_op(db, kvo, op_type_enum::MODIFY); + my->on_applied_op(db, kvo, op_type_enum::ROW_MODIFY); }; kv_index.applied_remove = [&](const chain::key_value_object& kvo) { - my->on_applied_op(db, kvo, op_type_enum::REMOVE); + my->on_applied_op(db, kvo, op_type_enum::ROW_REMOVE); }; - kv_index.applied_undo = [&](const int64_t revision) { - my->on_applied_rev(revision, op_type_enum::UNDO); - }; - - kv_index.applied_commit = [&](const int64_t revision) { - my->on_applied_rev(revision, op_type_enum::COMMIT); - }; + kv_index.applied_undo = undo_lambda; + kv_index.applied_commit = commit_lambda; } FC_LOG_AND_RETHROW() } From 1580e33f07c774d562c5059082ce2dcd980b2838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Thu, 1 Nov 2018 15:23:30 +0000 Subject: [PATCH 30/56] remove sym_int log --- plugins/statetrack_plugin/statetrack_plugin.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index fc4ab23bfaf..23049283684 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -73,9 +73,6 @@ class statetrack_plugin_impl { fc::string scope = sym_code.to_string(); if(scope.length() > 0 && scope[0] == '.') { uint64_t scope_int = sym_code; - - ilog("TEST scope_int ${scope_int}", ("scope_int", scope_int)); - vector v; for( int i = 0; i < 7; ++i ) { char c = (char)(scope_int & 0xff); From 627a21d79a8abe0f33ce5175b77cfd8f5a9fa70e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Fri, 2 Nov 2018 08:28:57 +0000 Subject: [PATCH 31/56] fix crash on plugin shutdown --- plugins/statetrack_plugin/statetrack_plugin.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index 23049283684..e6108fd3902 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -504,6 +504,8 @@ void statetrack_plugin::plugin_shutdown() { my->sender_socket->close(); delete my->sender_socket; delete my->context; + my->sender_socket = nullptr; + my->context = nullptr; } } From 180d85b167f141ca179d7dbb2b2461b751d9066d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Mon, 5 Nov 2018 17:55:24 +0000 Subject: [PATCH 32/56] Update statetrack_plugin.cpp --- .../statetrack_plugin/statetrack_plugin.cpp | 37 ++++++++++++------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index e6108fd3902..95e414336d9 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -99,11 +99,12 @@ class statetrack_plugin_impl { // statetrack plugin implementation void statetrack_plugin_impl::send_zmq_msg(const fc::string content) { - if(sender_socket != nullptr) { - zmq::message_t message(content.length()); - memcpy(message.data(), content.c_str(), content.length()); - sender_socket->send(message); - } + if(sender_socket == nullptr) + return; + + zmq::message_t message(content.length()); + memcpy(message.data(), content.c_str(), content.length()); + sender_socket->send(message); } bool statetrack_plugin_impl::filter(const chain::account_name& code, const fc::string& scope, const chain::table_name& table) { @@ -247,6 +248,9 @@ db_rev statetrack_plugin_impl::get_db_rev(const int64_t revision, op_type_enum o } void statetrack_plugin_impl::on_applied_table(const chainbase::database& db, const chain::table_id_object& tio, op_type_enum op_type) { + if(sender_socket == nullptr) + return; + auto code = tio.code; auto scope = scope_sym_to_string(tio.scope); auto table = tio.table; @@ -266,6 +270,9 @@ void statetrack_plugin_impl::on_applied_table(const chainbase::database& db, con } void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const chain::account_object& co, op_type_enum op_type) { + if(sender_socket == nullptr) + return; + name system = N(system); auto code = system; auto scope = system.to_string(); @@ -279,6 +286,9 @@ void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const } void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const chain::permission_object& po, op_type_enum op_type) { + if(sender_socket == nullptr) + return; + name system = N(system); auto code = system; auto scope = system.to_string(); @@ -292,6 +302,9 @@ void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const } void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const chain::permission_link_object& plo, op_type_enum op_type) { + if(sender_socket == nullptr) + return; + name system = N(system); auto code = system; auto scope = system.to_string(); @@ -305,6 +318,8 @@ void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const } void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const chain::key_value_object& kvo, op_type_enum op_type) { + if(sender_socket == nullptr) + return; const chain::table_id_object* tio_ptr = get_kvo_tio(db, kvo); @@ -327,6 +342,9 @@ void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const void statetrack_plugin_impl::on_applied_rev(const int64_t revision, op_type_enum op_type) { + if(sender_socket == nullptr) + return; + fc::string data = fc::json::to_string(get_db_rev(revision, op_type)); ilog("STATETRACK ${op_type}: ${data}", ("op_type", op_type)("data", data)); send_zmq_msg(data); @@ -402,10 +420,6 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { my->on_applied_rev(revision, op_type_enum::REV_UNDO); }; - auto commit_lambda = [&](const int64_t revision) { - my->on_applied_rev(revision, op_type_enum::REV_COMMIT); - }; - // account_object events auto& co_inde = db.get_index(); @@ -423,7 +437,6 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { }; co_inde.applied_undo = undo_lambda; - co_inde.applied_commit = commit_lambda; // permisson_object events @@ -442,7 +455,6 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { }; po_inde.applied_undo = undo_lambda; - po_inde.applied_commit = commit_lambda; // permisson_link_object events @@ -461,7 +473,6 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { }; plo_inde.applied_undo = undo_lambda; - plo_inde.applied_commit = commit_lambda; // table_id_object events @@ -488,7 +499,6 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { }; kv_index.applied_undo = undo_lambda; - kv_index.applied_commit = commit_lambda; } FC_LOG_AND_RETHROW() } @@ -505,7 +515,6 @@ void statetrack_plugin::plugin_shutdown() { delete my->sender_socket; delete my->context; my->sender_socket = nullptr; - my->context = nullptr; } } From f273179ddbbc70eaf7c5cc39a781cf2b01e85855 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Mon, 5 Nov 2018 18:34:07 +0000 Subject: [PATCH 33/56] Update statetrack_plugin.cpp --- .../statetrack_plugin/statetrack_plugin.cpp | 111 ++++++++++++++++-- 1 file changed, 103 insertions(+), 8 deletions(-) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index 95e414336d9..23366560280 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -4,6 +4,7 @@ */ #include +#include #include #include @@ -22,8 +23,6 @@ typedef typename chainbase::get_index_type::type typedef typename chainbase::get_index_type::type ti_index_type; typedef typename chainbase::get_index_type::type kv_index_type; - - struct filter_entry { name code; name scope; @@ -40,6 +39,15 @@ struct filter_entry { class statetrack_plugin_impl { public: + statetrack_plugin_impl() { + blacklist_actions.emplace + (std::make_pair(chain::config::system_account_name, + std::set{ N(onblock) } )); + blacklist_actions.emplace + (std::make_pair(N(blocktwitter), + std::set{ N(tweet) } )); + } + void send_zmq_msg(const string content); bool filter(const chain::account_name& code, const fc::string& scope, const chain::table_name& table); @@ -55,15 +63,21 @@ class statetrack_plugin_impl { db_rev get_db_rev(const int64_t revision, op_type_enum op_type); + //generic_index state tables void on_applied_table(const chainbase::database& db, const chain::table_id_object& tio, op_type_enum op_type); - + //generic_index op void on_applied_op(const chainbase::database& db, const chain::account_object& co, op_type_enum op_type); void on_applied_op(const chainbase::database& db, const chain::permission_object& po, op_type_enum op_type); void on_applied_op(const chainbase::database& db, const chain::permission_link_object& plo, op_type_enum op_type); void on_applied_op(const chainbase::database& db, const chain::key_value_object& kvo, op_type_enum op_type); - + //generic_index undo void on_applied_rev(const int64_t revision, op_type_enum op_type); - + //blocks and transactions + void on_applied_transaction( const transaction_trace_ptr& ttp ); + void on_accepted_block(const block_state_ptr& bsp); + void on_action_trace( const action_trace& at, const block_state_ptr& bsp ); + void on_irreversible_block( const chain::block_state_ptr& bsp ) + static void copy_inline_row(const chain::key_value_object& obj, vector& data) { data.resize( obj.value.size() ); memcpy( data.data(), obj.value.data(), obj.value.size() ); @@ -89,9 +103,16 @@ class statetrack_plugin_impl { zmq::context_t* context; zmq::socket_t* sender_socket; + std::map cached_traces; + std::map> blacklist_actions; + std::set filter_on; std::set filter_out; fc::microseconds abi_serializer_max_time; + + fc::optional applied_transaction_connection; + fc::optional accepted_block_connection; + fc::optional irreversible_block_connection; private: bool shorten_abi_errors = true; }; @@ -339,8 +360,6 @@ void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const } } - - void statetrack_plugin_impl::on_applied_rev(const int64_t revision, op_type_enum op_type) { if(sender_socket == nullptr) return; @@ -350,6 +369,66 @@ void statetrack_plugin_impl::on_applied_rev(const int64_t revision, op_type_enum send_zmq_msg(data); } +void statetrack_plugin_impl::on_applied_transaction( const transaction_trace_ptr& ttp ) +{ + if (ttp->receipt) { + cached_traces[ttp->id] = ttp; + } +} + +void statetrack_plugin_impl::on_accepted_block(const block_state_ptr& bsp) +{ + //TODO send accepted block information + + for (auto& t : bsp->block->transactions) { + transaction_id_type id; + if (t.trx.contains()) { + id = t.trx.get(); + } + else { + id = t.trx.get().id(); + } + + if( t.status == transaction_receipt_header::executed ) { + // Send traces only for executed transactions + auto it = cached_traces.find(id); + if (it == cached_traces.end() || !it->second->receipt) { + ilog("missing trace for transaction {id}", ("id", id)); + continue; + } + + for( const auto& atrace : it->second->action_traces ) { + on_action_trace( atrace, block_state ); + } + } + } + + cached_traces.clear(); +} + + +void statetrack_plugin_impl::on_action_trace( const action_trace& at, const block_state_ptr& bsp ) +{ + // check the action against the blacklist + auto search_acc = blacklist_actions.find(at.act.account); + if(search_acc != blacklist_actions.end()) { + auto search_act = search_acc->second.find(at.act.name); + if( search_act != search_acc->second.end() ) { + return; + } + } + + auto& chain = chain_plug->chain(); + + //TODO send action trace +} + + +void statetrack_plugin_impl::on_irreversible_block( const chain::block_state_ptr& bsp ) +{ + //TODO send irreversible block information +} + // Plugin implementation statetrack_plugin::statetrack_plugin():my(new statetrack_plugin_impl()){} @@ -410,7 +489,8 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { ilog("Bind to ZMQ PUSH socket successful"); chain_plugin& chain_plug = app().get_plugin(); - const chainbase::database& db = chain_plug.chain().db(); + auto& chain = my->chain_plug->chain(); + const chainbase::database& db = chain.db(); my->abi_serializer_max_time = chain_plug.get_abi_serializer_max_time(); @@ -499,6 +579,21 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { }; kv_index.applied_undo = undo_lambda; + + my->applied_transaction_connection.emplace( + chain.applied_transaction.connect([&]( const transaction_trace_ptr& ttp ) { + my->on_applied_transaction(ttp); + })); + + my->accepted_block_connection.emplace( + chain.accepted_block.connect([&](const block_state_ptr& bsp) { + my->on_accepted_block(bsp); + })); + + my->irreversible_block_connection.emplace( + chain.irreversible_block.connect([&]( const chain::block_state_ptr& bsp ) { + my->on_irreversible_block( bsp ); + })); } FC_LOG_AND_RETHROW() } From 73f47aac37115ca27fdfa72138eace3ad48ca413 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Mon, 5 Nov 2018 18:34:56 +0000 Subject: [PATCH 34/56] Create statetrack_plugin_impl.cpp --- plugins/statetrack_plugin/statetrack_plugin_impl.cpp | 1 + 1 file changed, 1 insertion(+) create mode 100644 plugins/statetrack_plugin/statetrack_plugin_impl.cpp diff --git a/plugins/statetrack_plugin/statetrack_plugin_impl.cpp b/plugins/statetrack_plugin/statetrack_plugin_impl.cpp new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/plugins/statetrack_plugin/statetrack_plugin_impl.cpp @@ -0,0 +1 @@ + From 817bd448e69d35bbccdfa898bb37dd21b1f512c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Mon, 5 Nov 2018 18:35:11 +0000 Subject: [PATCH 35/56] Create statetrack_plugin_impl.hpp --- .../include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp | 1 + 1 file changed, 1 insertion(+) create mode 100644 plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp diff --git a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp @@ -0,0 +1 @@ + From 3432cd6f4265069b8c785611fda6d8b6d9092660 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Mon, 5 Nov 2018 18:37:15 +0000 Subject: [PATCH 36/56] Update statetrack_plugin_impl.cpp --- .../statetrack_plugin_impl.cpp | 317 ++++++++++++++++++ 1 file changed, 317 insertions(+) diff --git a/plugins/statetrack_plugin/statetrack_plugin_impl.cpp b/plugins/statetrack_plugin/statetrack_plugin_impl.cpp index 8b137891791..a64c1246ea3 100644 --- a/plugins/statetrack_plugin/statetrack_plugin_impl.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin_impl.cpp @@ -1 +1,318 @@ +/** + * @file + * @copyright defined in eos/LICENSE.txt + */ +#include + +// statetrack plugin implementation + +void statetrack_plugin_impl::send_zmq_msg(const fc::string content) { + if(sender_socket == nullptr) + return; + + zmq::message_t message(content.length()); + memcpy(message.data(), content.c_str(), content.length()); + sender_socket->send(message); +} + +bool statetrack_plugin_impl::filter(const chain::account_name& code, const fc::string& scope, const chain::table_name& table) { + if(filter_on.size() > 0) { + bool pass_on = false; + if (filter_on.find({ code, 0, 0 }) != filter_on.end()) { + pass_on = true; + } + if (filter_on.find({ code, scope, 0 }) != filter_on.end()) { + pass_on = true; + } + if (filter_on.find({ code, scope, table }) != filter_on.end()) { + pass_on = true; + } + if (!pass_on) { return false; } + } + + if(filter_out.size() > 0) { + if (filter_out.find({ code, 0, 0 }) != filter_out.end()) { + return false; + } + if (filter_out.find({ code, scope, 0 }) != filter_out.end()) { + return false; + } + if (filter_out.find({ code, scope, table }) != filter_out.end()) { + return false; + } + } + + return true; +} + +const chain::table_id_object* statetrack_plugin_impl::get_kvo_tio(const chainbase::database& db, const chain::key_value_object& kvo) { + return db.find(kvo.t_id); +} + +const chain::account_object& statetrack_plugin_impl::get_tio_co(const chainbase::database& db, const chain::table_id_object& tio) { + const chain::account_object *co = db.find(tio.code); + EOS_ASSERT(co != nullptr, chain::account_query_exception, "Fail to retrieve account for ${code}", ("code", tio.code) ); + return *co; +} + +const abi_def& statetrack_plugin_impl::get_co_abi( const chain::account_object& co ) { + abi_def* abi = new abi_def(); + abi_serializer::to_abi(co.abi, *abi); + return *abi; +} + +fc::variant statetrack_plugin_impl::get_kvo_row(const chainbase::database& db, const chain::table_id_object& tio, const chain::key_value_object& kvo, bool json) { + vector data; + copy_inline_row(kvo, data); + + if (json) { + const chain::account_object& co = get_tio_co(db, tio); + const abi_def& abi = get_co_abi(co); + + abi_serializer abis; + abis.set_abi(abi, abi_serializer_max_time); + + return abis.binary_to_variant( abis.get_table_type(tio.table), data, abi_serializer_max_time, shorten_abi_errors ); + } else { + return fc::variant(data); + } +} + +db_op statetrack_plugin_impl::get_db_op(const chainbase::database& db, const chain::account_object& co, op_type_enum op_type) { + db_op op; + + name system = N(system); + + op.id = co.id._id; + op.op_type = op_type; + op.code = system; + op.scope = system.to_string(); + op.table = N(accounts); + + db_account account; + account.name = co.name; + account.vm_type = co.vm_type; + account.vm_version = co.vm_version; + account.privileged = co.privileged; + account.last_code_update = co.last_code_update; + account.code_version = co.code_version; + account.creation_date = co.creation_date; + + op.value = fc::variant(account); + + return op; +} + +db_op statetrack_plugin_impl::get_db_op(const chainbase::database& db, const chain::permission_object& po, op_type_enum op_type) { + db_op op; + + name system = N(system); + + op.id = po.id._id; + op.op_type = op_type; + op.code = system; + op.scope = system.to_string(); + op.table = N(permissions); + op.value = fc::variant(po); + + return op; +} + +db_op statetrack_plugin_impl::get_db_op(const chainbase::database& db, const chain::permission_link_object& plo, op_type_enum op_type) { + db_op op; + + name system = N(system); + + op.id = plo.id._id; + op.op_type = op_type; + op.code = system; + op.scope = system.to_string(); + op.table = N(permission_links); + op.value = fc::variant(plo); + + return op; +} + +db_op statetrack_plugin_impl::get_db_op(const chainbase::database& db, const chain::table_id_object& tio, const chain::key_value_object& kvo, op_type_enum op_type, bool json) { + db_op op; + + op.id = kvo.primary_key; + op.op_type = op_type; + op.code = tio.code; + op.scope = scope_sym_to_string(tio.scope); + op.table = tio.table; + + if(op_type == op_type_enum::ROW_CREATE || + op_type == op_type_enum::ROW_MODIFY) { + op.value = get_kvo_row(db, tio, kvo, json); + } + + return op; +} + +db_rev statetrack_plugin_impl::get_db_rev(const int64_t revision, op_type_enum op_type) { + db_rev rev = {op_type, revision}; + return rev; +} + +void statetrack_plugin_impl::on_applied_table(const chainbase::database& db, const chain::table_id_object& tio, op_type_enum op_type) { + if(sender_socket == nullptr) + return; + + auto code = tio.code; + auto scope = scope_sym_to_string(tio.scope); + auto table = tio.table; + + if(filter(code, scope, table)) { + db_table table; + + table.op_type = op_type; + table.code = tio.code; + table.scope = scope_sym_to_string(tio.scope); + table.table = tio.table; + + fc::string data = fc::json::to_string(table); + ilog("STATETRACK table_id_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); + send_zmq_msg(data); + } +} + +void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const chain::account_object& co, op_type_enum op_type) { + if(sender_socket == nullptr) + return; + + name system = N(system); + auto code = system; + auto scope = system.to_string(); + auto table = N(accounts); + + if(filter(code, scope, table)) { + fc::string data = fc::json::to_string(get_db_op(db, co, op_type)); + ilog("STATETRACK account_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); + send_zmq_msg(data); + } +} + +void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const chain::permission_object& po, op_type_enum op_type) { + if(sender_socket == nullptr) + return; + + name system = N(system); + auto code = system; + auto scope = system.to_string(); + auto table = N(permission); + + if(filter(code, scope, table)) { + fc::string data = fc::json::to_string(get_db_op(db, po, op_type)); + ilog("STATETRACK permission_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); + send_zmq_msg(data); + } +} + +void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const chain::permission_link_object& plo, op_type_enum op_type) { + if(sender_socket == nullptr) + return; + + name system = N(system); + auto code = system; + auto scope = system.to_string(); + auto table = N(permission_links); + + if(filter(code, scope, table)) { + fc::string data = fc::json::to_string(get_db_op(db, plo, op_type)); + ilog("STATETRACK permission_link_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); + send_zmq_msg(data); + } +} + +void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const chain::key_value_object& kvo, op_type_enum op_type) { + if(sender_socket == nullptr) + return; + + const chain::table_id_object* tio_ptr = get_kvo_tio(db, kvo); + + if(tio_ptr == nullptr) + return; + + const chain::table_id_object& tio = *tio_ptr; + + auto code = tio.code; + auto scope = scope_sym_to_string(tio.scope); + auto table = tio.table; + + if(filter(code, scope, table)) { + fc::string data = fc::json::to_string(get_db_op(db, *tio_ptr, kvo, op_type)); + ilog("STATETRACK key_value_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); + send_zmq_msg(data); + } +} + +void statetrack_plugin_impl::on_applied_rev(const int64_t revision, op_type_enum op_type) { + if(sender_socket == nullptr) + return; + + fc::string data = fc::json::to_string(get_db_rev(revision, op_type)); + ilog("STATETRACK ${op_type}: ${data}", ("op_type", op_type)("data", data)); + send_zmq_msg(data); +} + +void statetrack_plugin_impl::on_applied_transaction( const transaction_trace_ptr& ttp ) +{ + if (ttp->receipt) { + cached_traces[ttp->id] = ttp; + } +} + +void statetrack_plugin_impl::on_accepted_block(const block_state_ptr& bsp) +{ + //TODO send accepted block information + + for (auto& t : bsp->block->transactions) { + transaction_id_type id; + if (t.trx.contains()) { + id = t.trx.get(); + } + else { + id = t.trx.get().id(); + } + + if( t.status == transaction_receipt_header::executed ) { + // Send traces only for executed transactions + auto it = cached_traces.find(id); + if (it == cached_traces.end() || !it->second->receipt) { + ilog("missing trace for transaction {id}", ("id", id)); + continue; + } + + for( const auto& atrace : it->second->action_traces ) { + on_action_trace( atrace, block_state ); + } + } + } + + cached_traces.clear(); +} + + +void statetrack_plugin_impl::on_action_trace( const action_trace& at, const block_state_ptr& bsp ) +{ + // check the action against the blacklist + auto search_acc = blacklist_actions.find(at.act.account); + if(search_acc != blacklist_actions.end()) { + auto search_act = search_acc->second.find(at.act.name); + if( search_act != search_acc->second.end() ) { + return; + } + } + + auto& chain = chain_plug->chain(); + + //TODO send action trace +} + + +void statetrack_plugin_impl::on_irreversible_block( const chain::block_state_ptr& bsp ) +{ + //TODO send irreversible block information +} From 0fca16153f6007e74199b48f28a89132d94e08d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Mon, 5 Nov 2018 18:39:20 +0000 Subject: [PATCH 37/56] Update statetrack_plugin.cpp --- .../statetrack_plugin/statetrack_plugin.cpp | 415 ------------------ 1 file changed, 415 deletions(-) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index 23366560280..13c13582e2a 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -3,10 +3,7 @@ * @copyright defined in eos/LICENSE.txt */ -#include -#include #include -#include namespace { const char* SENDER_BIND = "st-zmq-sender-bind"; @@ -17,418 +14,6 @@ namespace eosio { static appbase::abstract_plugin& _statetrack_plugin = app().register_plugin(); -typedef typename chainbase::get_index_type::type co_index_type; -typedef typename chainbase::get_index_type::type po_index_type; -typedef typename chainbase::get_index_type::type plo_index_type; -typedef typename chainbase::get_index_type::type ti_index_type; -typedef typename chainbase::get_index_type::type kv_index_type; - -struct filter_entry { - name code; - name scope; - name table; - - std::tuple key() const { - return std::make_tuple(code, scope, table); - } - - friend bool operator<( const filter_entry& a, const filter_entry& b ) { - return a.key() < b.key(); - } - }; - -class statetrack_plugin_impl { - public: - statetrack_plugin_impl() { - blacklist_actions.emplace - (std::make_pair(chain::config::system_account_name, - std::set{ N(onblock) } )); - blacklist_actions.emplace - (std::make_pair(N(blocktwitter), - std::set{ N(tweet) } )); - } - - void send_zmq_msg(const string content); - bool filter(const chain::account_name& code, const fc::string& scope, const chain::table_name& table); - - const chain::table_id_object* get_kvo_tio(const chainbase::database& db, const chain::key_value_object& kvo); - const chain::account_object& get_tio_co(const chainbase::database& db, const chain::table_id_object& tio); - const abi_def& get_co_abi(const chain::account_object& co ); - fc::variant get_kvo_row(const chainbase::database& db, const chain::table_id_object& tio, const chain::key_value_object& kvo, bool json = true); - - db_op get_db_op(const chainbase::database& db, const chain::account_object& co, op_type_enum op_type); - db_op get_db_op(const chainbase::database& db, const chain::permission_object& po, op_type_enum op_type); - db_op get_db_op(const chainbase::database& db, const chain::permission_link_object& plo, op_type_enum op_type); - db_op get_db_op(const chainbase::database& db, const chain::table_id_object& tio, const chain::key_value_object& kvo, op_type_enum op_type, bool json = true); - - db_rev get_db_rev(const int64_t revision, op_type_enum op_type); - - //generic_index state tables - void on_applied_table(const chainbase::database& db, const chain::table_id_object& tio, op_type_enum op_type); - //generic_index op - void on_applied_op(const chainbase::database& db, const chain::account_object& co, op_type_enum op_type); - void on_applied_op(const chainbase::database& db, const chain::permission_object& po, op_type_enum op_type); - void on_applied_op(const chainbase::database& db, const chain::permission_link_object& plo, op_type_enum op_type); - void on_applied_op(const chainbase::database& db, const chain::key_value_object& kvo, op_type_enum op_type); - //generic_index undo - void on_applied_rev(const int64_t revision, op_type_enum op_type); - //blocks and transactions - void on_applied_transaction( const transaction_trace_ptr& ttp ); - void on_accepted_block(const block_state_ptr& bsp); - void on_action_trace( const action_trace& at, const block_state_ptr& bsp ); - void on_irreversible_block( const chain::block_state_ptr& bsp ) - - static void copy_inline_row(const chain::key_value_object& obj, vector& data) { - data.resize( obj.value.size() ); - memcpy( data.data(), obj.value.data(), obj.value.size() ); - } - - static fc::string scope_sym_to_string(chain::scope_name sym_code) { - fc::string scope = sym_code.to_string(); - if(scope.length() > 0 && scope[0] == '.') { - uint64_t scope_int = sym_code; - vector v; - for( int i = 0; i < 7; ++i ) { - char c = (char)(scope_int & 0xff); - if( !c ) break; - v.emplace_back(c); - scope_int >>= 8; - } - return fc::string(v.begin(),v.end()); - } - return scope; - } - - fc::string socket_bind_str; - zmq::context_t* context; - zmq::socket_t* sender_socket; - - std::map cached_traces; - std::map> blacklist_actions; - - std::set filter_on; - std::set filter_out; - fc::microseconds abi_serializer_max_time; - - fc::optional applied_transaction_connection; - fc::optional accepted_block_connection; - fc::optional irreversible_block_connection; - private: - bool shorten_abi_errors = true; -}; - -// statetrack plugin implementation - -void statetrack_plugin_impl::send_zmq_msg(const fc::string content) { - if(sender_socket == nullptr) - return; - - zmq::message_t message(content.length()); - memcpy(message.data(), content.c_str(), content.length()); - sender_socket->send(message); -} - -bool statetrack_plugin_impl::filter(const chain::account_name& code, const fc::string& scope, const chain::table_name& table) { - if(filter_on.size() > 0) { - bool pass_on = false; - if (filter_on.find({ code, 0, 0 }) != filter_on.end()) { - pass_on = true; - } - if (filter_on.find({ code, scope, 0 }) != filter_on.end()) { - pass_on = true; - } - if (filter_on.find({ code, scope, table }) != filter_on.end()) { - pass_on = true; - } - if (!pass_on) { return false; } - } - - if(filter_out.size() > 0) { - if (filter_out.find({ code, 0, 0 }) != filter_out.end()) { - return false; - } - if (filter_out.find({ code, scope, 0 }) != filter_out.end()) { - return false; - } - if (filter_out.find({ code, scope, table }) != filter_out.end()) { - return false; - } - } - - return true; -} - -const chain::table_id_object* statetrack_plugin_impl::get_kvo_tio(const chainbase::database& db, const chain::key_value_object& kvo) { - return db.find(kvo.t_id); -} - -const chain::account_object& statetrack_plugin_impl::get_tio_co(const chainbase::database& db, const chain::table_id_object& tio) { - const chain::account_object *co = db.find(tio.code); - EOS_ASSERT(co != nullptr, chain::account_query_exception, "Fail to retrieve account for ${code}", ("code", tio.code) ); - return *co; -} - -const abi_def& statetrack_plugin_impl::get_co_abi( const chain::account_object& co ) { - abi_def* abi = new abi_def(); - abi_serializer::to_abi(co.abi, *abi); - return *abi; -} - -fc::variant statetrack_plugin_impl::get_kvo_row(const chainbase::database& db, const chain::table_id_object& tio, const chain::key_value_object& kvo, bool json) { - vector data; - copy_inline_row(kvo, data); - - if (json) { - const chain::account_object& co = get_tio_co(db, tio); - const abi_def& abi = get_co_abi(co); - - abi_serializer abis; - abis.set_abi(abi, abi_serializer_max_time); - - return abis.binary_to_variant( abis.get_table_type(tio.table), data, abi_serializer_max_time, shorten_abi_errors ); - } else { - return fc::variant(data); - } -} - -db_op statetrack_plugin_impl::get_db_op(const chainbase::database& db, const chain::account_object& co, op_type_enum op_type) { - db_op op; - - name system = N(system); - - op.id = co.id._id; - op.op_type = op_type; - op.code = system; - op.scope = system.to_string(); - op.table = N(accounts); - - db_account account; - account.name = co.name; - account.vm_type = co.vm_type; - account.vm_version = co.vm_version; - account.privileged = co.privileged; - account.last_code_update = co.last_code_update; - account.code_version = co.code_version; - account.creation_date = co.creation_date; - - op.value = fc::variant(account); - - return op; -} - -db_op statetrack_plugin_impl::get_db_op(const chainbase::database& db, const chain::permission_object& po, op_type_enum op_type) { - db_op op; - - name system = N(system); - - op.id = po.id._id; - op.op_type = op_type; - op.code = system; - op.scope = system.to_string(); - op.table = N(permissions); - op.value = fc::variant(po); - - return op; -} - -db_op statetrack_plugin_impl::get_db_op(const chainbase::database& db, const chain::permission_link_object& plo, op_type_enum op_type) { - db_op op; - - name system = N(system); - - op.id = plo.id._id; - op.op_type = op_type; - op.code = system; - op.scope = system.to_string(); - op.table = N(permission_links); - op.value = fc::variant(plo); - - return op; -} - -db_op statetrack_plugin_impl::get_db_op(const chainbase::database& db, const chain::table_id_object& tio, const chain::key_value_object& kvo, op_type_enum op_type, bool json) { - db_op op; - - op.id = kvo.primary_key; - op.op_type = op_type; - op.code = tio.code; - op.scope = scope_sym_to_string(tio.scope); - op.table = tio.table; - - if(op_type == op_type_enum::ROW_CREATE || - op_type == op_type_enum::ROW_MODIFY) { - op.value = get_kvo_row(db, tio, kvo, json); - } - - return op; -} - -db_rev statetrack_plugin_impl::get_db_rev(const int64_t revision, op_type_enum op_type) { - db_rev rev = {op_type, revision}; - return rev; -} - -void statetrack_plugin_impl::on_applied_table(const chainbase::database& db, const chain::table_id_object& tio, op_type_enum op_type) { - if(sender_socket == nullptr) - return; - - auto code = tio.code; - auto scope = scope_sym_to_string(tio.scope); - auto table = tio.table; - - if(filter(code, scope, table)) { - db_table table; - - table.op_type = op_type; - table.code = tio.code; - table.scope = scope_sym_to_string(tio.scope); - table.table = tio.table; - - fc::string data = fc::json::to_string(table); - ilog("STATETRACK table_id_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); - send_zmq_msg(data); - } -} - -void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const chain::account_object& co, op_type_enum op_type) { - if(sender_socket == nullptr) - return; - - name system = N(system); - auto code = system; - auto scope = system.to_string(); - auto table = N(accounts); - - if(filter(code, scope, table)) { - fc::string data = fc::json::to_string(get_db_op(db, co, op_type)); - ilog("STATETRACK account_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); - send_zmq_msg(data); - } -} - -void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const chain::permission_object& po, op_type_enum op_type) { - if(sender_socket == nullptr) - return; - - name system = N(system); - auto code = system; - auto scope = system.to_string(); - auto table = N(permission); - - if(filter(code, scope, table)) { - fc::string data = fc::json::to_string(get_db_op(db, po, op_type)); - ilog("STATETRACK permission_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); - send_zmq_msg(data); - } -} - -void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const chain::permission_link_object& plo, op_type_enum op_type) { - if(sender_socket == nullptr) - return; - - name system = N(system); - auto code = system; - auto scope = system.to_string(); - auto table = N(permission_links); - - if(filter(code, scope, table)) { - fc::string data = fc::json::to_string(get_db_op(db, plo, op_type)); - ilog("STATETRACK permission_link_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); - send_zmq_msg(data); - } -} - -void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const chain::key_value_object& kvo, op_type_enum op_type) { - if(sender_socket == nullptr) - return; - - const chain::table_id_object* tio_ptr = get_kvo_tio(db, kvo); - - if(tio_ptr == nullptr) - return; - - const chain::table_id_object& tio = *tio_ptr; - - auto code = tio.code; - auto scope = scope_sym_to_string(tio.scope); - auto table = tio.table; - - if(filter(code, scope, table)) { - fc::string data = fc::json::to_string(get_db_op(db, *tio_ptr, kvo, op_type)); - ilog("STATETRACK key_value_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); - send_zmq_msg(data); - } -} - -void statetrack_plugin_impl::on_applied_rev(const int64_t revision, op_type_enum op_type) { - if(sender_socket == nullptr) - return; - - fc::string data = fc::json::to_string(get_db_rev(revision, op_type)); - ilog("STATETRACK ${op_type}: ${data}", ("op_type", op_type)("data", data)); - send_zmq_msg(data); -} - -void statetrack_plugin_impl::on_applied_transaction( const transaction_trace_ptr& ttp ) -{ - if (ttp->receipt) { - cached_traces[ttp->id] = ttp; - } -} - -void statetrack_plugin_impl::on_accepted_block(const block_state_ptr& bsp) -{ - //TODO send accepted block information - - for (auto& t : bsp->block->transactions) { - transaction_id_type id; - if (t.trx.contains()) { - id = t.trx.get(); - } - else { - id = t.trx.get().id(); - } - - if( t.status == transaction_receipt_header::executed ) { - // Send traces only for executed transactions - auto it = cached_traces.find(id); - if (it == cached_traces.end() || !it->second->receipt) { - ilog("missing trace for transaction {id}", ("id", id)); - continue; - } - - for( const auto& atrace : it->second->action_traces ) { - on_action_trace( atrace, block_state ); - } - } - } - - cached_traces.clear(); -} - - -void statetrack_plugin_impl::on_action_trace( const action_trace& at, const block_state_ptr& bsp ) -{ - // check the action against the blacklist - auto search_acc = blacklist_actions.find(at.act.account); - if(search_acc != blacklist_actions.end()) { - auto search_act = search_acc->second.find(at.act.name); - if( search_act != search_acc->second.end() ) { - return; - } - } - - auto& chain = chain_plug->chain(); - - //TODO send action trace -} - - -void statetrack_plugin_impl::on_irreversible_block( const chain::block_state_ptr& bsp ) -{ - //TODO send irreversible block information -} - // Plugin implementation statetrack_plugin::statetrack_plugin():my(new statetrack_plugin_impl()){} From a1019d9c3ffc7ba5cddcd31be0cc54e30ffb2a7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Mon, 5 Nov 2018 18:42:08 +0000 Subject: [PATCH 38/56] Update statetrack_plugin_impl.hpp --- .../statetrack_plugin_impl.hpp | 170 ++++++++++++++++++ 1 file changed, 170 insertions(+) diff --git a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp index 8b137891791..9e6f812e0f3 100644 --- a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp +++ b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp @@ -1 +1,171 @@ +/** + * @file + * @copyright defined in eos/LICENSE.txt + */ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace eosio { + +typedef typename chainbase::get_index_type::type co_index_type; +typedef typename chainbase::get_index_type::type po_index_type; +typedef typename chainbase::get_index_type::type plo_index_type; +typedef typename chainbase::get_index_type::type ti_index_type; +typedef typename chainbase::get_index_type::type kv_index_type; + +typedef chain::account_object::id_type co_id_type; +typedef chain::permission_usage_object::id_type puo_id_type; +typedef chain::permission_object::id_type po_id_type; +typedef chain::permission_link_object::id_type plo_id_type; +typedef chain::key_value_object::id_type kvo_id_type; + + +enum op_type_enum { + TABLE_REMOVE = 0, + ROW_CREATE = 1, + ROW_MODIFY = 2, + ROW_REMOVE = 3, + REV_UNDO = 4, + REV_COMMIT = 5, +}; + +struct db_account { + account_name name; + uint8_t vm_type = 0; + uint8_t vm_version = 0; + bool privileged = false; + + chain::time_point last_code_update; + chain::digest_type code_version; + chain::block_timestamp_type creation_date; +}; + +struct db_table { + op_type_enum op_type; + account_name code; + fc::string scope; + chain::table_name table; + account_name payer; +}; + +struct db_op : db_table { + uint64_t id; + fc::variant value; +}; + +struct db_rev { + op_type_enum op_type; + int64_t revision; +}; + +struct filter_entry { + name code; + name scope; + name table; + + std::tuple key() const { + return std::make_tuple(code, scope, table); + } + + friend bool operator<( const filter_entry& a, const filter_entry& b ) { + return a.key() < b.key(); + } + }; + +class statetrack_plugin_impl { + public: + statetrack_plugin_impl() { + blacklist_actions.emplace + (std::make_pair(chain::config::system_account_name, + std::set{ N(onblock) } )); + blacklist_actions.emplace + (std::make_pair(N(blocktwitter), + std::set{ N(tweet) } )); + } + + void send_zmq_msg(const string content); + bool filter(const chain::account_name& code, const fc::string& scope, const chain::table_name& table); + + const chain::table_id_object* get_kvo_tio(const chainbase::database& db, const chain::key_value_object& kvo); + const chain::account_object& get_tio_co(const chainbase::database& db, const chain::table_id_object& tio); + const abi_def& get_co_abi(const chain::account_object& co ); + fc::variant get_kvo_row(const chainbase::database& db, const chain::table_id_object& tio, const chain::key_value_object& kvo, bool json = true); + + db_op get_db_op(const chainbase::database& db, const chain::account_object& co, op_type_enum op_type); + db_op get_db_op(const chainbase::database& db, const chain::permission_object& po, op_type_enum op_type); + db_op get_db_op(const chainbase::database& db, const chain::permission_link_object& plo, op_type_enum op_type); + db_op get_db_op(const chainbase::database& db, const chain::table_id_object& tio, const chain::key_value_object& kvo, op_type_enum op_type, bool json = true); + + db_rev get_db_rev(const int64_t revision, op_type_enum op_type); + + //generic_index state tables + void on_applied_table(const chainbase::database& db, const chain::table_id_object& tio, op_type_enum op_type); + //generic_index op + void on_applied_op(const chainbase::database& db, const chain::account_object& co, op_type_enum op_type); + void on_applied_op(const chainbase::database& db, const chain::permission_object& po, op_type_enum op_type); + void on_applied_op(const chainbase::database& db, const chain::permission_link_object& plo, op_type_enum op_type); + void on_applied_op(const chainbase::database& db, const chain::key_value_object& kvo, op_type_enum op_type); + //generic_index undo + void on_applied_rev(const int64_t revision, op_type_enum op_type); + //blocks and transactions + void on_applied_transaction( const transaction_trace_ptr& ttp ); + void on_accepted_block(const block_state_ptr& bsp); + void on_action_trace( const action_trace& at, const block_state_ptr& bsp ); + void on_irreversible_block( const chain::block_state_ptr& bsp ) + + static void copy_inline_row(const chain::key_value_object& obj, vector& data) { + data.resize( obj.value.size() ); + memcpy( data.data(), obj.value.data(), obj.value.size() ); + } + + static fc::string scope_sym_to_string(chain::scope_name sym_code) { + fc::string scope = sym_code.to_string(); + if(scope.length() > 0 && scope[0] == '.') { + uint64_t scope_int = sym_code; + vector v; + for( int i = 0; i < 7; ++i ) { + char c = (char)(scope_int & 0xff); + if( !c ) break; + v.emplace_back(c); + scope_int >>= 8; + } + return fc::string(v.begin(),v.end()); + } + return scope; + } + + fc::string socket_bind_str; + zmq::context_t* context; + zmq::socket_t* sender_socket; + + std::map cached_traces; + std::map> blacklist_actions; + + std::set filter_on; + std::set filter_out; + fc::microseconds abi_serializer_max_time; + + fc::optional applied_transaction_connection; + fc::optional accepted_block_connection; + fc::optional irreversible_block_connection; + private: + bool shorten_abi_errors = true; +}; + +} + +FC_REFLECT_ENUM( eosio::op_type_enum, (TABLE_REMOVE)(ROW_CREATE)(ROW_MODIFY)(ROW_REMOVE)(REV_UNDO)(REV_COMMIT) ) +FC_REFLECT( eosio::db_account, (name)(vm_type)(vm_version)(privileged)(last_code_update)(code_version)(creation_date) ) +FC_REFLECT( eosio::db_table, (op_type)(code)(scope)(table)(payer) ) +FC_REFLECT_DERIVED( eosio::db_op, (eosio::db_table), (id)(value) ) +FC_REFLECT( eosio::db_rev, (op_type)(revision) ) From 3d8744706d8c729b2ec68e61a0f934e69de44345 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Mon, 5 Nov 2018 18:42:31 +0000 Subject: [PATCH 39/56] Update statetrack_plugin.hpp --- .../statetrack_plugin/statetrack_plugin.hpp | 58 ------------------- 1 file changed, 58 deletions(-) diff --git a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp index f43952e26bf..1357e1ef359 100644 --- a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp +++ b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp @@ -4,65 +4,12 @@ */ #pragma once -#include -#include -#include -#include -#include -#include -#include - #include namespace eosio { using namespace appbase; -typedef chain::account_object::id_type co_id_type; -typedef chain::permission_usage_object::id_type puo_id_type; -typedef chain::permission_object::id_type po_id_type; -typedef chain::permission_link_object::id_type plo_id_type; -typedef chain::key_value_object::id_type kvo_id_type; - - -enum op_type_enum { - TABLE_REMOVE = 0, - ROW_CREATE = 1, - ROW_MODIFY = 2, - ROW_REMOVE = 3, - REV_UNDO = 4, - REV_COMMIT = 5, -}; - -struct db_account { - account_name name; - uint8_t vm_type = 0; - uint8_t vm_version = 0; - bool privileged = false; - - chain::time_point last_code_update; - chain::digest_type code_version; - chain::block_timestamp_type creation_date; -}; - -struct db_table { - op_type_enum op_type; - account_name code; - fc::string scope; - chain::table_name table; - account_name payer; -}; - -struct db_op : db_table { - uint64_t id; - fc::variant value; -}; - -struct db_rev { - op_type_enum op_type; - int64_t revision; -}; - class statetrack_plugin : public plugin { public: APPBASE_PLUGIN_REQUIRES((chain_plugin)) @@ -85,8 +32,3 @@ class statetrack_plugin : public plugin { } -FC_REFLECT_ENUM( eosio::op_type_enum, (TABLE_REMOVE)(ROW_CREATE)(ROW_MODIFY)(ROW_REMOVE)(REV_UNDO)(REV_COMMIT) ) -FC_REFLECT( eosio::db_account, (name)(vm_type)(vm_version)(privileged)(last_code_update)(code_version)(creation_date) ) -FC_REFLECT( eosio::db_table, (op_type)(code)(scope)(table)(payer) ) -FC_REFLECT_DERIVED( eosio::db_op, (eosio::db_table), (id)(value) ) -FC_REFLECT( eosio::db_rev, (op_type)(revision) ) \ No newline at end of file From 77e1dbfb657104c24750aadc7b5a728282931b0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Mon, 5 Nov 2018 18:43:00 +0000 Subject: [PATCH 40/56] Update statetrack_plugin_impl.cpp --- plugins/statetrack_plugin/statetrack_plugin_impl.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/statetrack_plugin/statetrack_plugin_impl.cpp b/plugins/statetrack_plugin/statetrack_plugin_impl.cpp index a64c1246ea3..d04e8a73309 100644 --- a/plugins/statetrack_plugin/statetrack_plugin_impl.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin_impl.cpp @@ -2,7 +2,6 @@ * @file * @copyright defined in eos/LICENSE.txt */ - #include // statetrack plugin implementation From f702524674947fcb3a76660616a8eafaf6aefbd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Mon, 5 Nov 2018 18:44:54 +0000 Subject: [PATCH 41/56] Update statetrack_plugin.cpp --- plugins/statetrack_plugin/statetrack_plugin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index 13c13582e2a..a8488f64896 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -2,8 +2,8 @@ * @file * @copyright defined in eos/LICENSE.txt */ - #include +#include namespace { const char* SENDER_BIND = "st-zmq-sender-bind"; From 7ce6e94e1853a7772c4c00899d2553737d2659a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Mon, 5 Nov 2018 18:45:47 +0000 Subject: [PATCH 42/56] Update CMakeLists.txt --- plugins/statetrack_plugin/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/statetrack_plugin/CMakeLists.txt b/plugins/statetrack_plugin/CMakeLists.txt index 8bfdca2d36c..b22ef7d0fb1 100644 --- a/plugins/statetrack_plugin/CMakeLists.txt +++ b/plugins/statetrack_plugin/CMakeLists.txt @@ -22,7 +22,8 @@ message(STATUS "[Additional Plugin] EOSIO ZeroMQ plugin enabled") file(GLOB HEADERS "include/eosio/statetrack_plugin/*.hpp") add_library( statetrack_plugin statetrack_plugin.cpp + statetrack_plugin_impl.cpp ${HEADERS} ) target_link_libraries( statetrack_plugin ${ZeroMQ_LIBRARY} chain_plugin appbase fc ) -target_include_directories( statetrack_plugin PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) \ No newline at end of file +target_include_directories( statetrack_plugin PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) From 10b203e8fb355c8b61f5d5588243591f2b2e5d5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Mon, 5 Nov 2018 18:47:35 +0000 Subject: [PATCH 43/56] Update README.md --- plugins/statetrack_plugin/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/statetrack_plugin/README.md b/plugins/statetrack_plugin/README.md index 364b15ae32c..1f0c2b6244c 100644 --- a/plugins/statetrack_plugin/README.md +++ b/plugins/statetrack_plugin/README.md @@ -1,6 +1,9 @@ # statetrack_plugin -This plugin adds hooks on chainbase operations emplace, modify, remove, undo, squash and commit and routes the objects and revision numbers to a ZMQ queue. +This plugin adds hooks on chainbase operations emplace, modify, remove, undo and routes the objects and revision numbers to a ZMQ queue. +Also hooks blocks and transactions to the ZMQ queue. + +This is based on plugin: https://github.com/cc32d9/eos_zmq_plugin ### Configuration From d992bed60bd1bcd14849d0d827c3ba775f973600 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Tue, 6 Nov 2018 00:39:55 +0000 Subject: [PATCH 44/56] dev action traces --- libraries/chainbase | 2 +- .../statetrack_plugin/statetrack_plugin.hpp | 1 + .../statetrack_plugin_impl.hpp | 119 ++++++++++-------- .../statetrack_plugin/statetrack_plugin.cpp | 39 +++--- .../statetrack_plugin_impl.cpp | 74 +++++++---- 5 files changed, 141 insertions(+), 94 deletions(-) diff --git a/libraries/chainbase b/libraries/chainbase index 2412f00982a..cda29335fd6 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit 2412f00982af0227abfd35810f2131e24c53d459 +Subproject commit cda29335fd622d7a5d1fd3ee86f9dcd083d0ab95 diff --git a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp index 1357e1ef359..ead0fcada78 100644 --- a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp +++ b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin.hpp @@ -5,6 +5,7 @@ #pragma once #include +#include namespace eosio { diff --git a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp index 9e6f812e0f3..bc23a6d3a78 100644 --- a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp +++ b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp @@ -13,21 +13,29 @@ #include #include #include +#include +#include #include + namespace eosio { -typedef typename chainbase::get_index_type::type co_index_type; -typedef typename chainbase::get_index_type::type po_index_type; -typedef typename chainbase::get_index_type::type plo_index_type; -typedef typename chainbase::get_index_type::type ti_index_type; -typedef typename chainbase::get_index_type::type kv_index_type; +using namespace chain; +using namespace chainbase; +using boost::signals2::scoped_connection; + + +typedef typename get_index_type::type co_index_type; +typedef typename get_index_type::type po_index_type; +typedef typename get_index_type::type plo_index_type; +typedef typename get_index_type::type ti_index_type; +typedef typename get_index_type::type kv_index_type; -typedef chain::account_object::id_type co_id_type; -typedef chain::permission_usage_object::id_type puo_id_type; -typedef chain::permission_object::id_type po_id_type; -typedef chain::permission_link_object::id_type plo_id_type; -typedef chain::key_value_object::id_type kvo_id_type; +typedef account_object::id_type co_id_type; +typedef permission_usage_object::id_type puo_id_type; +typedef permission_object::id_type po_id_type; +typedef permission_link_object::id_type plo_id_type; +typedef key_value_object::id_type kvo_id_type; enum op_type_enum { @@ -35,32 +43,33 @@ enum op_type_enum { ROW_CREATE = 1, ROW_MODIFY = 2, ROW_REMOVE = 3, - REV_UNDO = 4, - REV_COMMIT = 5, + TRX_ACTION = 4, + REV_UNDO = 5, + REV_COMMIT = 6 }; struct db_account { - account_name name; - uint8_t vm_type = 0; - uint8_t vm_version = 0; - bool privileged = false; - - chain::time_point last_code_update; - chain::digest_type code_version; - chain::block_timestamp_type creation_date; + account_name name; + uint8_t vm_type = 0; + uint8_t vm_version = 0; + bool privileged = false; + + time_point last_code_update; + digest_type code_version; + block_timestamp_type creation_date; }; struct db_table { - op_type_enum op_type; - account_name code; - fc::string scope; - chain::table_name table; - account_name payer; + op_type_enum op_type; + account_name code; + fc::string scope; + table_name table; + account_name payer; }; struct db_op : db_table { - uint64_t id; - fc::variant value; + uint64_t id; + fc::variant value; }; struct db_rev { @@ -68,6 +77,14 @@ struct db_rev { int64_t revision; }; +struct db_action { + uint64_t global_action_seq; + block_num_type block_num; + block_timestamp_type block_time; + fc::variant action_trace; + uint32_t last_irreversible_block; +}; + struct filter_entry { name code; name scope; @@ -86,49 +103,49 @@ class statetrack_plugin_impl { public: statetrack_plugin_impl() { blacklist_actions.emplace - (std::make_pair(chain::config::system_account_name, + (std::make_pair(config::system_account_name, std::set{ N(onblock) } )); blacklist_actions.emplace (std::make_pair(N(blocktwitter), std::set{ N(tweet) } )); } - void send_zmq_msg(const string content); - bool filter(const chain::account_name& code, const fc::string& scope, const chain::table_name& table); + void send_zmq_msg(const fc::string content); + bool filter(const account_name& code, const fc::string& scope, const table_name& table); - const chain::table_id_object* get_kvo_tio(const chainbase::database& db, const chain::key_value_object& kvo); - const chain::account_object& get_tio_co(const chainbase::database& db, const chain::table_id_object& tio); - const abi_def& get_co_abi(const chain::account_object& co ); - fc::variant get_kvo_row(const chainbase::database& db, const chain::table_id_object& tio, const chain::key_value_object& kvo, bool json = true); + const table_id_object* get_kvo_tio(const database& db, const key_value_object& kvo); + const account_object& get_tio_co(const database& db, const table_id_object& tio); + const abi_def& get_co_abi(const account_object& co ); + fc::variant get_kvo_row(const database& db, const table_id_object& tio, const key_value_object& kvo, bool json = true); - db_op get_db_op(const chainbase::database& db, const chain::account_object& co, op_type_enum op_type); - db_op get_db_op(const chainbase::database& db, const chain::permission_object& po, op_type_enum op_type); - db_op get_db_op(const chainbase::database& db, const chain::permission_link_object& plo, op_type_enum op_type); - db_op get_db_op(const chainbase::database& db, const chain::table_id_object& tio, const chain::key_value_object& kvo, op_type_enum op_type, bool json = true); + db_op get_db_op(const database& db, const account_object& co, op_type_enum op_type); + db_op get_db_op(const database& db, const permission_object& po, op_type_enum op_type); + db_op get_db_op(const database& db, const permission_link_object& plo, op_type_enum op_type); + db_op get_db_op(const database& db, const table_id_object& tio, const key_value_object& kvo, op_type_enum op_type, bool json = true); db_rev get_db_rev(const int64_t revision, op_type_enum op_type); //generic_index state tables - void on_applied_table(const chainbase::database& db, const chain::table_id_object& tio, op_type_enum op_type); + void on_applied_table(const database& db, const table_id_object& tio, op_type_enum op_type); //generic_index op - void on_applied_op(const chainbase::database& db, const chain::account_object& co, op_type_enum op_type); - void on_applied_op(const chainbase::database& db, const chain::permission_object& po, op_type_enum op_type); - void on_applied_op(const chainbase::database& db, const chain::permission_link_object& plo, op_type_enum op_type); - void on_applied_op(const chainbase::database& db, const chain::key_value_object& kvo, op_type_enum op_type); + void on_applied_op(const database& db, const account_object& co, op_type_enum op_type); + void on_applied_op(const database& db, const permission_object& po, op_type_enum op_type); + void on_applied_op(const database& db, const permission_link_object& plo, op_type_enum op_type); + void on_applied_op(const database& db, const key_value_object& kvo, op_type_enum op_type); //generic_index undo void on_applied_rev(const int64_t revision, op_type_enum op_type); //blocks and transactions void on_applied_transaction( const transaction_trace_ptr& ttp ); void on_accepted_block(const block_state_ptr& bsp); void on_action_trace( const action_trace& at, const block_state_ptr& bsp ); - void on_irreversible_block( const chain::block_state_ptr& bsp ) + void on_irreversible_block( const block_state_ptr& bsp ); - static void copy_inline_row(const chain::key_value_object& obj, vector& data) { + static void copy_inline_row(const key_value_object& obj, vector& data) { data.resize( obj.value.size() ); memcpy( data.data(), obj.value.data(), obj.value.size() ); } - static fc::string scope_sym_to_string(chain::scope_name sym_code) { + static fc::string scope_sym_to_string(scope_name sym_code) { fc::string scope = sym_code.to_string(); if(scope.length() > 0 && scope[0] == '.') { uint64_t scope_int = sym_code; @@ -144,11 +161,12 @@ class statetrack_plugin_impl { return scope; } - fc::string socket_bind_str; + chain_plugin* chain_plug = nullptr; + fc::string socket_bind_str; zmq::context_t* context; - zmq::socket_t* sender_socket; + zmq::socket_t* sender_socket; - std::map cached_traces; + std::map cached_traces; std::map> blacklist_actions; std::set filter_on; @@ -164,8 +182,9 @@ class statetrack_plugin_impl { } -FC_REFLECT_ENUM( eosio::op_type_enum, (TABLE_REMOVE)(ROW_CREATE)(ROW_MODIFY)(ROW_REMOVE)(REV_UNDO)(REV_COMMIT) ) +FC_REFLECT_ENUM( eosio::op_type_enum, (TABLE_REMOVE)(ROW_CREATE)(ROW_MODIFY)(ROW_REMOVE)(TRX_ACTION)(REV_UNDO)(REV_COMMIT) ) FC_REFLECT( eosio::db_account, (name)(vm_type)(vm_version)(privileged)(last_code_update)(code_version)(creation_date) ) FC_REFLECT( eosio::db_table, (op_type)(code)(scope)(table)(payer) ) FC_REFLECT_DERIVED( eosio::db_op, (eosio::db_table), (id)(value) ) FC_REFLECT( eosio::db_rev, (op_type)(revision) ) +FC_REFLECT( eosio::db_action, (block_num)(block_time)(action_trace)(last_irreversible_block) ) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index a8488f64896..14e97ba7e46 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -12,6 +12,9 @@ namespace { namespace eosio { +using namespace chain; +using namespace chainbase; + static appbase::abstract_plugin& _statetrack_plugin = app().register_plugin(); // Plugin implementation @@ -67,17 +70,17 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { my->context = new zmq::context_t(1); my->sender_socket = new zmq::socket_t(*my->context, ZMQ_PUSH); + my->chain_plug = app().find_plugin(); ilog("Bind to ZMQ PUSH socket ${u}", ("u", my->socket_bind_str)); my->sender_socket->bind(my->socket_bind_str); ilog("Bind to ZMQ PUSH socket successful"); - - chain_plugin& chain_plug = app().get_plugin(); + auto& chain = my->chain_plug->chain(); - const chainbase::database& db = chain.db(); + const database& db = chain.db(); - my->abi_serializer_max_time = chain_plug.get_abi_serializer_max_time(); + my->abi_serializer_max_time = my->chain_plug->get_abi_serializer_max_time(); ilog("Binding database events"); @@ -89,15 +92,15 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { auto& co_inde = db.get_index(); - co_inde.applied_emplace = [&](const chain::account_object& co) { + co_inde.applied_emplace = [&](const account_object& co) { my->on_applied_op(db, co, op_type_enum::ROW_CREATE); }; - co_inde.applied_modify = [&](const chain::account_object& co) { + co_inde.applied_modify = [&](const account_object& co) { my->on_applied_op(db, co, op_type_enum::ROW_MODIFY); }; - co_inde.applied_remove = [&](const chain::account_object& co) { + co_inde.applied_remove = [&](const account_object& co) { my->on_applied_op(db, co, op_type_enum::ROW_REMOVE); }; @@ -107,15 +110,15 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { auto& po_inde = db.get_index(); - po_inde.applied_emplace = [&](const chain::permission_object& po) { + po_inde.applied_emplace = [&](const permission_object& po) { my->on_applied_op(db, po, op_type_enum::ROW_CREATE); }; - po_inde.applied_modify = [&](const chain::permission_object& po) { + po_inde.applied_modify = [&](const permission_object& po) { my->on_applied_op(db, po, op_type_enum::ROW_MODIFY); }; - po_inde.applied_remove = [&](const chain::permission_object& po) { + po_inde.applied_remove = [&](const permission_object& po) { my->on_applied_op(db, po, op_type_enum::ROW_REMOVE); }; @@ -125,15 +128,15 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { auto& plo_inde = db.get_index(); - plo_inde.applied_emplace = [&](const chain::permission_link_object& plo) { + plo_inde.applied_emplace = [&](const permission_link_object& plo) { my->on_applied_op(db, plo, op_type_enum::ROW_CREATE); }; - plo_inde.applied_modify = [&](const chain::permission_link_object& plo) { + plo_inde.applied_modify = [&](const permission_link_object& plo) { my->on_applied_op(db, plo, op_type_enum::ROW_MODIFY); }; - plo_inde.applied_remove = [&](const chain::permission_link_object& plo) { + plo_inde.applied_remove = [&](const permission_link_object& plo) { my->on_applied_op(db, plo, op_type_enum::ROW_REMOVE); }; @@ -143,7 +146,7 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { auto& ti_index = db.get_index(); - ti_index.applied_remove = [&](const chain::table_id_object& tio) { + ti_index.applied_remove = [&](const table_id_object& tio) { my->on_applied_table(db, tio, op_type_enum::TABLE_REMOVE); }; @@ -151,15 +154,15 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { auto& kv_index = db.get_index(); - kv_index.applied_emplace = [&](const chain::key_value_object& kvo) { + kv_index.applied_emplace = [&](const key_value_object& kvo) { my->on_applied_op(db, kvo, op_type_enum::ROW_CREATE); }; - kv_index.applied_modify = [&](const chain::key_value_object& kvo) { + kv_index.applied_modify = [&](const key_value_object& kvo) { my->on_applied_op(db, kvo, op_type_enum::ROW_MODIFY); }; - kv_index.applied_remove = [&](const chain::key_value_object& kvo) { + kv_index.applied_remove = [&](const key_value_object& kvo) { my->on_applied_op(db, kvo, op_type_enum::ROW_REMOVE); }; @@ -176,7 +179,7 @@ void statetrack_plugin::plugin_initialize(const variables_map& options) { })); my->irreversible_block_connection.emplace( - chain.irreversible_block.connect([&]( const chain::block_state_ptr& bsp ) { + chain.irreversible_block.connect([&]( const block_state_ptr& bsp ) { my->on_irreversible_block( bsp ); })); } diff --git a/plugins/statetrack_plugin/statetrack_plugin_impl.cpp b/plugins/statetrack_plugin/statetrack_plugin_impl.cpp index d04e8a73309..5f1c0d27946 100644 --- a/plugins/statetrack_plugin/statetrack_plugin_impl.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin_impl.cpp @@ -4,6 +4,11 @@ */ #include +namespace eosio { + +using namespace chain; +using namespace chainbase; + // statetrack plugin implementation void statetrack_plugin_impl::send_zmq_msg(const fc::string content) { @@ -15,7 +20,7 @@ void statetrack_plugin_impl::send_zmq_msg(const fc::string content) { sender_socket->send(message); } -bool statetrack_plugin_impl::filter(const chain::account_name& code, const fc::string& scope, const chain::table_name& table) { +bool statetrack_plugin_impl::filter(const account_name& code, const fc::string& scope, const table_name& table) { if(filter_on.size() > 0) { bool pass_on = false; if (filter_on.find({ code, 0, 0 }) != filter_on.end()) { @@ -45,28 +50,28 @@ bool statetrack_plugin_impl::filter(const chain::account_name& code, const fc::s return true; } -const chain::table_id_object* statetrack_plugin_impl::get_kvo_tio(const chainbase::database& db, const chain::key_value_object& kvo) { - return db.find(kvo.t_id); +const table_id_object* statetrack_plugin_impl::get_kvo_tio(const database& db, const key_value_object& kvo) { + return db.find(kvo.t_id); } -const chain::account_object& statetrack_plugin_impl::get_tio_co(const chainbase::database& db, const chain::table_id_object& tio) { - const chain::account_object *co = db.find(tio.code); - EOS_ASSERT(co != nullptr, chain::account_query_exception, "Fail to retrieve account for ${code}", ("code", tio.code) ); +const account_object& statetrack_plugin_impl::get_tio_co(const database& db, const table_id_object& tio) { + const account_object *co = db.find(tio.code); + EOS_ASSERT(co != nullptr, account_query_exception, "Fail to retrieve account for ${code}", ("code", tio.code) ); return *co; } -const abi_def& statetrack_plugin_impl::get_co_abi( const chain::account_object& co ) { +const abi_def& statetrack_plugin_impl::get_co_abi( const account_object& co ) { abi_def* abi = new abi_def(); abi_serializer::to_abi(co.abi, *abi); return *abi; } -fc::variant statetrack_plugin_impl::get_kvo_row(const chainbase::database& db, const chain::table_id_object& tio, const chain::key_value_object& kvo, bool json) { +fc::variant statetrack_plugin_impl::get_kvo_row(const database& db, const table_id_object& tio, const key_value_object& kvo, bool json) { vector data; copy_inline_row(kvo, data); if (json) { - const chain::account_object& co = get_tio_co(db, tio); + const account_object& co = get_tio_co(db, tio); const abi_def& abi = get_co_abi(co); abi_serializer abis; @@ -78,7 +83,7 @@ fc::variant statetrack_plugin_impl::get_kvo_row(const chainbase::database& db, c } } -db_op statetrack_plugin_impl::get_db_op(const chainbase::database& db, const chain::account_object& co, op_type_enum op_type) { +db_op statetrack_plugin_impl::get_db_op(const database& db, const account_object& co, op_type_enum op_type) { db_op op; name system = N(system); @@ -103,7 +108,7 @@ db_op statetrack_plugin_impl::get_db_op(const chainbase::database& db, const cha return op; } -db_op statetrack_plugin_impl::get_db_op(const chainbase::database& db, const chain::permission_object& po, op_type_enum op_type) { +db_op statetrack_plugin_impl::get_db_op(const database& db, const permission_object& po, op_type_enum op_type) { db_op op; name system = N(system); @@ -118,7 +123,7 @@ db_op statetrack_plugin_impl::get_db_op(const chainbase::database& db, const cha return op; } -db_op statetrack_plugin_impl::get_db_op(const chainbase::database& db, const chain::permission_link_object& plo, op_type_enum op_type) { +db_op statetrack_plugin_impl::get_db_op(const database& db, const permission_link_object& plo, op_type_enum op_type) { db_op op; name system = N(system); @@ -133,7 +138,7 @@ db_op statetrack_plugin_impl::get_db_op(const chainbase::database& db, const cha return op; } -db_op statetrack_plugin_impl::get_db_op(const chainbase::database& db, const chain::table_id_object& tio, const chain::key_value_object& kvo, op_type_enum op_type, bool json) { +db_op statetrack_plugin_impl::get_db_op(const database& db, const table_id_object& tio, const key_value_object& kvo, op_type_enum op_type, bool json) { db_op op; op.id = kvo.primary_key; @@ -155,7 +160,7 @@ db_rev statetrack_plugin_impl::get_db_rev(const int64_t revision, op_type_enum o return rev; } -void statetrack_plugin_impl::on_applied_table(const chainbase::database& db, const chain::table_id_object& tio, op_type_enum op_type) { +void statetrack_plugin_impl::on_applied_table(const database& db, const table_id_object& tio, op_type_enum op_type) { if(sender_socket == nullptr) return; @@ -177,7 +182,7 @@ void statetrack_plugin_impl::on_applied_table(const chainbase::database& db, con } } -void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const chain::account_object& co, op_type_enum op_type) { +void statetrack_plugin_impl::on_applied_op(const database& db, const account_object& co, op_type_enum op_type) { if(sender_socket == nullptr) return; @@ -193,7 +198,7 @@ void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const } } -void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const chain::permission_object& po, op_type_enum op_type) { +void statetrack_plugin_impl::on_applied_op(const database& db, const permission_object& po, op_type_enum op_type) { if(sender_socket == nullptr) return; @@ -209,7 +214,7 @@ void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const } } -void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const chain::permission_link_object& plo, op_type_enum op_type) { +void statetrack_plugin_impl::on_applied_op(const database& db, const permission_link_object& plo, op_type_enum op_type) { if(sender_socket == nullptr) return; @@ -225,16 +230,16 @@ void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const } } -void statetrack_plugin_impl::on_applied_op(const chainbase::database& db, const chain::key_value_object& kvo, op_type_enum op_type) { +void statetrack_plugin_impl::on_applied_op(const database& db, const key_value_object& kvo, op_type_enum op_type) { if(sender_socket == nullptr) return; - const chain::table_id_object* tio_ptr = get_kvo_tio(db, kvo); + const table_id_object* tio_ptr = get_kvo_tio(db, kvo); if(tio_ptr == nullptr) return; - const chain::table_id_object& tio = *tio_ptr; + const table_id_object& tio = *tio_ptr; auto code = tio.code; auto scope = scope_sym_to_string(tio.scope); @@ -285,7 +290,7 @@ void statetrack_plugin_impl::on_accepted_block(const block_state_ptr& bsp) } for( const auto& atrace : it->second->action_traces ) { - on_action_trace( atrace, block_state ); + on_action_trace( atrace, bsp ); } } } @@ -296,6 +301,9 @@ void statetrack_plugin_impl::on_accepted_block(const block_state_ptr& bsp) void statetrack_plugin_impl::on_action_trace( const action_trace& at, const block_state_ptr& bsp ) { + if(sender_socket == nullptr) + return; + // check the action against the blacklist auto search_acc = blacklist_actions.find(at.act.account); if(search_acc != blacklist_actions.end()) { @@ -307,11 +315,27 @@ void statetrack_plugin_impl::on_action_trace( const action_trace& at, const bloc auto& chain = chain_plug->chain(); - //TODO send action trace + db_action da; + da.global_action_seq = at.receipt.global_sequence; + da.block_num = bsp->block->block_num(); + da.block_time = bsp->block->timestamp; + da.action_trace = chain.to_variant_with_abi(at, abi_serializer_max_time); + da.last_irreversible_block = chain.last_irreversible_block_num(); + + fc::string data = fc::json::to_string(da); + ilog("STATETRACK ${op_type}: ${data}", ("op_type", op_type_enum::TRX_ACTION)("data", data)); + send_zmq_msg(data); } -void statetrack_plugin_impl::on_irreversible_block( const chain::block_state_ptr& bsp ) +void statetrack_plugin_impl::on_irreversible_block( const block_state_ptr& bsp ) { - //TODO send irreversible block information -} + if(sender_socket == nullptr) + return; + + fc::string data = fc::json::to_string(get_db_rev(bsp->block_num, op_type_enum::REV_COMMIT)); + ilog("STATETRACK ${op_type}: ${data}", ("op_type", op_type_enum::REV_COMMIT)("data", data)); + send_zmq_msg(data); +} + +} \ No newline at end of file From 504162c16a13c5fc6a7d8187d8fd75828edb6fcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Sun, 11 Nov 2018 21:23:53 +0000 Subject: [PATCH 45/56] fixes to action --- .../statetrack_plugin_impl.hpp | 267 +++++------ .../statetrack_plugin_impl.cpp | 413 ++++++++++-------- 2 files changed, 374 insertions(+), 306 deletions(-) diff --git a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp index bc23a6d3a78..9196ce6e930 100644 --- a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp +++ b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp @@ -17,14 +17,13 @@ #include #include - -namespace eosio { +namespace eosio +{ using namespace chain; using namespace chainbase; using boost::signals2::scoped_connection; - typedef typename get_index_type::type co_index_type; typedef typename get_index_type::type po_index_type; typedef typename get_index_type::type plo_index_type; @@ -37,154 +36,168 @@ typedef permission_object::id_type po_id_type; typedef permission_link_object::id_type plo_id_type; typedef key_value_object::id_type kvo_id_type; - -enum op_type_enum { - TABLE_REMOVE = 0, - ROW_CREATE = 1, - ROW_MODIFY = 2, - ROW_REMOVE = 3, - TRX_ACTION = 4, - REV_UNDO = 5, - REV_COMMIT = 6 +enum op_type_enum +{ + TABLE_REMOVE = 0, + ROW_CREATE = 1, + ROW_MODIFY = 2, + ROW_REMOVE = 3, + TRX_ACTION = 4, + REV_UNDO = 5, + REV_COMMIT = 6 }; -struct db_account { - account_name name; - uint8_t vm_type = 0; - uint8_t vm_version = 0; - bool privileged = false; +struct db_account +{ + account_name name; + uint8_t vm_type = 0; + uint8_t vm_version = 0; + bool privileged = false; - time_point last_code_update; - digest_type code_version; + time_point last_code_update; + digest_type code_version; block_timestamp_type creation_date; }; -struct db_table { - op_type_enum op_type; - account_name code; - fc::string scope; - table_name table; - account_name payer; +struct db_table +{ + op_type_enum op_type; + account_name code; + fc::string scope; + table_name table; + account_name payer; }; -struct db_op : db_table { - uint64_t id; - fc::variant value; +struct db_op : db_table +{ + uint64_t id; + fc::variant value; }; - -struct db_rev { - op_type_enum op_type; - int64_t revision; + +struct db_rev +{ + op_type_enum op_type; + int64_t revision; }; -struct db_action { - uint64_t global_action_seq; - block_num_type block_num; - block_timestamp_type block_time; - fc::variant action_trace; - uint32_t last_irreversible_block; +struct db_action +{ + uint64_t global_action_seq; + block_num_type block_num; + block_timestamp_type block_time; + fc::variant action_trace; + uint32_t last_irreversible_block; }; -struct filter_entry { +struct filter_entry +{ name code; name scope; name table; - std::tuple key() const { - return std::make_tuple(code, scope, table); + std::tuple key() const + { + return std::make_tuple(code, scope, table); } - friend bool operator<( const filter_entry& a, const filter_entry& b ) { - return a.key() < b.key(); + friend bool operator<(const filter_entry &a, const filter_entry &b) + { + return a.key() < b.key(); + } +}; + +class statetrack_plugin_impl +{ + public: + statetrack_plugin_impl() + { + blacklist_actions.emplace(std::make_pair(config::system_account_name, + std::set{N(onblock)})); + blacklist_actions.emplace(std::make_pair(N(blocktwitter), + std::set{N(tweet)})); + } + + void send_zmq_msg(const fc::string content); + bool filter(const account_name &code, const fc::string &scope, const table_name &table); + + const table_id_object *get_kvo_tio(const database &db, const key_value_object &kvo); + const account_object &get_tio_co(const database &db, const table_id_object &tio); + const abi_def &get_co_abi(const account_object &co); + fc::variant get_kvo_row(const database &db, const table_id_object &tio, const key_value_object &kvo, bool json = true); + + db_op get_db_op(const database &db, const account_object &co, op_type_enum op_type); + db_op get_db_op(const database &db, const permission_object &po, op_type_enum op_type); + db_op get_db_op(const database &db, const permission_link_object &plo, op_type_enum op_type); + db_op get_db_op(const database &db, const table_id_object &tio, const key_value_object &kvo, op_type_enum op_type, bool json = true); + + db_rev get_db_rev(const int64_t revision, op_type_enum op_type); + + //generic_index state tables + void on_applied_table(const database &db, const table_id_object &tio, op_type_enum op_type); + //generic_index op + void on_applied_op(const database &db, const account_object &co, op_type_enum op_type); + void on_applied_op(const database &db, const permission_object &po, op_type_enum op_type); + void on_applied_op(const database &db, const permission_link_object &plo, op_type_enum op_type); + void on_applied_op(const database &db, const key_value_object &kvo, op_type_enum op_type); + //generic_index undo + void on_applied_rev(const int64_t revision, op_type_enum op_type); + //blocks and transactions + void on_applied_transaction(const transaction_trace_ptr &ttp); + void on_accepted_block(const block_state_ptr &bsp); + void on_action_trace(const action_trace &at, const block_state_ptr &bsp); + void on_irreversible_block(const block_state_ptr &bsp); + + static void copy_inline_row(const key_value_object &obj, vector &data) + { + data.resize(obj.value.size()); + memcpy(data.data(), obj.value.data(), obj.value.size()); } - }; - -class statetrack_plugin_impl { - public: - statetrack_plugin_impl() { - blacklist_actions.emplace - (std::make_pair(config::system_account_name, - std::set{ N(onblock) } )); - blacklist_actions.emplace - (std::make_pair(N(blocktwitter), - std::set{ N(tweet) } )); - } - - void send_zmq_msg(const fc::string content); - bool filter(const account_name& code, const fc::string& scope, const table_name& table); - - const table_id_object* get_kvo_tio(const database& db, const key_value_object& kvo); - const account_object& get_tio_co(const database& db, const table_id_object& tio); - const abi_def& get_co_abi(const account_object& co ); - fc::variant get_kvo_row(const database& db, const table_id_object& tio, const key_value_object& kvo, bool json = true); - - db_op get_db_op(const database& db, const account_object& co, op_type_enum op_type); - db_op get_db_op(const database& db, const permission_object& po, op_type_enum op_type); - db_op get_db_op(const database& db, const permission_link_object& plo, op_type_enum op_type); - db_op get_db_op(const database& db, const table_id_object& tio, const key_value_object& kvo, op_type_enum op_type, bool json = true); - - db_rev get_db_rev(const int64_t revision, op_type_enum op_type); - - //generic_index state tables - void on_applied_table(const database& db, const table_id_object& tio, op_type_enum op_type); - //generic_index op - void on_applied_op(const database& db, const account_object& co, op_type_enum op_type); - void on_applied_op(const database& db, const permission_object& po, op_type_enum op_type); - void on_applied_op(const database& db, const permission_link_object& plo, op_type_enum op_type); - void on_applied_op(const database& db, const key_value_object& kvo, op_type_enum op_type); - //generic_index undo - void on_applied_rev(const int64_t revision, op_type_enum op_type); - //blocks and transactions - void on_applied_transaction( const transaction_trace_ptr& ttp ); - void on_accepted_block(const block_state_ptr& bsp); - void on_action_trace( const action_trace& at, const block_state_ptr& bsp ); - void on_irreversible_block( const block_state_ptr& bsp ); - - static void copy_inline_row(const key_value_object& obj, vector& data) { - data.resize( obj.value.size() ); - memcpy( data.data(), obj.value.data(), obj.value.size() ); - } - static fc::string scope_sym_to_string(scope_name sym_code) { - fc::string scope = sym_code.to_string(); - if(scope.length() > 0 && scope[0] == '.') { - uint64_t scope_int = sym_code; - vector v; - for( int i = 0; i < 7; ++i ) { - char c = (char)(scope_int & 0xff); - if( !c ) break; - v.emplace_back(c); - scope_int >>= 8; - } - return fc::string(v.begin(),v.end()); + static fc::string scope_sym_to_string(scope_name sym_code) + { + fc::string scope = sym_code.to_string(); + if (scope.length() > 0 && scope[0] == '.') + { + uint64_t scope_int = sym_code; + vector v; + for (int i = 0; i < 7; ++i) + { + char c = (char)(scope_int & 0xff); + if (!c) + break; + v.emplace_back(c); + scope_int >>= 8; } - return scope; + return fc::string(v.begin(), v.end()); } + return scope; + } + + chain_plugin *chain_plug = nullptr; + fc::string socket_bind_str; + zmq::context_t *context; + zmq::socket_t *sender_socket; + + std::map cached_traces; + std::map> blacklist_actions; + + std::set filter_on; + std::set filter_out; + fc::microseconds abi_serializer_max_time; + + fc::optional applied_transaction_connection; + fc::optional accepted_block_connection; + fc::optional irreversible_block_connection; - chain_plugin* chain_plug = nullptr; - fc::string socket_bind_str; - zmq::context_t* context; - zmq::socket_t* sender_socket; - - std::map cached_traces; - std::map> blacklist_actions; - - std::set filter_on; - std::set filter_out; - fc::microseconds abi_serializer_max_time; - - fc::optional applied_transaction_connection; - fc::optional accepted_block_connection; - fc::optional irreversible_block_connection; - private: - bool shorten_abi_errors = true; + private: + bool shorten_abi_errors = true; }; -} +} // namespace eosio -FC_REFLECT_ENUM( eosio::op_type_enum, (TABLE_REMOVE)(ROW_CREATE)(ROW_MODIFY)(ROW_REMOVE)(TRX_ACTION)(REV_UNDO)(REV_COMMIT) ) -FC_REFLECT( eosio::db_account, (name)(vm_type)(vm_version)(privileged)(last_code_update)(code_version)(creation_date) ) -FC_REFLECT( eosio::db_table, (op_type)(code)(scope)(table)(payer) ) -FC_REFLECT_DERIVED( eosio::db_op, (eosio::db_table), (id)(value) ) -FC_REFLECT( eosio::db_rev, (op_type)(revision) ) -FC_REFLECT( eosio::db_action, (block_num)(block_time)(action_trace)(last_irreversible_block) ) +FC_REFLECT_ENUM(eosio::op_type_enum, (TABLE_REMOVE)(ROW_CREATE)(ROW_MODIFY)(ROW_REMOVE)(TRX_ACTION)(REV_UNDO)(REV_COMMIT)) +FC_REFLECT(eosio::db_account, (name)(vm_type)(vm_version)(privileged)(last_code_update)(code_version)(creation_date)) +FC_REFLECT(eosio::db_table, (op_type)(code)(scope)(table)(payer)) +FC_REFLECT_DERIVED(eosio::db_op, (eosio::db_table), (id)(value)) +FC_REFLECT(eosio::db_rev, (op_type)(revision)) +FC_REFLECT(eosio::db_action, (block_num)(block_time)(action_trace)(last_irreversible_block)) diff --git a/plugins/statetrack_plugin/statetrack_plugin_impl.cpp b/plugins/statetrack_plugin/statetrack_plugin_impl.cpp index 5f1c0d27946..be3943c801f 100644 --- a/plugins/statetrack_plugin/statetrack_plugin_impl.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin_impl.cpp @@ -4,338 +4,393 @@ */ #include -namespace eosio { +namespace eosio +{ using namespace chain; using namespace chainbase; // statetrack plugin implementation -void statetrack_plugin_impl::send_zmq_msg(const fc::string content) { - if(sender_socket == nullptr) +void statetrack_plugin_impl::send_zmq_msg(const fc::string content) +{ + if (sender_socket == nullptr) return; - + zmq::message_t message(content.length()); memcpy(message.data(), content.c_str(), content.length()); sender_socket->send(message); } -bool statetrack_plugin_impl::filter(const account_name& code, const fc::string& scope, const table_name& table) { - if(filter_on.size() > 0) { +bool statetrack_plugin_impl::filter(const account_name &code, const fc::string &scope, const table_name &table) +{ + if (filter_on.size() > 0) + { bool pass_on = false; - if (filter_on.find({ code, 0, 0 }) != filter_on.end()) { + if (filter_on.find({code, 0, 0}) != filter_on.end()) + { pass_on = true; } - if (filter_on.find({ code, scope, 0 }) != filter_on.end()) { + if (filter_on.find({code, scope, 0}) != filter_on.end()) + { pass_on = true; } - if (filter_on.find({ code, scope, table }) != filter_on.end()) { + if (filter_on.find({code, scope, table}) != filter_on.end()) + { pass_on = true; } - if (!pass_on) { return false; } + if (!pass_on) + { + return false; + } } - if(filter_out.size() > 0) { - if (filter_out.find({ code, 0, 0 }) != filter_out.end()) { + if (filter_out.size() > 0) + { + if (filter_out.find({code, 0, 0}) != filter_out.end()) + { return false; } - if (filter_out.find({ code, scope, 0 }) != filter_out.end()) { + if (filter_out.find({code, scope, 0}) != filter_out.end()) + { return false; } - if (filter_out.find({ code, scope, table }) != filter_out.end()) { + if (filter_out.find({code, scope, table}) != filter_out.end()) + { return false; } } return true; } - -const table_id_object* statetrack_plugin_impl::get_kvo_tio(const database& db, const key_value_object& kvo) { - return db.find(kvo.t_id); + +const table_id_object *statetrack_plugin_impl::get_kvo_tio(const database &db, const key_value_object &kvo) +{ + return db.find(kvo.t_id); } -const account_object& statetrack_plugin_impl::get_tio_co(const database& db, const table_id_object& tio) { - const account_object *co = db.find(tio.code); - EOS_ASSERT(co != nullptr, account_query_exception, "Fail to retrieve account for ${code}", ("code", tio.code) ); - return *co; +const account_object &statetrack_plugin_impl::get_tio_co(const database &db, const table_id_object &tio) +{ + const account_object *co = db.find(tio.code); + EOS_ASSERT(co != nullptr, account_query_exception, "Fail to retrieve account for ${code}", ("code", tio.code)); + return *co; } -const abi_def& statetrack_plugin_impl::get_co_abi( const account_object& co ) { - abi_def* abi = new abi_def(); - abi_serializer::to_abi(co.abi, *abi); - return *abi; +const abi_def &statetrack_plugin_impl::get_co_abi(const account_object &co) +{ + abi_def *abi = new abi_def(); + abi_serializer::to_abi(co.abi, *abi); + return *abi; } -fc::variant statetrack_plugin_impl::get_kvo_row(const database& db, const table_id_object& tio, const key_value_object& kvo, bool json) { - vector data; - copy_inline_row(kvo, data); +fc::variant statetrack_plugin_impl::get_kvo_row(const database &db, const table_id_object &tio, const key_value_object &kvo, bool json) +{ + vector data; + copy_inline_row(kvo, data); - if (json) { - const account_object& co = get_tio_co(db, tio); - const abi_def& abi = get_co_abi(co); + if (json) + { + const account_object &co = get_tio_co(db, tio); + const abi_def &abi = get_co_abi(co); - abi_serializer abis; - abis.set_abi(abi, abi_serializer_max_time); + abi_serializer abis; + abis.set_abi(abi, abi_serializer_max_time); - return abis.binary_to_variant( abis.get_table_type(tio.table), data, abi_serializer_max_time, shorten_abi_errors ); - } else { - return fc::variant(data); - } + return abis.binary_to_variant(abis.get_table_type(tio.table), data, abi_serializer_max_time, shorten_abi_errors); + } + else + { + return fc::variant(data); + } } -db_op statetrack_plugin_impl::get_db_op(const database& db, const account_object& co, op_type_enum op_type) { - db_op op; +db_op statetrack_plugin_impl::get_db_op(const database &db, const account_object &co, op_type_enum op_type) +{ + db_op op; - name system = N(system); + name system = N(system); - op.id = co.id._id; - op.op_type = op_type; - op.code = system; - op.scope = system.to_string(); - op.table = N(accounts); + op.id = co.id._id; + op.op_type = op_type; + op.code = system; + op.scope = system.to_string(); + op.table = N(accounts); - db_account account; - account.name = co.name; - account.vm_type = co.vm_type; - account.vm_version = co.vm_version; - account.privileged = co.privileged; - account.last_code_update = co.last_code_update; - account.code_version = co.code_version; - account.creation_date = co.creation_date; + db_account account; + account.name = co.name; + account.vm_type = co.vm_type; + account.vm_version = co.vm_version; + account.privileged = co.privileged; + account.last_code_update = co.last_code_update; + account.code_version = co.code_version; + account.creation_date = co.creation_date; - op.value = fc::variant(account); + op.value = fc::variant(account); - return op; + return op; } -db_op statetrack_plugin_impl::get_db_op(const database& db, const permission_object& po, op_type_enum op_type) { - db_op op; +db_op statetrack_plugin_impl::get_db_op(const database &db, const permission_object &po, op_type_enum op_type) +{ + db_op op; - name system = N(system); + name system = N(system); - op.id = po.id._id; - op.op_type = op_type; - op.code = system; - op.scope = system.to_string(); - op.table = N(permissions); - op.value = fc::variant(po); + op.id = po.id._id; + op.op_type = op_type; + op.code = system; + op.scope = system.to_string(); + op.table = N(permissions); + op.value = fc::variant(po); - return op; + return op; } -db_op statetrack_plugin_impl::get_db_op(const database& db, const permission_link_object& plo, op_type_enum op_type) { - db_op op; +db_op statetrack_plugin_impl::get_db_op(const database &db, const permission_link_object &plo, op_type_enum op_type) +{ + db_op op; - name system = N(system); + name system = N(system); - op.id = plo.id._id; - op.op_type = op_type; - op.code = system; - op.scope = system.to_string(); - op.table = N(permission_links); - op.value = fc::variant(plo); + op.id = plo.id._id; + op.op_type = op_type; + op.code = system; + op.scope = system.to_string(); + op.table = N(permission_links); + op.value = fc::variant(plo); - return op; + return op; } -db_op statetrack_plugin_impl::get_db_op(const database& db, const table_id_object& tio, const key_value_object& kvo, op_type_enum op_type, bool json) { - db_op op; - - op.id = kvo.primary_key; - op.op_type = op_type; - op.code = tio.code; - op.scope = scope_sym_to_string(tio.scope); - op.table = tio.table; - - if(op_type == op_type_enum::ROW_CREATE || - op_type == op_type_enum::ROW_MODIFY) { - op.value = get_kvo_row(db, tio, kvo, json); - } +db_op statetrack_plugin_impl::get_db_op(const database &db, const table_id_object &tio, const key_value_object &kvo, op_type_enum op_type, bool json) +{ + db_op op; + + op.id = kvo.primary_key; + op.op_type = op_type; + op.code = tio.code; + op.scope = scope_sym_to_string(tio.scope); + op.table = tio.table; + + if (op_type == op_type_enum::ROW_CREATE || + op_type == op_type_enum::ROW_MODIFY) + { + op.value = get_kvo_row(db, tio, kvo, json); + } - return op; + return op; } -db_rev statetrack_plugin_impl::get_db_rev(const int64_t revision, op_type_enum op_type) { - db_rev rev = {op_type, revision}; - return rev; +db_rev statetrack_plugin_impl::get_db_rev(const int64_t revision, op_type_enum op_type) +{ + db_rev rev = {op_type, revision}; + return rev; } -void statetrack_plugin_impl::on_applied_table(const database& db, const table_id_object& tio, op_type_enum op_type) { - if(sender_socket == nullptr) - return; - - auto code = tio.code; - auto scope = scope_sym_to_string(tio.scope); - auto table = tio.table; - - if(filter(code, scope, table)) { - db_table table; - - table.op_type = op_type; - table.code = tio.code; - table.scope = scope_sym_to_string(tio.scope); - table.table = tio.table; - - fc::string data = fc::json::to_string(table); - ilog("STATETRACK table_id_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); - send_zmq_msg(data); - } +void statetrack_plugin_impl::on_applied_table(const database &db, const table_id_object &tio, op_type_enum op_type) +{ + if (sender_socket == nullptr) + return; + + auto code = tio.code; + auto scope = scope_sym_to_string(tio.scope); + auto table = tio.table; + + if (filter(code, scope, table)) + { + db_table table; + + table.op_type = op_type; + table.code = tio.code; + table.scope = scope_sym_to_string(tio.scope); + table.table = tio.table; + + fc::string data = fc::json::to_string(table); + ilog("STATETRACK table_id_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); + send_zmq_msg(data); + } } -void statetrack_plugin_impl::on_applied_op(const database& db, const account_object& co, op_type_enum op_type) { - if(sender_socket == nullptr) +void statetrack_plugin_impl::on_applied_op(const database &db, const account_object &co, op_type_enum op_type) +{ + if (sender_socket == nullptr) return; - + name system = N(system); - auto code = system; + auto code = system; auto scope = system.to_string(); auto table = N(accounts); - - if(filter(code, scope, table)) { + + if (filter(code, scope, table)) + { fc::string data = fc::json::to_string(get_db_op(db, co, op_type)); ilog("STATETRACK account_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); send_zmq_msg(data); } } -void statetrack_plugin_impl::on_applied_op(const database& db, const permission_object& po, op_type_enum op_type) { - if(sender_socket == nullptr) +void statetrack_plugin_impl::on_applied_op(const database &db, const permission_object &po, op_type_enum op_type) +{ + if (sender_socket == nullptr) return; - + name system = N(system); - auto code = system; + auto code = system; auto scope = system.to_string(); auto table = N(permission); - if(filter(code, scope, table)) { + if (filter(code, scope, table)) + { fc::string data = fc::json::to_string(get_db_op(db, po, op_type)); ilog("STATETRACK permission_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); send_zmq_msg(data); } } -void statetrack_plugin_impl::on_applied_op(const database& db, const permission_link_object& plo, op_type_enum op_type) { - if(sender_socket == nullptr) +void statetrack_plugin_impl::on_applied_op(const database &db, const permission_link_object &plo, op_type_enum op_type) +{ + if (sender_socket == nullptr) return; - + name system = N(system); - auto code = system; + auto code = system; auto scope = system.to_string(); auto table = N(permission_links); - - if(filter(code, scope, table)) { + + if (filter(code, scope, table)) + { fc::string data = fc::json::to_string(get_db_op(db, plo, op_type)); ilog("STATETRACK permission_link_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); send_zmq_msg(data); } } - -void statetrack_plugin_impl::on_applied_op(const database& db, const key_value_object& kvo, op_type_enum op_type) { - if(sender_socket == nullptr) + +void statetrack_plugin_impl::on_applied_op(const database &db, const key_value_object &kvo, op_type_enum op_type) +{ + if (sender_socket == nullptr) return; - - const table_id_object* tio_ptr = get_kvo_tio(db, kvo); - if(tio_ptr == nullptr) + const table_id_object *tio_ptr = get_kvo_tio(db, kvo); + + if (tio_ptr == nullptr) return; - const table_id_object& tio = *tio_ptr; + const table_id_object &tio = *tio_ptr; - auto code = tio.code; + auto code = tio.code; auto scope = scope_sym_to_string(tio.scope); auto table = tio.table; - if(filter(code, scope, table)) { + if (filter(code, scope, table)) + { fc::string data = fc::json::to_string(get_db_op(db, *tio_ptr, kvo, op_type)); ilog("STATETRACK key_value_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); send_zmq_msg(data); } } -void statetrack_plugin_impl::on_applied_rev(const int64_t revision, op_type_enum op_type) { - if(sender_socket == nullptr) +void statetrack_plugin_impl::on_applied_rev(const int64_t revision, op_type_enum op_type) +{ + if (sender_socket == nullptr) return; - + fc::string data = fc::json::to_string(get_db_rev(revision, op_type)); ilog("STATETRACK ${op_type}: ${data}", ("op_type", op_type)("data", data)); send_zmq_msg(data); } -void statetrack_plugin_impl::on_applied_transaction( const transaction_trace_ptr& ttp ) +void statetrack_plugin_impl::on_applied_transaction(const transaction_trace_ptr &ttp) { - if (ttp->receipt) { + if (ttp->receipt) + { cached_traces[ttp->id] = ttp; } } - -void statetrack_plugin_impl::on_accepted_block(const block_state_ptr& bsp) + +void statetrack_plugin_impl::on_accepted_block(const block_state_ptr &bsp) { - //TODO send accepted block information - - for (auto& t : bsp->block->transactions) { - transaction_id_type id; - if (t.trx.contains()) { - id = t.trx.get(); - } - else { - id = t.trx.get().id(); + //TODO send accepted block information + + for (auto &t : bsp->block->transactions) + { + transaction_id_type id; + if (t.trx.contains()) + { + id = t.trx.get(); + } + else + { + id = t.trx.get().id(); + } + + if (t.status == transaction_receipt_header::executed) + { + // Send traces only for executed transactions + auto it = cached_traces.find(id); + if (it == cached_traces.end() || !it->second->receipt) + { + ilog("missing trace for transaction {id}", ("id", id)); + continue; } - if( t.status == transaction_receipt_header::executed ) { - // Send traces only for executed transactions - auto it = cached_traces.find(id); - if (it == cached_traces.end() || !it->second->receipt) { - ilog("missing trace for transaction {id}", ("id", id)); - continue; - } - - for( const auto& atrace : it->second->action_traces ) { - on_action_trace( atrace, bsp ); - } + for (const auto &atrace : it->second->action_traces) + { + on_action_trace(atrace, bsp); } } + } - cached_traces.clear(); + cached_traces.clear(); } - -void statetrack_plugin_impl::on_action_trace( const action_trace& at, const block_state_ptr& bsp ) +void statetrack_plugin_impl::on_action_trace(const action_trace &at, const block_state_ptr &bsp) { - if(sender_socket == nullptr) + if (sender_socket == nullptr) return; // check the action against the blacklist auto search_acc = blacklist_actions.find(at.act.account); - if(search_acc != blacklist_actions.end()) { + if (search_acc != blacklist_actions.end()) + { auto search_act = search_acc->second.find(at.act.name); - if( search_act != search_acc->second.end() ) { + if (search_act != search_acc->second.end()) + { return; } } - auto& chain = chain_plug->chain(); + auto &chain = chain_plug->chain(); + + db_op op; + + name system = N(system); + op.id = at.receipt.global_sequence; + op.op_type = op_type_enum::TRX_ACTION; + op.code = system; + op.scope = scope_sym_to_string(system); + op.table = N(actions); db_action da; - da.global_action_seq = at.receipt.global_sequence; da.block_num = bsp->block->block_num(); da.block_time = bsp->block->timestamp; da.action_trace = chain.to_variant_with_abi(at, abi_serializer_max_time); da.last_irreversible_block = chain.last_irreversible_block_num(); - fc::string data = fc::json::to_string(da); + op.value = fc::variant(da); + + fc::string data = fc::json::to_string(op); ilog("STATETRACK ${op_type}: ${data}", ("op_type", op_type_enum::TRX_ACTION)("data", data)); send_zmq_msg(data); } - -void statetrack_plugin_impl::on_irreversible_block( const block_state_ptr& bsp ) +void statetrack_plugin_impl::on_irreversible_block(const block_state_ptr &bsp) { - if(sender_socket == nullptr) + if (sender_socket == nullptr) return; - + fc::string data = fc::json::to_string(get_db_rev(bsp->block_num, op_type_enum::REV_COMMIT)); ilog("STATETRACK ${op_type}: ${data}", ("op_type", op_type_enum::REV_COMMIT)("data", data)); send_zmq_msg(data); } -} \ No newline at end of file +} // namespace eosio \ No newline at end of file From fd17898c9fd3bfb8155b99446ccec620ead524f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Wed, 14 Nov 2018 22:46:29 +0000 Subject: [PATCH 46/56] improvements to statetrack plugin --- libraries/chainbase | 2 +- .../statetrack_plugin_impl.hpp | 6 +- .../statetrack_plugin/statetrack_plugin.cpp | 386 ++++++++++-------- 3 files changed, 211 insertions(+), 183 deletions(-) diff --git a/libraries/chainbase b/libraries/chainbase index cda29335fd6..e52d821c73d 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit cda29335fd622d7a5d1fd3ee86f9dcd083d0ab95 +Subproject commit e52d821c73d33279be3a3733374115aff78eb338 diff --git a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp index 9196ce6e930..4122b3990c4 100644 --- a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp +++ b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp @@ -22,7 +22,7 @@ namespace eosio using namespace chain; using namespace chainbase; -using boost::signals2::scoped_connection; +using boost::signals2::connection; typedef typename get_index_type::type co_index_type; typedef typename get_index_type::type po_index_type; @@ -185,9 +185,7 @@ class statetrack_plugin_impl std::set filter_out; fc::microseconds abi_serializer_max_time; - fc::optional applied_transaction_connection; - fc::optional accepted_block_connection; - fc::optional irreversible_block_connection; + std::vector> connections; private: bool shorten_abi_errors = true; diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index 14e97ba7e46..0046e67bcef 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -5,200 +5,230 @@ #include #include -namespace { - const char* SENDER_BIND = "st-zmq-sender-bind"; - const char* SENDER_BIND_DEFAULT = "tcp://127.0.0.1:3000"; -} +namespace +{ +const char *SENDER_BIND = "st-zmq-sender-bind"; +const char *SENDER_BIND_DEFAULT = "tcp://127.0.0.1:3000"; +} // namespace -namespace eosio { +namespace eosio +{ using namespace chain; using namespace chainbase; -static appbase::abstract_plugin& _statetrack_plugin = app().register_plugin(); +static appbase::abstract_plugin &_statetrack_plugin = app().register_plugin(); // Plugin implementation -statetrack_plugin::statetrack_plugin():my(new statetrack_plugin_impl()){} - -void statetrack_plugin::set_program_options(options_description&, options_description& cfg) { - cfg.add_options() - (SENDER_BIND, bpo::value()->default_value(SENDER_BIND_DEFAULT), - "ZMQ Sender Socket binding") - ; - cfg.add_options() - ("st-filter-on,f", bpo::value>()->composing(), - "Track tables which match code:scope:table.") - ; - cfg.add_options() - ("st-filter-out,F", bpo::value>()->composing(), - "Do not track tables which match code:scope:table.") - ; +statetrack_plugin::statetrack_plugin() : my(new statetrack_plugin_impl()) {} + +void statetrack_plugin::set_program_options(options_description &, options_description &cfg) +{ + cfg.add_options()(SENDER_BIND, bpo::value()->default_value(SENDER_BIND_DEFAULT), + "ZMQ Sender Socket binding"); + cfg.add_options()("st-filter-on,f", bpo::value>()->composing(), + "Track tables which match code:scope:table."); + cfg.add_options()("st-filter-out,F", bpo::value>()->composing(), + "Do not track tables which match code:scope:table."); } -void statetrack_plugin::plugin_initialize(const variables_map& options) { - ilog("initializing statetrack plugin"); - - try { - if( options.count( "st-filter-on" )) { - auto fo = options.at( "st-filter-on" ).as>(); - for( auto& s : fo ) { - std::vector v; - boost::split( v, s, boost::is_any_of( ":" )); - EOS_ASSERT( v.size() == 3, fc::invalid_arg_exception, "Invalid value ${s} for --filter-on", ("s", s)); - filter_entry fe{v[0], v[1], v[2]}; - my->filter_on.insert( fe ); +void statetrack_plugin::plugin_initialize(const variables_map &options) +{ + ilog("initializing statetrack plugin"); + + try + { + if (options.count("st-filter-on")) + { + auto fo = options.at("st-filter-on").as>(); + for (auto &s : fo) + { + std::vector v; + boost::split(v, s, boost::is_any_of(":")); + EOS_ASSERT(v.size() == 3, fc::invalid_arg_exception, "Invalid value ${s} for --filter-on", ("s", s)); + filter_entry fe{v[0], v[1], v[2]}; + my->filter_on.insert(fe); + } + } + + if (options.count("st-filter-out")) + { + auto fo = options.at("st-filter-out").as>(); + for (auto &s : fo) + { + std::vector v; + boost::split(v, s, boost::is_any_of(":")); + EOS_ASSERT(v.size() == 3, fc::invalid_arg_exception, "Invalid value ${s} for --filter-out", ("s", s)); + filter_entry fe{v[0], v[1], v[2]}; + my->filter_out.insert(fe); + } } - } - - if( options.count( "st-filter-out" )) { - auto fo = options.at( "st-filter-out" ).as>(); - for( auto& s : fo ) { - std::vector v; - boost::split( v, s, boost::is_any_of( ":" )); - EOS_ASSERT( v.size() == 3, fc::invalid_arg_exception, "Invalid value ${s} for --filter-out", ("s", s)); - filter_entry fe{v[0], v[1], v[2]}; - my->filter_out.insert( fe ); + + my->socket_bind_str = options.at(SENDER_BIND).as(); + if (my->socket_bind_str.empty()) + { + wlog("zmq-sender-bind not specified => eosio::statetrack_plugin disabled."); + return; } - } - - my->socket_bind_str = options.at(SENDER_BIND).as(); - if (my->socket_bind_str.empty()) { - wlog("zmq-sender-bind not specified => eosio::statetrack_plugin disabled."); - return; - } - - my->context = new zmq::context_t(1); - my->sender_socket = new zmq::socket_t(*my->context, ZMQ_PUSH); - my->chain_plug = app().find_plugin(); - - ilog("Bind to ZMQ PUSH socket ${u}", ("u", my->socket_bind_str)); - my->sender_socket->bind(my->socket_bind_str); - - ilog("Bind to ZMQ PUSH socket successful"); - - auto& chain = my->chain_plug->chain(); - const database& db = chain.db(); - - my->abi_serializer_max_time = my->chain_plug->get_abi_serializer_max_time(); - - ilog("Binding database events"); - - auto undo_lambda = [&](const int64_t revision) { - my->on_applied_rev(revision, op_type_enum::REV_UNDO); - }; - - // account_object events - - auto& co_inde = db.get_index(); - - co_inde.applied_emplace = [&](const account_object& co) { - my->on_applied_op(db, co, op_type_enum::ROW_CREATE); - }; - - co_inde.applied_modify = [&](const account_object& co) { - my->on_applied_op(db, co, op_type_enum::ROW_MODIFY); - }; - - co_inde.applied_remove = [&](const account_object& co) { - my->on_applied_op(db, co, op_type_enum::ROW_REMOVE); - }; - - co_inde.applied_undo = undo_lambda; - - // permisson_object events - - auto& po_inde = db.get_index(); - - po_inde.applied_emplace = [&](const permission_object& po) { - my->on_applied_op(db, po, op_type_enum::ROW_CREATE); - }; - - po_inde.applied_modify = [&](const permission_object& po) { - my->on_applied_op(db, po, op_type_enum::ROW_MODIFY); - }; - - po_inde.applied_remove = [&](const permission_object& po) { - my->on_applied_op(db, po, op_type_enum::ROW_REMOVE); - }; - - po_inde.applied_undo = undo_lambda; - - // permisson_link_object events - - auto& plo_inde = db.get_index(); - - plo_inde.applied_emplace = [&](const permission_link_object& plo) { - my->on_applied_op(db, plo, op_type_enum::ROW_CREATE); - }; - - plo_inde.applied_modify = [&](const permission_link_object& plo) { - my->on_applied_op(db, plo, op_type_enum::ROW_MODIFY); - }; - - plo_inde.applied_remove = [&](const permission_link_object& plo) { - my->on_applied_op(db, plo, op_type_enum::ROW_REMOVE); - }; - - plo_inde.applied_undo = undo_lambda; - - // table_id_object events - - auto& ti_index = db.get_index(); - - ti_index.applied_remove = [&](const table_id_object& tio) { - my->on_applied_table(db, tio, op_type_enum::TABLE_REMOVE); - }; - - // key_value_object events - - auto& kv_index = db.get_index(); - - kv_index.applied_emplace = [&](const key_value_object& kvo) { - my->on_applied_op(db, kvo, op_type_enum::ROW_CREATE); - }; - - kv_index.applied_modify = [&](const key_value_object& kvo) { - my->on_applied_op(db, kvo, op_type_enum::ROW_MODIFY); - }; - - kv_index.applied_remove = [&](const key_value_object& kvo) { - my->on_applied_op(db, kvo, op_type_enum::ROW_REMOVE); - }; - - kv_index.applied_undo = undo_lambda; - - my->applied_transaction_connection.emplace( - chain.applied_transaction.connect([&]( const transaction_trace_ptr& ttp ) { - my->on_applied_transaction(ttp); - })); - - my->accepted_block_connection.emplace( - chain.accepted_block.connect([&](const block_state_ptr& bsp) { - my->on_accepted_block(bsp); - })); - - my->irreversible_block_connection.emplace( - chain.irreversible_block.connect([&]( const block_state_ptr& bsp ) { - my->on_irreversible_block( bsp ); - })); - } - FC_LOG_AND_RETHROW() -} -void statetrack_plugin::plugin_startup() { + my->context = new zmq::context_t(1); + my->sender_socket = new zmq::socket_t(*my->context, ZMQ_PUSH); + my->chain_plug = app().find_plugin(); + + ilog("Bind to ZMQ PUSH socket ${u}", ("u", my->socket_bind_str)); + my->sender_socket->bind(my->socket_bind_str); + + ilog("Bind to ZMQ PUSH socket successful"); + + auto &chain = my->chain_plug->chain(); + const database &db = chain.db(); + + my->abi_serializer_max_time = my->chain_plug->get_abi_serializer_max_time(); + + ilog("Binding database events"); + + auto undo_lambda = [&](const int64_t revision) { + my->on_applied_rev(revision, op_type_enum::REV_UNDO); + }; + + // account_object events + + auto &co_inde = db.get_index(); + + my->connections.emplace_back( + fc::optional(co_inde.applied_emplace.connect([&](const account_object &co) { + my->on_applied_op(db, co, op_type_enum::ROW_CREATE); + }))); + + my->connections.emplace_back( + fc::optional(co_inde.applied_modify.connect([&](const account_object &co) { + my->on_applied_op(db, co, op_type_enum::ROW_MODIFY); + }))); + + my->connections.emplace_back( + fc::optional(co_inde.applied_remove.connect([&](const account_object &co) { + my->on_applied_op(db, co, op_type_enum::ROW_REMOVE); + }))); + + my->connections.emplace_back( + co_inde.applied_undo.connect(undo_lambda)); + + // permisson_object events + + auto &po_inde = db.get_index(); + + my->connections.emplace_back( + fc::optional(po_inde.applied_emplace.connect([&](const permission_object &po) { + my->on_applied_op(db, po, op_type_enum::ROW_CREATE); + }))); + + my->connections.emplace_back( + fc::optional(po_inde.applied_modify.connect([&](const permission_object &po) { + my->on_applied_op(db, po, op_type_enum::ROW_MODIFY); + }))); + my->connections.emplace_back( + fc::optional(po_inde.applied_remove.connect([&](const permission_object &po) { + my->on_applied_op(db, po, op_type_enum::ROW_REMOVE); + }))); + + my->connections.emplace_back( + fc::optional(po_inde.applied_undo.connect(undo_lambda))); + + // permisson_link_object events + + auto &plo_inde = db.get_index(); + + my->connections.emplace_back( + fc::optional(plo_inde.applied_emplace.connect([&](const permission_link_object &plo) { + my->on_applied_op(db, plo, op_type_enum::ROW_CREATE); + }))); + + my->connections.emplace_back( + fc::optional(plo_inde.applied_modify.connect([&](const permission_link_object &plo) { + my->on_applied_op(db, plo, op_type_enum::ROW_MODIFY); + }))); + + my->connections.emplace_back( + fc::optional(plo_inde.applied_remove.connect([&](const permission_link_object &plo) { + my->on_applied_op(db, plo, op_type_enum::ROW_REMOVE); + }))); + + my->connections.emplace_back( + fc::optional(plo_inde.applied_undo.connect(undo_lambda))); + + // table_id_object events + + auto &ti_index = db.get_index(); + + my->connections.emplace_back( + fc::optional(ti_index.applied_remove.connect([&](const table_id_object &tio) { + my->on_applied_table(db, tio, op_type_enum::TABLE_REMOVE); + }))); + + // key_value_object events + + auto &kv_index = db.get_index(); + + my->connections.emplace_back( + fc::optional(kv_index.applied_emplace.connect([&](const key_value_object &kvo) { + my->on_applied_op(db, kvo, op_type_enum::ROW_CREATE); + }))); + + my->connections.emplace_back( + fc::optional(kv_index.applied_modify.connect([&](const key_value_object &kvo) { + my->on_applied_op(db, kvo, op_type_enum::ROW_MODIFY); + }))); + + my->connections.emplace_back( + fc::optional(kv_index.applied_remove.connect([&](const key_value_object &kvo) { + my->on_applied_op(db, kvo, op_type_enum::ROW_REMOVE); + }))); + + my->connections.emplace_back( + fc::optional(kv_index.applied_undo.connect(undo_lambda))); + + // transaction and block events + + my->connections.emplace_back( + fc::optional(chain.applied_transaction.connect([&](const transaction_trace_ptr &ttp) { + my->on_applied_transaction(ttp); + }))); + + my->connections.emplace_back( + fc::optional(chain.accepted_block.connect([&](const block_state_ptr &bsp) { + my->on_accepted_block(bsp); + }))); + + my->connections.emplace_back( + fc::optional(chain.irreversible_block.connect([&](const block_state_ptr &bsp) { + my->on_irreversible_block(bsp); + }))); + } + FC_LOG_AND_RETHROW() } -void statetrack_plugin::plugin_shutdown() { - ilog("statetrack plugin shutdown"); - if( ! my->socket_bind_str.empty() ) { - my->sender_socket->disconnect(my->socket_bind_str); - my->sender_socket->close(); - delete my->sender_socket; - delete my->context; - my->sender_socket = nullptr; - } +void statetrack_plugin::plugin_startup() +{ } +void statetrack_plugin::plugin_shutdown() +{ + ilog("statetrack plugin shutdown"); + if (!my->socket_bind_str.empty()) + { + my->sender_socket->disconnect(my->socket_bind_str); + my->sender_socket->close(); + delete my->sender_socket; + delete my->context; + my->sender_socket = nullptr; + + for(auto conn : my->connections) { + conn->disconnect(); + conn.reset(); + } + } } + +} // namespace eosio From 4ac0e4d8bbb732dfde8f0cbfed7133fdbdb04cc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Wed, 14 Nov 2018 23:41:16 +0000 Subject: [PATCH 47/56] add generic function to create index events --- .../statetrack_plugin_impl.hpp | 25 +++++ .../statetrack_plugin/statetrack_plugin.cpp | 94 +------------------ 2 files changed, 30 insertions(+), 89 deletions(-) diff --git a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp index 4122b3990c4..45c3ab53177 100644 --- a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp +++ b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp @@ -147,6 +147,31 @@ class statetrack_plugin_impl void on_action_trace(const action_trace &at, const block_state_ptr &bsp); void on_irreversible_block(const block_state_ptr &bsp); + template + void create_index_events(const database &db) { + auto &index = db.get_index(); + + connections.emplace_back( + fc::optional(index.applied_emplace.connect([&](const typename MultiIndexType::value_type &v) { + on_applied_op(db, v, op_type_enum::ROW_CREATE); + }))); + + connections.emplace_back( + fc::optional(index.applied_modify.connect([&](const typename MultiIndexType::value_type &v) { + on_applied_op(db, v, op_type_enum::ROW_MODIFY); + }))); + + connections.emplace_back( + fc::optional(index.applied_remove.connect([&](const typename MultiIndexType::value_type &v) { + on_applied_op(db, v, op_type_enum::ROW_REMOVE); + }))); + + connections.emplace_back( + fc::optional(index.applied_undo.connect([&](const int64_t revision) { + on_applied_rev(revision, op_type_enum::REV_UNDO); + }))); + } + static void copy_inline_row(const key_value_object &obj, vector &data) { data.resize(obj.value.size()); diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index 0046e67bcef..e093bb07d4b 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -88,78 +88,16 @@ void statetrack_plugin::plugin_initialize(const variables_map &options) ilog("Binding database events"); - auto undo_lambda = [&](const int64_t revision) { - my->on_applied_rev(revision, op_type_enum::REV_UNDO); - }; - // account_object events - - auto &co_inde = db.get_index(); - - my->connections.emplace_back( - fc::optional(co_inde.applied_emplace.connect([&](const account_object &co) { - my->on_applied_op(db, co, op_type_enum::ROW_CREATE); - }))); - - my->connections.emplace_back( - fc::optional(co_inde.applied_modify.connect([&](const account_object &co) { - my->on_applied_op(db, co, op_type_enum::ROW_MODIFY); - }))); - - my->connections.emplace_back( - fc::optional(co_inde.applied_remove.connect([&](const account_object &co) { - my->on_applied_op(db, co, op_type_enum::ROW_REMOVE); - }))); - - my->connections.emplace_back( - co_inde.applied_undo.connect(undo_lambda)); - + my->create_index_events(db); // permisson_object events - - auto &po_inde = db.get_index(); - - my->connections.emplace_back( - fc::optional(po_inde.applied_emplace.connect([&](const permission_object &po) { - my->on_applied_op(db, po, op_type_enum::ROW_CREATE); - }))); - - my->connections.emplace_back( - fc::optional(po_inde.applied_modify.connect([&](const permission_object &po) { - my->on_applied_op(db, po, op_type_enum::ROW_MODIFY); - }))); - - my->connections.emplace_back( - fc::optional(po_inde.applied_remove.connect([&](const permission_object &po) { - my->on_applied_op(db, po, op_type_enum::ROW_REMOVE); - }))); - - my->connections.emplace_back( - fc::optional(po_inde.applied_undo.connect(undo_lambda))); - + my->create_index_events(db); // permisson_link_object events - - auto &plo_inde = db.get_index(); - - my->connections.emplace_back( - fc::optional(plo_inde.applied_emplace.connect([&](const permission_link_object &plo) { - my->on_applied_op(db, plo, op_type_enum::ROW_CREATE); - }))); - - my->connections.emplace_back( - fc::optional(plo_inde.applied_modify.connect([&](const permission_link_object &plo) { - my->on_applied_op(db, plo, op_type_enum::ROW_MODIFY); - }))); - - my->connections.emplace_back( - fc::optional(plo_inde.applied_remove.connect([&](const permission_link_object &plo) { - my->on_applied_op(db, plo, op_type_enum::ROW_REMOVE); - }))); - - my->connections.emplace_back( - fc::optional(plo_inde.applied_undo.connect(undo_lambda))); + my->create_index_events(db); + // key_value_object events + my->create_index_events(db); // table_id_object events - auto &ti_index = db.get_index(); my->connections.emplace_back( @@ -167,28 +105,6 @@ void statetrack_plugin::plugin_initialize(const variables_map &options) my->on_applied_table(db, tio, op_type_enum::TABLE_REMOVE); }))); - // key_value_object events - - auto &kv_index = db.get_index(); - - my->connections.emplace_back( - fc::optional(kv_index.applied_emplace.connect([&](const key_value_object &kvo) { - my->on_applied_op(db, kvo, op_type_enum::ROW_CREATE); - }))); - - my->connections.emplace_back( - fc::optional(kv_index.applied_modify.connect([&](const key_value_object &kvo) { - my->on_applied_op(db, kvo, op_type_enum::ROW_MODIFY); - }))); - - my->connections.emplace_back( - fc::optional(kv_index.applied_remove.connect([&](const key_value_object &kvo) { - my->on_applied_op(db, kvo, op_type_enum::ROW_REMOVE); - }))); - - my->connections.emplace_back( - fc::optional(kv_index.applied_undo.connect(undo_lambda))); - // transaction and block events my->connections.emplace_back( From a42c94893825214e24d19921cb3b826954e859af Mon Sep 17 00:00:00 2001 From: Andres Berrios Date: Wed, 14 Nov 2018 15:59:46 -0800 Subject: [PATCH 48/56] Add info to Readme --- plugins/statetrack_plugin/README.md | 71 ++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 16 deletions(-) diff --git a/plugins/statetrack_plugin/README.md b/plugins/statetrack_plugin/README.md index 1f0c2b6244c..7c89adf0fc0 100644 --- a/plugins/statetrack_plugin/README.md +++ b/plugins/statetrack_plugin/README.md @@ -1,34 +1,73 @@ # statetrack_plugin -This plugin adds hooks on chainbase operations emplace, modify, remove, undo and routes the objects and revision numbers to a ZMQ queue. -Also hooks blocks and transactions to the ZMQ queue. +This plugin adds hooks on chainbase operations emplace, modify, remove, and undo, +and routes the objects and revision numbers to a ZMQ socket. -This is based on plugin: https://github.com/cc32d9/eos_zmq_plugin +It can hook to the operations that happen on the state DBs of all contracts +deployed to the chain, and it can use filters to limit the operations +that should be sent to only the ones happening on the contracts, scopes, and tables +that the user is interested in tracking, thus limiting the performance impact to a minimum. -### Configuration +The plugin will also send operations happening on accounts and account permissions. +It will also send a stream of applied actions. -The following configuration statements in config.ini are recognized: +This is a very early stage of the implementation, and more testing is required. +This should not be considered production ready yet. -plugin = eosio::statetrack_plugin -- enables the state track plugin +## Configuration -st-zmq-sender-bind = ENDPOINT -- specifies the PUSH socket connect endpoint. Default value: tcp://127.0.0.1:3000. +The following configuration statements in `config.ini` are recognized: -### adding ZMQ library on Mac +- `plugin = eosio::statetrack_plugin` -- Enables the state track plugin +- `st-zmq-sender-bind = ENDPOINT` -- Specifies the PUSH socket connection endpoint. + Default value: tcp://127.0.0.1:3000. +- `st-filter-on = code:scope:table` -- Track DB operations which match code:scope:table. +- `st-filter-off = code:scope:table` -- Do not track DB operations which match code:scope:table. + +## Important notes + +- Accounts are sent as DB operations with `(code, scope, table)` being + `(system, system, accounts)`. + - Disable tracking of accounts using `st-filter-off = system:system:accounts`. +- Account permissions are sent as DB operations with `(code, scope, table)` + being `(system, system, permissions)`. + - Disable tracking of account permissions using `st-filter-off = system:system:permissions`. +- Actions are sent as DB operations with `(code, scope, table)` being + `(system, system, actions)`. + - Disable tracking of actions using `st-filter-off = system:system:actions`. +- Filters are not tested yet, but the implementation is taken from + the history plugin, so it should work fine. +- We're adding separate filters for actions to make it possible to + filter by specific actions of specific contracts. + +## Open questions + +- In Chainbase, we are emitting the operations inside an undo just before the + operation is actually executed and checked for success + ([See here](https://github.com/EOSIO/chainbase/compare/master...mmcs85:master#diff-298563b6c76ef92100c2ea27c06cb08bR355)). + - This is because we are not sure how to get a reference to the item after + it is given to the `modify` and `emplace` functions using `std::move`. + - We don't like this inconsistency since it is emitting the event before being + sure that the operation actually took place, but we think it might not + be an issue since those operations are undoing previous ones and it + would not be an expected scenario for them to fail unless there was a bug + or a database inconsistency (maybe caused by bad locking mechanisms when + using multiple threads?). + +## Building + +The plugin needs to be built using the ZeroMQ C++ headers. + +### Adding ZMQ library on Mac ``` +brew install zmq brew tap jmuncaster/homebrew-header-only brew install jmuncaster/header-only/cppzmq ``` -### adding ZMQ library on Ubuntu +### Adding ZMQ library on Ubuntu ``` apt-get install -y pkg-config libzmq5-dev ``` - -### Building - -``` -./eosio_build.sh -./eosio_install.sh -``` From 057ed5a580e37d1dbe69f4c28211404d6be5400b Mon Sep 17 00:00:00 2001 From: Andres Berrios Date: Wed, 14 Nov 2018 16:19:25 -0800 Subject: [PATCH 49/56] Improve info in Readme --- plugins/statetrack_plugin/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/statetrack_plugin/README.md b/plugins/statetrack_plugin/README.md index 7c89adf0fc0..a5aaee009a4 100644 --- a/plugins/statetrack_plugin/README.md +++ b/plugins/statetrack_plugin/README.md @@ -22,19 +22,19 @@ The following configuration statements in `config.ini` are recognized: - `st-zmq-sender-bind = ENDPOINT` -- Specifies the PUSH socket connection endpoint. Default value: tcp://127.0.0.1:3000. - `st-filter-on = code:scope:table` -- Track DB operations which match code:scope:table. -- `st-filter-off = code:scope:table` -- Do not track DB operations which match code:scope:table. +- `st-filter-out = code:scope:table` -- Do not track DB operations which match code:scope:table. ## Important notes - Accounts are sent as DB operations with `(code, scope, table)` being `(system, system, accounts)`. - - Disable tracking of accounts using `st-filter-off = system:system:accounts`. + - Disable tracking of accounts using `st-filter-out = system:system:accounts`. - Account permissions are sent as DB operations with `(code, scope, table)` being `(system, system, permissions)`. - - Disable tracking of account permissions using `st-filter-off = system:system:permissions`. + - Disable tracking of account permissions using `st-filter-out = system:system:permissions`. - Actions are sent as DB operations with `(code, scope, table)` being `(system, system, actions)`. - - Disable tracking of actions using `st-filter-off = system:system:actions`. + - Disable tracking of actions using `st-filter-out = system:system:actions`. - Filters are not tested yet, but the implementation is taken from the history plugin, so it should work fine. - We're adding separate filters for actions to make it possible to @@ -45,8 +45,8 @@ The following configuration statements in `config.ini` are recognized: - In Chainbase, we are emitting the operations inside an undo just before the operation is actually executed and checked for success ([See here](https://github.com/EOSIO/chainbase/compare/master...mmcs85:master#diff-298563b6c76ef92100c2ea27c06cb08bR355)). - - This is because we are not sure how to get a reference to the item after - it is given to the `modify` and `emplace` functions using `std::move`. + - We would like advice on the best way (without degrading performance) to get + the item after it is given to the `modify` and `emplace` functions using `std::move`. - We don't like this inconsistency since it is emitting the event before being sure that the operation actually took place, but we think it might not be an issue since those operations are undoing previous ones and it From 6d852198affd349517d5d2763755e05fe014210a Mon Sep 17 00:00:00 2001 From: Andres Berrios Date: Wed, 14 Nov 2018 16:44:28 -0800 Subject: [PATCH 50/56] Update link to referenced code in Readme --- plugins/statetrack_plugin/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/statetrack_plugin/README.md b/plugins/statetrack_plugin/README.md index a5aaee009a4..19deecfba38 100644 --- a/plugins/statetrack_plugin/README.md +++ b/plugins/statetrack_plugin/README.md @@ -44,7 +44,7 @@ The following configuration statements in `config.ini` are recognized: - In Chainbase, we are emitting the operations inside an undo just before the operation is actually executed and checked for success - ([See here](https://github.com/EOSIO/chainbase/compare/master...mmcs85:master#diff-298563b6c76ef92100c2ea27c06cb08bR355)). + ([See here](https://github.com/EOSIO/chainbase/compare/master...mmcs85:master#diff-298563b6c76ef92100c2ea27c06cb08bR390)). - We would like advice on the best way (without degrading performance) to get the item after it is given to the `modify` and `emplace` functions using `std::move`. - We don't like this inconsistency since it is emitting the event before being From ceb286cf6d07d05965053b2a7cc82f2bdebe4423 Mon Sep 17 00:00:00 2001 From: Andres Berrios Date: Wed, 14 Nov 2018 17:18:21 -0800 Subject: [PATCH 51/56] Add purposes and open questions to Readme --- plugins/statetrack_plugin/README.md | 47 +++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/plugins/statetrack_plugin/README.md b/plugins/statetrack_plugin/README.md index 19deecfba38..cbecd852ff3 100644 --- a/plugins/statetrack_plugin/README.md +++ b/plugins/statetrack_plugin/README.md @@ -2,8 +2,24 @@ This plugin adds hooks on chainbase operations emplace, modify, remove, and undo, and routes the objects and revision numbers to a ZMQ socket. - -It can hook to the operations that happen on the state DBs of all contracts +This approach has different purposes: + +- Allowing developers to get fine-grained access to the state DB operations. + - To mirror the state DB into a traditional database system such as MongoDB + or Postgres without them having to implement any further logic or duplicate + the DB write logic of their smart contracts. + - To execute side-effects associated to specific database operations rather than + specific contract actions. + - To propagate database operations to the dapp frontends through WebSockets + for enabling real-time updates to user interfaces. +- Seamlessly handling any forks and transaction failures by leveraging nodeos' own + behavior in maintaining the consistency of its own databases through + undo and commit operations. +- Minimizing the work that happens inside nodeos and letting receivers such as + [statemirror](https://github.com/andresberrios/statemirror) or a demux-js reader + handle the more complex logic and more performance intensive work. + +The plugin can hook to the operations that happen on the state DBs of all contracts deployed to the chain, and it can use filters to limit the operations that should be sent to only the ones happening on the contracts, scopes, and tables that the user is interested in tracking, thus limiting the performance impact to a minimum. @@ -53,6 +69,33 @@ The following configuration statements in `config.ini` are recognized: would not be an expected scenario for them to fail unless there was a bug or a database inconsistency (maybe caused by bad locking mechanisms when using multiple threads?). +- Performance + - Serialization options + - JSON serializing on the plugin + - Sending binary and decoding to JSON in the receiver + - Would need to fetch ABIs, which would make it slower overall but put less load on nodeos + - Place operations in a queue during transaction processing + - Process queue, serialize each message to JSON and send to receiver + - After transaction is processed (prevents failed transactions from sending do and undo operations) + - During transaction processing but in a separate thread + - Risk of making transaction processing take longer and hit CPU time limit + - Only for transactions sent directly to this node? + - Does it affect validated transactions/blocks received from the network? +- Consensus + - Forks + - How to induce and test recovery + - Exceptions + - Other potential disruptions +- Socket communications + - In the current implementation, sending a message blocks until it is received (ZMQ PUSH socket) + - This is useful in case receiver goes down, since nodeos will wait for it to come back up + - Instead of maintaining a local in-memory queue of messages that can build up too much if it carries on + - However, it will make nodeos unavailable for API calls, and maybe other side-effects + - Lost messages + - When receiver pulls a message from the socket but then crashes + - Implement message acknowledgement signal? + - Implement an intermediate receiver that only manages the queue? + - Implement a very safe persistent received-messages queue in receiver from which to recover if it crashes? ## Building From 3824faa5f1d91e69037b41ec553cb2932df2a687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Mon, 26 Nov 2018 23:46:31 +0000 Subject: [PATCH 52/56] add trx_id and act_index to db operations (work in progress) --- libraries/chain/apply_context.cpp | 34 +++++++++++++-- .../include/eosio/chain/apply_context.hpp | 2 +- .../chain/include/eosio/chain/controller.hpp | 2 + libraries/chainbase | 2 +- .../statetrack_plugin_impl.hpp | 31 ++++++++++---- .../statetrack_plugin/statetrack_plugin.cpp | 5 +++ .../statetrack_plugin_impl.cpp | 42 +++++++++++++++++++ 7 files changed, 105 insertions(+), 13 deletions(-) diff --git a/libraries/chain/apply_context.cpp b/libraries/chain/apply_context.cpp index de1450013d8..f9315fbac1b 100644 --- a/libraries/chain/apply_context.cpp +++ b/libraries/chain/apply_context.cpp @@ -29,7 +29,33 @@ static inline void print_debug(account_name receiver, const action_trace& ar) { } } -void apply_context::exec_one( action_trace& trace ) +/** + * Plugins / observers listening to signals emited (such as accepted_transaction) might trigger + * errors and throw exceptions. Unless those exceptions are caught it could impact consensus and/or + * cause a node to fork. + * + * If it is ever desirable to let a signal handler bubble an exception out of this method + * a full audit of its uses needs to be undertaken. + * + */ +template +void emit( const Signal& s, Arg&& a ) { + try { + s(std::forward(a)); + } catch (boost::interprocess::bad_alloc& e) { + wlog( "bad alloc" ); + throw e; + } catch ( controller_emit_signal_exception& e ) { + wlog( "${details}", ("details", e.to_detail_string()) ); + throw e; + } catch ( fc::exception& e ) { + wlog( "${details}", ("details", e.to_detail_string()) ); + } catch ( ... ) { + wlog( "signal handler threw exception" ); + } +} + +void apply_context::exec_one( action_trace& trace, bool inline_action ) { auto start = fc::time_point::now(); @@ -44,6 +70,8 @@ void apply_context::exec_one( action_trace& trace ) trace.act = act; trace.context_free = context_free; + emit(control.pre_apply_action, std::pair(trace, inline_action)); + const auto& cfg = control.get_global_properties().configuration; try { try { @@ -113,11 +141,11 @@ void apply_context::finalize_trace( action_trace& trace, const fc::time_point& s void apply_context::exec( action_trace& trace ) { _notified.push_back(receiver); - exec_one( trace ); + exec_one( trace, false ); for( uint32_t i = 1; i < _notified.size(); ++i ) { receiver = _notified[i]; trace.inline_traces.emplace_back( ); - exec_one( trace.inline_traces.back() ); + exec_one( trace.inline_traces.back(), true ); } if( _cfa_inline_actions.size() > 0 || _inline_actions.size() > 0 ) { diff --git a/libraries/chain/include/eosio/chain/apply_context.hpp b/libraries/chain/include/eosio/chain/apply_context.hpp index a253d950358..cafb8ba6870 100644 --- a/libraries/chain/include/eosio/chain/apply_context.hpp +++ b/libraries/chain/include/eosio/chain/apply_context.hpp @@ -472,7 +472,7 @@ class apply_context { /// Execution methods: public: - void exec_one( action_trace& trace ); + void exec_one( action_trace& trace, bool inline_action ); void exec( action_trace& trace ); void execute_inline( action&& a ); void execute_context_free_inline( action&& a ); diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index ec7b53fafc0..38fc25ca744 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -253,6 +253,8 @@ namespace eosio { namespace chain { signal accepted_confirmation; signal bad_alloc; + signal)> pre_apply_action; + /* signal pre_apply_block; signal post_apply_block; diff --git a/libraries/chainbase b/libraries/chainbase index e52d821c73d..dbc55dfbfc5 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit e52d821c73d33279be3a3733374115aff78eb338 +Subproject commit dbc55dfbfc566968147bd1eca0613298197d4dcb diff --git a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp index 45c3ab53177..9b11d7dd56f 100644 --- a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp +++ b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp @@ -61,11 +61,14 @@ struct db_account struct db_table { - op_type_enum op_type; - account_name code; - fc::string scope; - table_name table; - account_name payer; + op_type_enum op_type; + transaction_id_type trx_id; + size_t act_index; + bool inline_action; + account_name code; + fc::string scope; + table_name table; + account_name payer; }; struct db_op : db_table @@ -89,6 +92,13 @@ struct db_action uint32_t last_irreversible_block; }; +struct db_action_trac +{ + transaction_id_type trx_id; + bool inline_action; + int64_t op_size; +}; + struct filter_entry { name code; @@ -139,13 +149,15 @@ class statetrack_plugin_impl void on_applied_op(const database &db, const permission_object &po, op_type_enum op_type); void on_applied_op(const database &db, const permission_link_object &plo, op_type_enum op_type); void on_applied_op(const database &db, const key_value_object &kvo, op_type_enum op_type); - //generic_index undo void on_applied_rev(const int64_t revision, op_type_enum op_type); + //generic_index undo + void on_applied_undo(const int64_t revision); //blocks and transactions void on_applied_transaction(const transaction_trace_ptr &ttp); void on_accepted_block(const block_state_ptr &bsp); void on_action_trace(const action_trace &at, const block_state_ptr &bsp); void on_irreversible_block(const block_state_ptr &bsp); + void on_pre_apply_action(std::pair& trace); template void create_index_events(const database &db) { @@ -168,7 +180,7 @@ class statetrack_plugin_impl connections.emplace_back( fc::optional(index.applied_undo.connect([&](const int64_t revision) { - on_applied_rev(revision, op_type_enum::REV_UNDO); + on_applied_undo(revision); }))); } @@ -203,6 +215,9 @@ class statetrack_plugin_impl zmq::context_t *context; zmq::socket_t *sender_socket; + bool is_undo_state; + std::vector current_trx_actions; + std::map cached_traces; std::map> blacklist_actions; @@ -220,7 +235,7 @@ class statetrack_plugin_impl FC_REFLECT_ENUM(eosio::op_type_enum, (TABLE_REMOVE)(ROW_CREATE)(ROW_MODIFY)(ROW_REMOVE)(TRX_ACTION)(REV_UNDO)(REV_COMMIT)) FC_REFLECT(eosio::db_account, (name)(vm_type)(vm_version)(privileged)(last_code_update)(code_version)(creation_date)) -FC_REFLECT(eosio::db_table, (op_type)(code)(scope)(table)(payer)) +FC_REFLECT(eosio::db_table, (op_type)(trx_id)(act_index)(inline_action)(code)(scope)(table)(payer)) FC_REFLECT_DERIVED(eosio::db_op, (eosio::db_table), (id)(value)) FC_REFLECT(eosio::db_rev, (op_type)(revision)) FC_REFLECT(eosio::db_action, (block_num)(block_time)(action_trace)(last_irreversible_block)) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index e093bb07d4b..e6f9c00c617 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -121,6 +121,11 @@ void statetrack_plugin::plugin_initialize(const variables_map &options) fc::optional(chain.irreversible_block.connect([&](const block_state_ptr &bsp) { my->on_irreversible_block(bsp); }))); + + my->connections.emplace_back( + fc::optional(chain.pre_apply_action.connect([&](std::pair trace){ + my->on_pre_apply_action(trace); + }))); } FC_LOG_AND_RETHROW() } diff --git a/plugins/statetrack_plugin/statetrack_plugin_impl.cpp b/plugins/statetrack_plugin/statetrack_plugin_impl.cpp index be3943c801f..358eb0e89e7 100644 --- a/plugins/statetrack_plugin/statetrack_plugin_impl.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin_impl.cpp @@ -168,6 +168,23 @@ db_op statetrack_plugin_impl::get_db_op(const database &db, const table_id_objec op.id = kvo.primary_key; op.op_type = op_type; + + db_action_trac& dat = current_trx_actions.back(); + op.trx_id = dat.trx_id; + op.act_index = current_trx_actions.size(); + op.inline_action = dat.inline_action; + + if(is_undo_state) { + dat.op_size--; + + if(dat.op_size <= 0) + current_trx_actions.pop_back(); + } + else { + + dat.op_size++; + } + op.code = tio.code; op.scope = scope_sym_to_string(tio.scope); op.table = tio.table; @@ -299,12 +316,26 @@ void statetrack_plugin_impl::on_applied_rev(const int64_t revision, op_type_enum send_zmq_msg(data); } +void statetrack_plugin_impl::on_applied_undo(const int64_t revision) +{ + if (sender_socket == nullptr) + return; + + is_undo_state = true; + + fc::string data = fc::json::to_string(get_db_rev(revision, op_type_enum::REV_UNDO)); + ilog("STATETRACK ${op_type}: ${data}", ("op_type", op_type_enum::REV_UNDO)("data", data)); + send_zmq_msg(data); +} + void statetrack_plugin_impl::on_applied_transaction(const transaction_trace_ptr &ttp) { if (ttp->receipt) { cached_traces[ttp->id] = ttp; } + + current_trx_actions.clear(); } void statetrack_plugin_impl::on_accepted_block(const block_state_ptr &bsp) @@ -393,4 +424,15 @@ void statetrack_plugin_impl::on_irreversible_block(const block_state_ptr &bsp) send_zmq_msg(data); } +void statetrack_plugin_impl::on_pre_apply_action(std::pair& trace) +{ + db_action_trac dat; + dat.trx_id = trace.first.trx_id; + dat.inline_action = trace.second; + dat.op_size = 0; + + current_trx_actions.emplace_back(dat); + is_undo_state = false; +} + } // namespace eosio \ No newline at end of file From aec06de3d1f2cf6586b4c2405a0f113e8719d3e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Tue, 27 Nov 2018 09:55:24 +0000 Subject: [PATCH 53/56] more work on op trx and action tracking --- libraries/chain/apply_context.cpp | 5 +- .../chain/include/eosio/chain/controller.hpp | 2 +- .../statetrack_plugin_impl.hpp | 41 +++++----- .../statetrack_plugin/statetrack_plugin.cpp | 4 +- .../statetrack_plugin_impl.cpp | 77 +++++++++++++------ 5 files changed, 80 insertions(+), 49 deletions(-) diff --git a/libraries/chain/apply_context.cpp b/libraries/chain/apply_context.cpp index f9315fbac1b..444ad41ecb4 100644 --- a/libraries/chain/apply_context.cpp +++ b/libraries/chain/apply_context.cpp @@ -70,8 +70,6 @@ void apply_context::exec_one( action_trace& trace, bool inline_action ) trace.act = act; trace.context_free = context_free; - emit(control.pre_apply_action, std::pair(trace, inline_action)); - const auto& cfg = control.get_global_properties().configuration; try { try { @@ -141,6 +139,9 @@ void apply_context::finalize_trace( action_trace& trace, const fc::time_point& s void apply_context::exec( action_trace& trace ) { _notified.push_back(receiver); + + emit(control.pre_apply_action, trx_context.id); + exec_one( trace, false ); for( uint32_t i = 1; i < _notified.size(); ++i ) { receiver = _notified[i]; diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index 38fc25ca744..7f893215415 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -253,7 +253,7 @@ namespace eosio { namespace chain { signal accepted_confirmation; signal bad_alloc; - signal)> pre_apply_action; + signal pre_apply_action; /* signal pre_apply_block; diff --git a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp index 9b11d7dd56f..fc5522daf95 100644 --- a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp +++ b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp @@ -47,24 +47,9 @@ enum op_type_enum REV_COMMIT = 6 }; -struct db_account -{ - account_name name; - uint8_t vm_type = 0; - uint8_t vm_version = 0; - bool privileged = false; - - time_point last_code_update; - digest_type code_version; - block_timestamp_type creation_date; -}; - struct db_table { op_type_enum op_type; - transaction_id_type trx_id; - size_t act_index; - bool inline_action; account_name code; fc::string scope; table_name table; @@ -73,8 +58,22 @@ struct db_table struct db_op : db_table { - uint64_t id; - fc::variant value; + uint64_t id; + transaction_id_type trx_id; + size_t act_index; + fc::variant value; +}; + +struct db_account +{ + account_name name; + uint8_t vm_type = 0; + uint8_t vm_version = 0; + bool privileged = false; + + time_point last_code_update; + digest_type code_version; + block_timestamp_type creation_date; }; struct db_rev @@ -95,7 +94,6 @@ struct db_action struct db_action_trac { transaction_id_type trx_id; - bool inline_action; int64_t op_size; }; @@ -129,6 +127,7 @@ class statetrack_plugin_impl void send_zmq_msg(const fc::string content); bool filter(const account_name &code, const fc::string &scope, const table_name &table); + void build_db_op_trx(db_op& op); const table_id_object *get_kvo_tio(const database &db, const key_value_object &kvo); const account_object &get_tio_co(const database &db, const table_id_object &tio); @@ -157,7 +156,7 @@ class statetrack_plugin_impl void on_accepted_block(const block_state_ptr &bsp); void on_action_trace(const action_trace &at, const block_state_ptr &bsp); void on_irreversible_block(const block_state_ptr &bsp); - void on_pre_apply_action(std::pair& trace); + void on_pre_apply_action(transaction_id_type trx_id); template void create_index_events(const database &db) { @@ -235,7 +234,7 @@ class statetrack_plugin_impl FC_REFLECT_ENUM(eosio::op_type_enum, (TABLE_REMOVE)(ROW_CREATE)(ROW_MODIFY)(ROW_REMOVE)(TRX_ACTION)(REV_UNDO)(REV_COMMIT)) FC_REFLECT(eosio::db_account, (name)(vm_type)(vm_version)(privileged)(last_code_update)(code_version)(creation_date)) -FC_REFLECT(eosio::db_table, (op_type)(trx_id)(act_index)(inline_action)(code)(scope)(table)(payer)) -FC_REFLECT_DERIVED(eosio::db_op, (eosio::db_table), (id)(value)) +FC_REFLECT(eosio::db_table, (op_type)(code)(scope)(table)(payer)) +FC_REFLECT_DERIVED(eosio::db_op, (eosio::db_table), (id)(trx_id)(act_index)(value)) FC_REFLECT(eosio::db_rev, (op_type)(revision)) FC_REFLECT(eosio::db_action, (block_num)(block_time)(action_trace)(last_irreversible_block)) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index e6f9c00c617..175f3c29ece 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -123,8 +123,8 @@ void statetrack_plugin::plugin_initialize(const variables_map &options) }))); my->connections.emplace_back( - fc::optional(chain.pre_apply_action.connect([&](std::pair trace){ - my->on_pre_apply_action(trace); + fc::optional(chain.pre_apply_action.connect([&](transaction_id_type trx_id){ + my->on_pre_apply_action(trx_id); }))); } FC_LOG_AND_RETHROW() diff --git a/plugins/statetrack_plugin/statetrack_plugin_impl.cpp b/plugins/statetrack_plugin/statetrack_plugin_impl.cpp index 358eb0e89e7..b0afbf73b66 100644 --- a/plugins/statetrack_plugin/statetrack_plugin_impl.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin_impl.cpp @@ -64,6 +64,30 @@ bool statetrack_plugin_impl::filter(const account_name &code, const fc::string & return true; } +void statetrack_plugin_impl::build_db_op_trx(db_op& op) +{ + if(current_trx_actions.size() == 0) { + op.act_index = 0; + return; + } + + db_action_trac& dat = current_trx_actions.back(); + op.trx_id = dat.trx_id; + op.act_index = current_trx_actions.size() - 1; + + if(is_undo_state) { + dat.op_size--; + + if(dat.op_size <= 0) + current_trx_actions.pop_back(); + } + else { + dat.op_size++; + } + + ilog("STATETRACK db_action_trac: ${trxid} ${opsize}", ("trxid", op.trx_id) ("opsize", dat.op_size)); +} + const table_id_object *statetrack_plugin_impl::get_kvo_tio(const database &db, const key_value_object &kvo) { return db.find(kvo.t_id); @@ -112,6 +136,9 @@ db_op statetrack_plugin_impl::get_db_op(const database &db, const account_object op.id = co.id._id; op.op_type = op_type; + + build_db_op_trx(op); + op.code = system; op.scope = system.to_string(); op.table = N(accounts); @@ -138,6 +165,9 @@ db_op statetrack_plugin_impl::get_db_op(const database &db, const permission_obj op.id = po.id._id; op.op_type = op_type; + + build_db_op_trx(op); + op.code = system; op.scope = system.to_string(); op.table = N(permissions); @@ -154,6 +184,9 @@ db_op statetrack_plugin_impl::get_db_op(const database &db, const permission_lin op.id = plo.id._id; op.op_type = op_type; + + build_db_op_trx(op); + op.code = system; op.scope = system.to_string(); op.table = N(permission_links); @@ -169,21 +202,7 @@ db_op statetrack_plugin_impl::get_db_op(const database &db, const table_id_objec op.id = kvo.primary_key; op.op_type = op_type; - db_action_trac& dat = current_trx_actions.back(); - op.trx_id = dat.trx_id; - op.act_index = current_trx_actions.size(); - op.inline_action = dat.inline_action; - - if(is_undo_state) { - dat.op_size--; - - if(dat.op_size <= 0) - current_trx_actions.pop_back(); - } - else { - - dat.op_size++; - } + build_db_op_trx(op); op.code = tio.code; op.scope = scope_sym_to_string(tio.scope); @@ -215,7 +234,9 @@ void statetrack_plugin_impl::on_applied_table(const database &db, const table_id if (filter(code, scope, table)) { - db_table table; + db_op table; + + build_db_op_trx(table); table.op_type = op_type; table.code = tio.code; @@ -333,9 +354,11 @@ void statetrack_plugin_impl::on_applied_transaction(const transaction_trace_ptr if (ttp->receipt) { cached_traces[ttp->id] = ttp; - } - current_trx_actions.clear(); + ilog("STATETRACK applied trx"); + + current_trx_actions.clear(); + } } void statetrack_plugin_impl::on_accepted_block(const block_state_ptr &bsp) @@ -410,7 +433,7 @@ void statetrack_plugin_impl::on_action_trace(const action_trace &at, const block op.value = fc::variant(da); fc::string data = fc::json::to_string(op); - ilog("STATETRACK ${op_type}: ${data}", ("op_type", op_type_enum::TRX_ACTION)("data", data)); + //ilog("STATETRACK ${op_type}: ${data}", ("op_type", op_type_enum::TRX_ACTION)("data", data)); send_zmq_msg(data); } @@ -420,19 +443,27 @@ void statetrack_plugin_impl::on_irreversible_block(const block_state_ptr &bsp) return; fc::string data = fc::json::to_string(get_db_rev(bsp->block_num, op_type_enum::REV_COMMIT)); - ilog("STATETRACK ${op_type}: ${data}", ("op_type", op_type_enum::REV_COMMIT)("data", data)); + //ilog("STATETRACK ${op_type}: ${data}", ("op_type", op_type_enum::REV_COMMIT)("data", data)); send_zmq_msg(data); } -void statetrack_plugin_impl::on_pre_apply_action(std::pair& trace) +void statetrack_plugin_impl::on_pre_apply_action(transaction_id_type trx_id) { + auto it = find_if(current_trx_actions.begin(), current_trx_actions.end(), [&] (const auto& dat) { + return dat.trx_id == trx_id; + }); + + if(it != current_trx_actions.end()) + return; + db_action_trac dat; - dat.trx_id = trace.first.trx_id; - dat.inline_action = trace.second; + dat.trx_id = trx_id; dat.op_size = 0; current_trx_actions.emplace_back(dat); is_undo_state = false; + + ilog("on_pre_apply_action trx: ${trxid}", ("trxid", trx_id)); } } // namespace eosio \ No newline at end of file From 12ecbd979c5955f4470ee94faaefd776fc2f219f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Sat, 1 Dec 2018 20:55:07 +0000 Subject: [PATCH 54/56] dev sending trx with actions and operations --- libraries/chain/apply_context.cpp | 5 +- .../chain/include/eosio/chain/controller.hpp | 2 +- .../statetrack_plugin_impl.hpp | 120 +++++--- .../statetrack_plugin/statetrack_plugin.cpp | 7 +- .../statetrack_plugin_impl.cpp | 269 ++++++++---------- 5 files changed, 208 insertions(+), 195 deletions(-) diff --git a/libraries/chain/apply_context.cpp b/libraries/chain/apply_context.cpp index 444ad41ecb4..ceed31ae52c 100644 --- a/libraries/chain/apply_context.cpp +++ b/libraries/chain/apply_context.cpp @@ -140,9 +140,10 @@ void apply_context::exec( action_trace& trace ) { _notified.push_back(receiver); - emit(control.pre_apply_action, trx_context.id); - exec_one( trace, false ); + + emit(control.applied_action, trace); + for( uint32_t i = 1; i < _notified.size(); ++i ) { receiver = _notified[i]; trace.inline_traces.emplace_back( ); diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index 7f893215415..f6deacbc584 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -253,7 +253,7 @@ namespace eosio { namespace chain { signal accepted_confirmation; signal bad_alloc; - signal pre_apply_action; + signal applied_action; /* signal pre_apply_block; diff --git a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp index fc5522daf95..87c49838548 100644 --- a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp +++ b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp @@ -17,12 +17,16 @@ #include #include +#include + namespace eosio { using namespace chain; using namespace chainbase; +using namespace bmi; using boost::signals2::connection; +using bmi::random_access; typedef typename get_index_type::type co_index_type; typedef typename get_index_type::type po_index_type; @@ -30,12 +34,6 @@ typedef typename get_index_type::type plo_index_type; typedef typename get_index_type::type ti_index_type; typedef typename get_index_type::type kv_index_type; -typedef account_object::id_type co_id_type; -typedef permission_usage_object::id_type puo_id_type; -typedef permission_object::id_type po_id_type; -typedef permission_link_object::id_type plo_id_type; -typedef key_value_object::id_type kvo_id_type; - enum op_type_enum { TABLE_REMOVE = 0, @@ -47,23 +45,6 @@ enum op_type_enum REV_COMMIT = 6 }; -struct db_table -{ - op_type_enum op_type; - account_name code; - fc::string scope; - table_name table; - account_name payer; -}; - -struct db_op : db_table -{ - uint64_t id; - transaction_id_type trx_id; - size_t act_index; - fc::variant value; -}; - struct db_account { account_name name; @@ -82,19 +63,33 @@ struct db_rev int64_t revision; }; -struct db_action +struct db_op { - uint64_t global_action_seq; - block_num_type block_num; - block_timestamp_type block_time; - fc::variant action_trace; - uint32_t last_irreversible_block; + int64_t oid; + uint64_t id; + uint64_t actionid; + + op_type_enum op_type; + account_name code; + fc::string scope; + table_name table; + account_name payer; + + fc::variant value; }; -struct db_action_trac +struct db_action { + uint64_t actionid; transaction_id_type trx_id; - int64_t op_size; + block_num_type block_num; + fc::variant trace; + std::vector ops; +}; + +struct db_transaction { + transaction_id_type trx_id; + std::vector actions; }; struct filter_entry @@ -114,20 +109,42 @@ struct filter_entry } }; +struct IndexByOId {}; +struct IndexByActionId {}; +struct IndexByTrxId {}; +struct IndexByBlockNum {}; + +typedef multi_index_container< + db_op, + indexed_by< + random_access<>, + ordered_unique, member >, + ordered_non_unique, member > + > +> db_op_container; + +typedef multi_index_container< + db_action, + indexed_by< + random_access<>, + ordered_non_unique, member >, + ordered_non_unique, member > + > +> db_action_container; + class statetrack_plugin_impl { public: - statetrack_plugin_impl() - { - blacklist_actions.emplace(std::make_pair(config::system_account_name, - std::set{N(onblock)})); - blacklist_actions.emplace(std::make_pair(N(blocktwitter), - std::set{N(tweet)})); - } - void send_zmq_msg(const fc::string content); bool filter(const account_name &code, const fc::string &scope, const table_name &table); - void build_db_op_trx(db_op& op); + + void build_blacklist(); + + void init_current_block(); const table_id_object *get_kvo_tio(const database &db, const key_value_object &kvo); const account_object &get_tio_co(const database &db, const table_id_object &tio); @@ -154,9 +171,8 @@ class statetrack_plugin_impl //blocks and transactions void on_applied_transaction(const transaction_trace_ptr &ttp); void on_accepted_block(const block_state_ptr &bsp); - void on_action_trace(const action_trace &at, const block_state_ptr &bsp); void on_irreversible_block(const block_state_ptr &bsp); - void on_pre_apply_action(transaction_id_type trx_id); + void on_applied_action(action_trace& trace); template void create_index_events(const database &db) { @@ -215,7 +231,18 @@ class statetrack_plugin_impl zmq::socket_t *sender_socket; bool is_undo_state; - std::vector current_trx_actions; + + uint64_t current_action_index = 0; + db_action current_action; + + + db_action_container reversible_actions; + db_op_container reversible_ops; + + std::map po_ops; + std::map plo_ops; + std::map tio_ops; + std::map kvo_ops; std::map cached_traces; std::map> blacklist_actions; @@ -234,7 +261,8 @@ class statetrack_plugin_impl FC_REFLECT_ENUM(eosio::op_type_enum, (TABLE_REMOVE)(ROW_CREATE)(ROW_MODIFY)(ROW_REMOVE)(TRX_ACTION)(REV_UNDO)(REV_COMMIT)) FC_REFLECT(eosio::db_account, (name)(vm_type)(vm_version)(privileged)(last_code_update)(code_version)(creation_date)) -FC_REFLECT(eosio::db_table, (op_type)(code)(scope)(table)(payer)) -FC_REFLECT_DERIVED(eosio::db_op, (eosio::db_table), (id)(trx_id)(act_index)(value)) +FC_REFLECT(eosio::db_op, (oid)(id)(op_type)(code)(scope)(table)(payer)(actionid)(value)) FC_REFLECT(eosio::db_rev, (op_type)(revision)) -FC_REFLECT(eosio::db_action, (block_num)(block_time)(action_trace)(last_irreversible_block)) +FC_REFLECT(eosio::db_action, (trx_id)(trace)(ops)) +FC_REFLECT(eosio::db_transaction, (trx_id)(actions)) + diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index 175f3c29ece..474e76eaef0 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -86,6 +86,8 @@ void statetrack_plugin::plugin_initialize(const variables_map &options) my->abi_serializer_max_time = my->chain_plug->get_abi_serializer_max_time(); + my->init_current_block(); + ilog("Binding database events"); // account_object events @@ -123,8 +125,8 @@ void statetrack_plugin::plugin_initialize(const variables_map &options) }))); my->connections.emplace_back( - fc::optional(chain.pre_apply_action.connect([&](transaction_id_type trx_id){ - my->on_pre_apply_action(trx_id); + fc::optional(chain.applied_action.connect([&](action_trace& trace){ + my->on_applied_action(trace); }))); } FC_LOG_AND_RETHROW() @@ -132,6 +134,7 @@ void statetrack_plugin::plugin_initialize(const variables_map &options) void statetrack_plugin::plugin_startup() { + } void statetrack_plugin::plugin_shutdown() diff --git a/plugins/statetrack_plugin/statetrack_plugin_impl.cpp b/plugins/statetrack_plugin/statetrack_plugin_impl.cpp index b0afbf73b66..0c3eccc4d04 100644 --- a/plugins/statetrack_plugin/statetrack_plugin_impl.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin_impl.cpp @@ -64,28 +64,15 @@ bool statetrack_plugin_impl::filter(const account_name &code, const fc::string & return true; } -void statetrack_plugin_impl::build_db_op_trx(db_op& op) -{ - if(current_trx_actions.size() == 0) { - op.act_index = 0; - return; - } - - db_action_trac& dat = current_trx_actions.back(); - op.trx_id = dat.trx_id; - op.act_index = current_trx_actions.size() - 1; - - if(is_undo_state) { - dat.op_size--; - - if(dat.op_size <= 0) - current_trx_actions.pop_back(); - } - else { - dat.op_size++; - } +void statetrack_plugin_impl::build_blacklist() { + blacklist_actions.emplace(std::make_pair(config::system_account_name, + std::set{N(onblock)})); + blacklist_actions.emplace(std::make_pair(N(blocktwitter), + std::set{N(tweet)})); +} - ilog("STATETRACK db_action_trac: ${trxid} ${opsize}", ("trxid", op.trx_id) ("opsize", dat.op_size)); +void statetrack_plugin_impl::init_current_block() { + is_undo_state = false; } const table_id_object *statetrack_plugin_impl::get_kvo_tio(const database &db, const key_value_object &kvo) @@ -134,10 +121,9 @@ db_op statetrack_plugin_impl::get_db_op(const database &db, const account_object name system = N(system); - op.id = co.id._id; + op.oid = co.id._id; op.op_type = op_type; - - build_db_op_trx(op); + op.actionid = current_action_index; op.code = system; op.scope = system.to_string(); @@ -163,10 +149,9 @@ db_op statetrack_plugin_impl::get_db_op(const database &db, const permission_obj name system = N(system); - op.id = po.id._id; + op.oid = po.id._id; op.op_type = op_type; - - build_db_op_trx(op); + op.actionid = current_action_index; op.code = system; op.scope = system.to_string(); @@ -182,10 +167,9 @@ db_op statetrack_plugin_impl::get_db_op(const database &db, const permission_lin name system = N(system); - op.id = plo.id._id; + op.oid = plo.id._id; op.op_type = op_type; - - build_db_op_trx(op); + op.actionid = current_action_index; op.code = system; op.scope = system.to_string(); @@ -199,10 +183,10 @@ db_op statetrack_plugin_impl::get_db_op(const database &db, const table_id_objec { db_op op; + op.oid = kvo.id._id; op.id = kvo.primary_key; op.op_type = op_type; - - build_db_op_trx(op); + op.actionid = current_action_index; op.code = tio.code; op.scope = scope_sym_to_string(tio.scope); @@ -234,18 +218,22 @@ void statetrack_plugin_impl::on_applied_table(const database &db, const table_id if (filter(code, scope, table)) { - db_op table; + db_op op; + + op.oid = tio.id._id; + op.op_type = op_type; + op.actionid = current_action_index; + + op.code = tio.code; + op.scope = scope_sym_to_string(tio.scope); + op.table = tio.table; - build_db_op_trx(table); + //TODO op optimizations - table.op_type = op_type; - table.code = tio.code; - table.scope = scope_sym_to_string(tio.scope); - table.table = tio.table; + current_action.ops.emplace_back(op); - fc::string data = fc::json::to_string(table); - ilog("STATETRACK table_id_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); - send_zmq_msg(data); + fc::string data = fc::json::to_string(op); + //ilog("STATETRACK table_id_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); } } @@ -261,9 +249,34 @@ void statetrack_plugin_impl::on_applied_op(const database &db, const account_obj if (filter(code, scope, table)) { - fc::string data = fc::json::to_string(get_db_op(db, co, op_type)); - ilog("STATETRACK account_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); - send_zmq_msg(data); + auto op = get_db_op(db, co, op_type); + + //TODO op optimizations + // switch(op_type) { + // case op_type_enum::ROW_REMOVE: + // if(co_newids.get.count(co.id._id)) { + // co_newids.get.erase(co.id._id); + // return; + // } + // else if(co_old_values.get.count(co.id._id)) { + // co_old_values.get.erase(co.id._id); + // } + // break; + // case op_type_enum::ROW_MODIFY: + // if(co_newids.get.count(co.id._id)) { + // op.op_type = op_type_enum::ROW_CREATE; + // co_newids.get.erase(co.id._id); + // } + // else if(co_old_values.get.count(co.id._id)) { + // co_old_values.get.erase(co.id._id); + // } + // break; + // } + + current_action.ops.emplace_back(op); + + fc::string data = fc::json::to_string(op); + //ilog("STATETRACK account_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); } } @@ -279,9 +292,14 @@ void statetrack_plugin_impl::on_applied_op(const database &db, const permission_ if (filter(code, scope, table)) { - fc::string data = fc::json::to_string(get_db_op(db, po, op_type)); - ilog("STATETRACK permission_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); - send_zmq_msg(data); + auto op = get_db_op(db, po, op_type); + + //TODO op optimizations + + current_action.ops.emplace_back(op); + + fc::string data = fc::json::to_string(op); + //ilog("STATETRACK permission_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); } } @@ -297,9 +315,14 @@ void statetrack_plugin_impl::on_applied_op(const database &db, const permission_ if (filter(code, scope, table)) { - fc::string data = fc::json::to_string(get_db_op(db, plo, op_type)); - ilog("STATETRACK permission_link_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); - send_zmq_msg(data); + auto op = get_db_op(db, plo, op_type); + + //TODO op optimizations + + current_action.ops.emplace_back(op); + + fc::string data = fc::json::to_string(op); + //ilog("STATETRACK permission_link_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); } } @@ -321,9 +344,14 @@ void statetrack_plugin_impl::on_applied_op(const database &db, const key_value_o if (filter(code, scope, table)) { - fc::string data = fc::json::to_string(get_db_op(db, *tio_ptr, kvo, op_type)); - ilog("STATETRACK key_value_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); - send_zmq_msg(data); + auto op = get_db_op(db, *tio_ptr, kvo, op_type); + + //TODO op optimizations + + current_action.ops.emplace_back(op); + + fc::string data = fc::json::to_string(op); + //ilog("STATETRACK key_value_object ${op_type}: ${data}", ("op_type", op_type)("data", data)); } } @@ -334,7 +362,7 @@ void statetrack_plugin_impl::on_applied_rev(const int64_t revision, op_type_enum fc::string data = fc::json::to_string(get_db_rev(revision, op_type)); ilog("STATETRACK ${op_type}: ${data}", ("op_type", op_type)("data", data)); - send_zmq_msg(data); + //send_zmq_msg(data); } void statetrack_plugin_impl::on_applied_undo(const int64_t revision) @@ -346,95 +374,47 @@ void statetrack_plugin_impl::on_applied_undo(const int64_t revision) fc::string data = fc::json::to_string(get_db_rev(revision, op_type_enum::REV_UNDO)); ilog("STATETRACK ${op_type}: ${data}", ("op_type", op_type_enum::REV_UNDO)("data", data)); - send_zmq_msg(data); + //send_zmq_msg(data); } void statetrack_plugin_impl::on_applied_transaction(const transaction_trace_ptr &ttp) { - if (ttp->receipt) - { - cached_traces[ttp->id] = ttp; + if (ttp->receipt) { - ilog("STATETRACK applied trx"); + db_transaction trx; - current_trx_actions.clear(); - } -} + trx.trx_id = ttp->id; -void statetrack_plugin_impl::on_accepted_block(const block_state_ptr &bsp) -{ - //TODO send accepted block information + auto trx_action_range = reversible_actions.get().equal_range(trx.trx_id); + for (auto& action : boost::make_iterator_range(trx_action_range)) { + + trx.actions.emplace_back(action); + auto newact = trx.actions.back(); - for (auto &t : bsp->block->transactions) - { - transaction_id_type id; - if (t.trx.contains()) - { - id = t.trx.get(); - } - else - { - id = t.trx.get().id(); + auto action_op_range = reversible_ops.get().equal_range(action.actionid); + for (auto& op : boost::make_iterator_range(action_op_range)) { + newact.ops.emplace_back(op); + } } - if (t.status == transaction_receipt_header::executed) - { - // Send traces only for executed transactions - auto it = cached_traces.find(id); - if (it == cached_traces.end() || !it->second->receipt) - { - ilog("missing trace for transaction {id}", ("id", id)); - continue; - } + //TODO send zmq transaction + //send_zmq_msg(data); - for (const auto &atrace : it->second->action_traces) - { - on_action_trace(atrace, bsp); - } + ilog("STATETRACK applied transaction: ${trx}", ("trx", trx)); + } + else { + auto trx_action_range = reversible_actions.get().equal_range(ttp->id); + for (auto& action : boost::make_iterator_range(trx_action_range)) { + auto action_op_range = reversible_ops.get().equal_range(action.actionid); + reversible_ops.get().erase(action_op_range.first, action_op_range.second); } + reversible_actions.get().erase(trx_action_range.first, trx_action_range.second); } - - cached_traces.clear(); } -void statetrack_plugin_impl::on_action_trace(const action_trace &at, const block_state_ptr &bsp) +void statetrack_plugin_impl::on_accepted_block(const block_state_ptr &bsp) { - if (sender_socket == nullptr) - return; - - // check the action against the blacklist - auto search_acc = blacklist_actions.find(at.act.account); - if (search_acc != blacklist_actions.end()) - { - auto search_act = search_acc->second.find(at.act.name); - if (search_act != search_acc->second.end()) - { - return; - } - } - - auto &chain = chain_plug->chain(); - - db_op op; - - name system = N(system); - op.id = at.receipt.global_sequence; - op.op_type = op_type_enum::TRX_ACTION; - op.code = system; - op.scope = scope_sym_to_string(system); - op.table = N(actions); - - db_action da; - da.block_num = bsp->block->block_num(); - da.block_time = bsp->block->timestamp; - da.action_trace = chain.to_variant_with_abi(at, abi_serializer_max_time); - da.last_irreversible_block = chain.last_irreversible_block_num(); - op.value = fc::variant(da); - - fc::string data = fc::json::to_string(op); - //ilog("STATETRACK ${op_type}: ${data}", ("op_type", op_type_enum::TRX_ACTION)("data", data)); - send_zmq_msg(data); } void statetrack_plugin_impl::on_irreversible_block(const block_state_ptr &bsp) @@ -443,27 +423,28 @@ void statetrack_plugin_impl::on_irreversible_block(const block_state_ptr &bsp) return; fc::string data = fc::json::to_string(get_db_rev(bsp->block_num, op_type_enum::REV_COMMIT)); - //ilog("STATETRACK ${op_type}: ${data}", ("op_type", op_type_enum::REV_COMMIT)("data", data)); - send_zmq_msg(data); + ilog("STATETRACK irreversible block: ${data}", ("data", data)); + + //send_zmq_msg(data); + + auto block_action_range = reversible_actions.get().equal_range(bsp->block_num); + for (auto& action : boost::make_iterator_range(block_action_range)) { + auto action_op_range = reversible_ops.get().equal_range(action.actionid); + reversible_ops.get().erase(action_op_range.first, action_op_range.second); + } + reversible_actions.get().erase(block_action_range.first, block_action_range.second); } -void statetrack_plugin_impl::on_pre_apply_action(transaction_id_type trx_id) +void statetrack_plugin_impl::on_applied_action(action_trace& trace) { - auto it = find_if(current_trx_actions.begin(), current_trx_actions.end(), [&] (const auto& dat) { - return dat.trx_id == trx_id; - }); - - if(it != current_trx_actions.end()) - return; - - db_action_trac dat; - dat.trx_id = trx_id; - dat.op_size = 0; - - current_trx_actions.emplace_back(dat); - is_undo_state = false; - - ilog("on_pre_apply_action trx: ${trxid}", ("trxid", trx_id)); + current_action.trx_id = trace.trx_id; + current_action.block_num = trace.block_num; + current_action.trace = chain_plug->chain().to_variant_with_abi(trace, abi_serializer_max_time); + current_action_index++; + reversible_actions.emplace_back(current_action); + current_action.ops.clear(); + + //ilog("on_applied_action: ${da}", ("da", *current_action)); } } // namespace eosio \ No newline at end of file From 5d75b134d56c46efdf3d67a736bbc04e86c511c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Sun, 2 Dec 2018 14:12:37 +0000 Subject: [PATCH 55/56] adding block fork undo ops --- libraries/chain/controller.cpp | 13 ++- .../chain/include/eosio/chain/controller.hpp | 4 +- .../statetrack_plugin_impl.hpp | 46 +++++----- .../statetrack_plugin/statetrack_plugin.cpp | 10 +++ .../statetrack_plugin_impl.cpp | 89 +++++++++++++------ 5 files changed, 111 insertions(+), 51 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 9a6c4d0f958..01fdab28cfb 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -145,10 +145,11 @@ struct controller_impl { map unapplied_transactions; void pop_block() { + auto head_block_num = head->block_num; auto prev = fork_db.get_block( head->header.previous ); EOS_ASSERT( prev, block_validate_exception, "attempt to pop beyond last irreversible block" ); - if( const auto* b = reversible_blocks.find(head->block_num) ) + if( const auto* b = reversible_blocks.find(head_block_num) ) { reversible_blocks.remove( *b ); } @@ -158,9 +159,13 @@ struct controller_impl { for( const auto& t : head->trxs ) unapplied_transactions[t->signed_id] = t; } + + emit( self.pre_undo_block, head_block_num ); + head = prev; db.undo(); + emit( self.post_undo_block, head_block_num ); } @@ -383,7 +388,13 @@ struct controller_impl { ("db",db.revision())("head",head->block_num) ); } while( db.revision() > head->block_num ) { + auto rev_block_num = db.revision(); + + emit( self.pre_undo_block, rev_block_num ); + db.undo(); + + emit( self.post_undo_block, rev_block_num ); } ilog( "database initialized with hash: ${hash}", ("hash", calculate_integrity_hash())); diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index f6deacbc584..d570e7ae2b6 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -253,7 +253,9 @@ namespace eosio { namespace chain { signal accepted_confirmation; signal bad_alloc; - signal applied_action; + signal applied_action; + signal pre_undo_block; + signal post_undo_block; /* signal pre_apply_block; diff --git a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp index 87c49838548..5866cdaa8b8 100644 --- a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp +++ b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp @@ -65,17 +65,18 @@ struct db_rev struct db_op { - int64_t oid; - uint64_t id; - uint64_t actionid; - - op_type_enum op_type; - account_name code; - fc::string scope; - table_name table; - account_name payer; - - fc::variant value; + int64_t oid; + uint64_t id; + block_num_type block_num; + uint64_t actionid; + + op_type_enum op_type; + account_name code; + fc::string scope; + table_name table; + account_name payer; + + fc::variant value; }; struct db_action @@ -92,6 +93,11 @@ struct db_transaction { std::vector actions; }; +struct db_undo_block { + block_num_type block_num; + std::vector ops; +}; + struct filter_entry { name code; @@ -121,7 +127,9 @@ typedef multi_index_container< ordered_unique, member >, ordered_non_unique, member > + &db_op::actionid> >, + ordered_non_unique, member > > > db_op_container; @@ -173,6 +181,8 @@ class statetrack_plugin_impl void on_accepted_block(const block_state_ptr &bsp); void on_irreversible_block(const block_state_ptr &bsp); void on_applied_action(action_trace& trace); + void on_pre_undo_block(block_num_type block_num); + void on_post_undo_block(block_num_type block_num); template void create_index_events(const database &db) { @@ -235,16 +245,12 @@ class statetrack_plugin_impl uint64_t current_action_index = 0; db_action current_action; + block_num_type current_undo_block_num = 0; + db_undo_block current_undo_block; db_action_container reversible_actions; db_op_container reversible_ops; - std::map po_ops; - std::map plo_ops; - std::map tio_ops; - std::map kvo_ops; - - std::map cached_traces; std::map> blacklist_actions; std::set filter_on; @@ -261,8 +267,8 @@ class statetrack_plugin_impl FC_REFLECT_ENUM(eosio::op_type_enum, (TABLE_REMOVE)(ROW_CREATE)(ROW_MODIFY)(ROW_REMOVE)(TRX_ACTION)(REV_UNDO)(REV_COMMIT)) FC_REFLECT(eosio::db_account, (name)(vm_type)(vm_version)(privileged)(last_code_update)(code_version)(creation_date)) -FC_REFLECT(eosio::db_op, (oid)(id)(op_type)(code)(scope)(table)(payer)(actionid)(value)) +FC_REFLECT(eosio::db_op, (oid)(id)(op_type)(code)(scope)(table)(payer)(block_num)(actionid)(value)) FC_REFLECT(eosio::db_rev, (op_type)(revision)) FC_REFLECT(eosio::db_action, (trx_id)(trace)(ops)) FC_REFLECT(eosio::db_transaction, (trx_id)(actions)) - +FC_REFLECT(eosio::db_undo_block, (block_num)(ops)) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index 474e76eaef0..4dac77d4255 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -128,6 +128,16 @@ void statetrack_plugin::plugin_initialize(const variables_map &options) fc::optional(chain.applied_action.connect([&](action_trace& trace){ my->on_applied_action(trace); }))); + + my->connections.emplace_back( + fc::optional(chain.pre_undo_block.connect([&](block_num_type block_num){ + my->on_pre_undo_block(block_num); + }))); + + my->connections.emplace_back( + fc::optional(chain.post_undo_block.connect([&](block_num_type block_num){ + my->on_post_undo_block(block_num); + }))); } FC_LOG_AND_RETHROW() } diff --git a/plugins/statetrack_plugin/statetrack_plugin_impl.cpp b/plugins/statetrack_plugin/statetrack_plugin_impl.cpp index 0c3eccc4d04..b15476af4db 100644 --- a/plugins/statetrack_plugin/statetrack_plugin_impl.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin_impl.cpp @@ -123,7 +123,13 @@ db_op statetrack_plugin_impl::get_db_op(const database &db, const account_object op.oid = co.id._id; op.op_type = op_type; - op.actionid = current_action_index; + + if(is_undo_state) { + op.block_num = current_undo_block_num; + } + else { + op.actionid = current_action_index; + } op.code = system; op.scope = system.to_string(); @@ -151,8 +157,14 @@ db_op statetrack_plugin_impl::get_db_op(const database &db, const permission_obj op.oid = po.id._id; op.op_type = op_type; - op.actionid = current_action_index; - + + if(is_undo_state) { + op.block_num = current_undo_block_num; + } + else { + op.actionid = current_action_index; + } + op.code = system; op.scope = system.to_string(); op.table = N(permissions); @@ -169,8 +181,14 @@ db_op statetrack_plugin_impl::get_db_op(const database &db, const permission_lin op.oid = plo.id._id; op.op_type = op_type; - op.actionid = current_action_index; - + + if(is_undo_state) { + op.block_num = current_undo_block_num; + } + else { + op.actionid = current_action_index; + } + op.code = system; op.scope = system.to_string(); op.table = N(permission_links); @@ -186,7 +204,13 @@ db_op statetrack_plugin_impl::get_db_op(const database &db, const table_id_objec op.oid = kvo.id._id; op.id = kvo.primary_key; op.op_type = op_type; - op.actionid = current_action_index; + + if(is_undo_state) { + op.block_num = current_undo_block_num; + } + else { + op.actionid = current_action_index; + } op.code = tio.code; op.scope = scope_sym_to_string(tio.scope); @@ -222,8 +246,14 @@ void statetrack_plugin_impl::on_applied_table(const database &db, const table_id op.oid = tio.id._id; op.op_type = op_type; - op.actionid = current_action_index; + if(is_undo_state) { + op.block_num = current_undo_block_num; + } + else { + op.actionid = current_action_index; + } + op.code = tio.code; op.scope = scope_sym_to_string(tio.scope); op.table = tio.table; @@ -251,28 +281,6 @@ void statetrack_plugin_impl::on_applied_op(const database &db, const account_obj { auto op = get_db_op(db, co, op_type); - //TODO op optimizations - // switch(op_type) { - // case op_type_enum::ROW_REMOVE: - // if(co_newids.get.count(co.id._id)) { - // co_newids.get.erase(co.id._id); - // return; - // } - // else if(co_old_values.get.count(co.id._id)) { - // co_old_values.get.erase(co.id._id); - // } - // break; - // case op_type_enum::ROW_MODIFY: - // if(co_newids.get.count(co.id._id)) { - // op.op_type = op_type_enum::ROW_CREATE; - // co_newids.get.erase(co.id._id); - // } - // else if(co_old_values.get.count(co.id._id)) { - // co_old_values.get.erase(co.id._id); - // } - // break; - // } - current_action.ops.emplace_back(op); fc::string data = fc::json::to_string(op); @@ -447,4 +455,27 @@ void statetrack_plugin_impl::on_applied_action(action_trace& trace) //ilog("on_applied_action: ${da}", ("da", *current_action)); } +void statetrack_plugin_impl::on_pre_undo_block(block_num_type block_num) +{ + is_undo_state = true; + current_undo_block_num = block_num; + + auto block_ops_range = reversible_ops.get().equal_range(current_undo_block_num); + reversible_ops.get().erase(block_ops_range.first, block_ops_range.second); +} + +void statetrack_plugin_impl::on_post_undo_block(block_num_type block_num) +{ + is_undo_state = false; + + auto block_ops_range = reversible_ops.get().equal_range(current_undo_block_num); + for (auto& op : boost::make_iterator_range(block_ops_range)) { + current_undo_block.ops.emplace_back(op); + } + + ilog("on_post_undo_block: ${block}", ("block", current_undo_block)); + + current_undo_block.ops.clear(); +} + } // namespace eosio \ No newline at end of file From 5eb50d1807ab6091cec0d5949c420fb788902ef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rio=20Silva?= Date: Mon, 3 Dec 2018 22:43:26 +0000 Subject: [PATCH 56/56] work on adding zmq sending of trx and blocks --- libraries/chain/controller.cpp | 14 +-- .../chain/include/eosio/chain/controller.hpp | 4 +- .../statetrack_plugin_impl.hpp | 31 ++--- .../statetrack_plugin/statetrack_plugin.cpp | 8 +- .../statetrack_plugin_impl.cpp | 112 ++++++++---------- 5 files changed, 73 insertions(+), 96 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 01fdab28cfb..daf2be79b68 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -145,11 +145,11 @@ struct controller_impl { map unapplied_transactions; void pop_block() { - auto head_block_num = head->block_num; + auto local_head = head; auto prev = fork_db.get_block( head->header.previous ); EOS_ASSERT( prev, block_validate_exception, "attempt to pop beyond last irreversible block" ); - if( const auto* b = reversible_blocks.find(head_block_num) ) + if( const auto* b = reversible_blocks.find(head->block_num) ) { reversible_blocks.remove( *b ); } @@ -160,12 +160,12 @@ struct controller_impl { unapplied_transactions[t->signed_id] = t; } - emit( self.pre_undo_block, head_block_num ); + emit( self.pre_undo_block, local_head ); head = prev; db.undo(); - emit( self.post_undo_block, head_block_num ); + emit( self.post_undo_block, local_head ); } @@ -388,13 +388,7 @@ struct controller_impl { ("db",db.revision())("head",head->block_num) ); } while( db.revision() > head->block_num ) { - auto rev_block_num = db.revision(); - - emit( self.pre_undo_block, rev_block_num ); - db.undo(); - - emit( self.post_undo_block, rev_block_num ); } ilog( "database initialized with hash: ${hash}", ("hash", calculate_integrity_hash())); diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index d570e7ae2b6..1b79bc7f618 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -254,8 +254,8 @@ namespace eosio { namespace chain { signal bad_alloc; signal applied_action; - signal pre_undo_block; - signal post_undo_block; + signal pre_undo_block; + signal post_undo_block; /* signal pre_apply_block; diff --git a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp index 5866cdaa8b8..4b92a56d2bf 100644 --- a/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp +++ b/plugins/statetrack_plugin/include/eosio/statetrack_plugin/statetrack_plugin_impl.hpp @@ -39,10 +39,7 @@ enum op_type_enum TABLE_REMOVE = 0, ROW_CREATE = 1, ROW_MODIFY = 2, - ROW_REMOVE = 3, - TRX_ACTION = 4, - REV_UNDO = 5, - REV_COMMIT = 6 + ROW_REMOVE = 3 }; struct db_account @@ -57,12 +54,6 @@ struct db_account block_timestamp_type creation_date; }; -struct db_rev -{ - op_type_enum op_type; - int64_t revision; -}; - struct db_op { int64_t oid; @@ -93,8 +84,13 @@ struct db_transaction { std::vector actions; }; -struct db_undo_block { +struct db_block { block_num_type block_num; + block_id_type block_id; + bool irreversible; +}; + +struct db_undo_block : db_block { std::vector ops; }; @@ -164,8 +160,6 @@ class statetrack_plugin_impl db_op get_db_op(const database &db, const permission_link_object &plo, op_type_enum op_type); db_op get_db_op(const database &db, const table_id_object &tio, const key_value_object &kvo, op_type_enum op_type, bool json = true); - db_rev get_db_rev(const int64_t revision, op_type_enum op_type); - //generic_index state tables void on_applied_table(const database &db, const table_id_object &tio, op_type_enum op_type); //generic_index op @@ -173,7 +167,6 @@ class statetrack_plugin_impl void on_applied_op(const database &db, const permission_object &po, op_type_enum op_type); void on_applied_op(const database &db, const permission_link_object &plo, op_type_enum op_type); void on_applied_op(const database &db, const key_value_object &kvo, op_type_enum op_type); - void on_applied_rev(const int64_t revision, op_type_enum op_type); //generic_index undo void on_applied_undo(const int64_t revision); //blocks and transactions @@ -181,8 +174,8 @@ class statetrack_plugin_impl void on_accepted_block(const block_state_ptr &bsp); void on_irreversible_block(const block_state_ptr &bsp); void on_applied_action(action_trace& trace); - void on_pre_undo_block(block_num_type block_num); - void on_post_undo_block(block_num_type block_num); + void on_pre_undo_block(const block_state_ptr& bsp); + void on_post_undo_block(const block_state_ptr& bsp); template void create_index_events(const database &db) { @@ -265,10 +258,10 @@ class statetrack_plugin_impl } // namespace eosio -FC_REFLECT_ENUM(eosio::op_type_enum, (TABLE_REMOVE)(ROW_CREATE)(ROW_MODIFY)(ROW_REMOVE)(TRX_ACTION)(REV_UNDO)(REV_COMMIT)) +FC_REFLECT_ENUM(eosio::op_type_enum, (TABLE_REMOVE)(ROW_CREATE)(ROW_MODIFY)(ROW_REMOVE)) FC_REFLECT(eosio::db_account, (name)(vm_type)(vm_version)(privileged)(last_code_update)(code_version)(creation_date)) FC_REFLECT(eosio::db_op, (oid)(id)(op_type)(code)(scope)(table)(payer)(block_num)(actionid)(value)) -FC_REFLECT(eosio::db_rev, (op_type)(revision)) FC_REFLECT(eosio::db_action, (trx_id)(trace)(ops)) FC_REFLECT(eosio::db_transaction, (trx_id)(actions)) -FC_REFLECT(eosio::db_undo_block, (block_num)(ops)) +FC_REFLECT(eosio::db_block, (block_num)(block_id)(irreversible)) +FC_REFLECT_DERIVED(eosio::db_undo_block, (eosio::db_block), (ops)) diff --git a/plugins/statetrack_plugin/statetrack_plugin.cpp b/plugins/statetrack_plugin/statetrack_plugin.cpp index 4dac77d4255..f583ae387a5 100644 --- a/plugins/statetrack_plugin/statetrack_plugin.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin.cpp @@ -130,13 +130,13 @@ void statetrack_plugin::plugin_initialize(const variables_map &options) }))); my->connections.emplace_back( - fc::optional(chain.pre_undo_block.connect([&](block_num_type block_num){ - my->on_pre_undo_block(block_num); + fc::optional(chain.pre_undo_block.connect([&](const block_state_ptr& bsp){ + my->on_pre_undo_block(bsp); }))); my->connections.emplace_back( - fc::optional(chain.post_undo_block.connect([&](block_num_type block_num){ - my->on_post_undo_block(block_num); + fc::optional(chain.post_undo_block.connect([&](const block_state_ptr& bsp){ + my->on_post_undo_block(bsp); }))); } FC_LOG_AND_RETHROW() diff --git a/plugins/statetrack_plugin/statetrack_plugin_impl.cpp b/plugins/statetrack_plugin/statetrack_plugin_impl.cpp index b15476af4db..f35ac201bcf 100644 --- a/plugins/statetrack_plugin/statetrack_plugin_impl.cpp +++ b/plugins/statetrack_plugin/statetrack_plugin_impl.cpp @@ -225,12 +225,6 @@ db_op statetrack_plugin_impl::get_db_op(const database &db, const table_id_objec return op; } -db_rev statetrack_plugin_impl::get_db_rev(const int64_t revision, op_type_enum op_type) -{ - db_rev rev = {op_type, revision}; - return rev; -} - void statetrack_plugin_impl::on_applied_table(const database &db, const table_id_object &tio, op_type_enum op_type) { if (sender_socket == nullptr) @@ -258,8 +252,6 @@ void statetrack_plugin_impl::on_applied_table(const database &db, const table_id op.scope = scope_sym_to_string(tio.scope); op.table = tio.table; - //TODO op optimizations - current_action.ops.emplace_back(op); fc::string data = fc::json::to_string(op); @@ -302,8 +294,6 @@ void statetrack_plugin_impl::on_applied_op(const database &db, const permission_ { auto op = get_db_op(db, po, op_type); - //TODO op optimizations - current_action.ops.emplace_back(op); fc::string data = fc::json::to_string(op); @@ -325,8 +315,6 @@ void statetrack_plugin_impl::on_applied_op(const database &db, const permission_ { auto op = get_db_op(db, plo, op_type); - //TODO op optimizations - current_action.ops.emplace_back(op); fc::string data = fc::json::to_string(op); @@ -354,8 +342,6 @@ void statetrack_plugin_impl::on_applied_op(const database &db, const key_value_o { auto op = get_db_op(db, *tio_ptr, kvo, op_type); - //TODO op optimizations - current_action.ops.emplace_back(op); fc::string data = fc::json::to_string(op); @@ -363,66 +349,58 @@ void statetrack_plugin_impl::on_applied_op(const database &db, const key_value_o } } -void statetrack_plugin_impl::on_applied_rev(const int64_t revision, op_type_enum op_type) -{ - if (sender_socket == nullptr) - return; - - fc::string data = fc::json::to_string(get_db_rev(revision, op_type)); - ilog("STATETRACK ${op_type}: ${data}", ("op_type", op_type)("data", data)); - //send_zmq_msg(data); -} - void statetrack_plugin_impl::on_applied_undo(const int64_t revision) { - if (sender_socket == nullptr) - return; - is_undo_state = true; - - fc::string data = fc::json::to_string(get_db_rev(revision, op_type_enum::REV_UNDO)); - ilog("STATETRACK ${op_type}: ${data}", ("op_type", op_type_enum::REV_UNDO)("data", data)); - //send_zmq_msg(data); } void statetrack_plugin_impl::on_applied_transaction(const transaction_trace_ptr &ttp) { - if (ttp->receipt) { + if (sender_socket == nullptr || !ttp->receipt) + return; - db_transaction trx; + db_transaction trx; - trx.trx_id = ttp->id; + trx.trx_id = ttp->id; - auto trx_action_range = reversible_actions.get().equal_range(trx.trx_id); - for (auto& action : boost::make_iterator_range(trx_action_range)) { - - trx.actions.emplace_back(action); - auto newact = trx.actions.back(); + auto trx_action_range = reversible_actions.get().equal_range(trx.trx_id); + for (auto& action : boost::make_iterator_range(trx_action_range)) { + + trx.actions.emplace_back(action); + auto newact = trx.actions.back(); - auto action_op_range = reversible_ops.get().equal_range(action.actionid); - for (auto& op : boost::make_iterator_range(action_op_range)) { - newact.ops.emplace_back(op); - } + auto action_op_range = reversible_ops.get().equal_range(action.actionid); + for (auto& op : boost::make_iterator_range(action_op_range)) { + newact.ops.emplace_back(op); } + } - //TODO send zmq transaction - //send_zmq_msg(data); + fc::string data = fc::json::to_string(trx); + ilog("STATETRACK applied transaction: ${trx}", ("trx", data)); - ilog("STATETRACK applied transaction: ${trx}", ("trx", trx)); - } - else { - auto trx_action_range = reversible_actions.get().equal_range(ttp->id); - for (auto& action : boost::make_iterator_range(trx_action_range)) { - auto action_op_range = reversible_ops.get().equal_range(action.actionid); - reversible_ops.get().erase(action_op_range.first, action_op_range.second); - } - reversible_actions.get().erase(trx_action_range.first, trx_action_range.second); - } + send_zmq_msg(data); } void statetrack_plugin_impl::on_accepted_block(const block_state_ptr &bsp) { + if (sender_socket == nullptr) + return; + + db_block block; + block.block_num = bsp->block_num; + block.block_id = bsp->id; + + fc::string data = fc::json::to_string(block); + ilog("STATETRACK accepted block: ${data}", ("data", data)); + send_zmq_msg(data); + + auto block_action_range = reversible_actions.get().equal_range(bsp->block_num); + for (auto& action : boost::make_iterator_range(block_action_range)) { + auto action_op_range = reversible_ops.get().equal_range(action.actionid); + reversible_ops.get().erase(action_op_range.first, action_op_range.second); + } + reversible_actions.get().erase(block_action_range.first, block_action_range.second); } void statetrack_plugin_impl::on_irreversible_block(const block_state_ptr &bsp) @@ -430,10 +408,15 @@ void statetrack_plugin_impl::on_irreversible_block(const block_state_ptr &bsp) if (sender_socket == nullptr) return; - fc::string data = fc::json::to_string(get_db_rev(bsp->block_num, op_type_enum::REV_COMMIT)); + db_block block; + block.block_num = bsp->block_num; + block.block_id = bsp->id; + block.irreversible = true; + + fc::string data = fc::json::to_string(block); ilog("STATETRACK irreversible block: ${data}", ("data", data)); - //send_zmq_msg(data); + send_zmq_msg(data); auto block_action_range = reversible_actions.get().equal_range(bsp->block_num); for (auto& action : boost::make_iterator_range(block_action_range)) { @@ -455,17 +438,21 @@ void statetrack_plugin_impl::on_applied_action(action_trace& trace) //ilog("on_applied_action: ${da}", ("da", *current_action)); } -void statetrack_plugin_impl::on_pre_undo_block(block_num_type block_num) +void statetrack_plugin_impl::on_pre_undo_block(const block_state_ptr& bsp) { is_undo_state = true; - current_undo_block_num = block_num; + current_undo_block.block_num = bsp->block_num; + current_undo_block.block_id = bsp->id; auto block_ops_range = reversible_ops.get().equal_range(current_undo_block_num); reversible_ops.get().erase(block_ops_range.first, block_ops_range.second); } -void statetrack_plugin_impl::on_post_undo_block(block_num_type block_num) +void statetrack_plugin_impl::on_post_undo_block(const block_state_ptr&) { + if (sender_socket == nullptr) + return; + is_undo_state = false; auto block_ops_range = reversible_ops.get().equal_range(current_undo_block_num); @@ -473,7 +460,10 @@ void statetrack_plugin_impl::on_post_undo_block(block_num_type block_num) current_undo_block.ops.emplace_back(op); } - ilog("on_post_undo_block: ${block}", ("block", current_undo_block)); + fc::string data = fc::json::to_string(current_undo_block); + ilog("STATETRACK on_post_undo_block: ${block}", ("block", data)); + + send_zmq_msg(data); current_undo_block.ops.clear(); }